From 2a540206a74af8d38a01aaa5a37adc1008cad6ca Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Tue, 19 Aug 2014 13:57:00 -0300 Subject: [PATCH 001/127] fix xmliter namespace on selected node --- scrapy/utils/iterators.py | 22 +++++++++++++++------- tests/test_utils_iterators.py | 6 +++++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 150b077ae..11b873f2e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -20,19 +20,27 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) + DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S) HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) + END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S) + NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S) text = _body_or_str(obj) - header_start = re.search(HEADER_START_RE, text) - header_start = header_start.group(1).strip() if header_start else '' - header_end = re_rsearch(HEADER_END_RE, text) - header_end = text[header_end[1]:].strip() if header_end else '' + document_header = re.search(DOCUMENT_HEADER_RE, text) + document_header = document_header.group().strip() if document_header else '' + header_end_idx = re_rsearch(HEADER_END_RE, text) + header_end = text[header_end_idx[1]:].strip() if header_end_idx else '' + namespaces = {} + if header_end: + for tagname in reversed(re.findall(END_TAG_RE, header_end)): + tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + if tag: + namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): - nodetext = header_start + match.group() + header_end - yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] + nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + yield Selector(text=nodetext, type='xml') def csviter(obj, delimiter=None, headers=None, encoding=None): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index fe53f831f..8b5941605 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -61,7 +61,6 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'item') - node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) @@ -74,6 +73,11 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').extract(), []) self.assertEqual(node.xpath('price/text()').extract(), []) + my_iter = self.xmliter(response, 'g:image_link') + node = next(my_iter) + node.register_namespace('g', 'http://base.google.com/ns/1.0') + self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_exception(self): body = u"""onetwo""" From ad6075440c0285d903ec7238354d093ea300b8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Oct 2019 19:00:03 +0200 Subject: [PATCH 002/127] Fix references to Python types in parameter type fields --- docs/topics/contracts.rst | 2 +- docs/topics/email.rst | 10 ++++----- docs/topics/exporters.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/link-extractors.rst | 14 ++++++------- docs/topics/loaders.rst | 6 +++--- docs/topics/request-response.rst | 36 ++++++++++++++++---------------- scrapy/crawler.py | 2 +- scrapy/robotstxt.py | 4 ++-- scrapy/settings/__init__.py | 33 ++++++++++++----------------- scrapy/signalmanager.py | 2 +- 11 files changed, 53 insertions(+), 60 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 62f9a743b..15443f4cc 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -85,7 +85,7 @@ override three methods: .. class:: Contract(method, \*args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 949cdc638..73b1bdc3b 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -63,10 +63,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :type smtpport: int :param smtptls: enforce using SMTP STARTTLS - :type smtptls: boolean + :type smtptls: bool :param smtpssl: enforce using a secure SSL connection - :type smtpssl: boolean + :type smtpssl: bool .. classmethod:: from_settings(settings) @@ -81,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: str or list of str + :type to: str or list :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: str or list of str + :type cc: str or list :param body: the e-mail body :type body: str @@ -97,7 +97,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. appear on the e-mail's attachment, ``mimetype`` is the mimetype of the attachment and ``file_object`` is a readable file object with the contents of the attachment - :type attachs: iterable + :type attachs: collections.abc.Iterable :param mimetype: the MIME type of the e-mail :type mimetype: str diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index a698a6a4e..da304922d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -300,7 +300,7 @@ CsvItemExporter :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. - :type include_headers_line: boolean + :type include_headers_line: bool :param join_multivalued: The char (or chars) that will be used for joining multi-valued fields, if found. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 8278e9849..657f1cc61 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. :param ignore: if given, all objects from the specified class (or tuple of classes) will be ignored. - :type ignore: class or classes tuple + :type ignore: type or tuple .. function:: get_oldest(class_name) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 713a94e10..13b9ad7a5 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -59,13 +59,13 @@ LxmlLinkExtractor :param allow: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be extracted. If not given (or empty), it will match all links. - :type allow: a regular expression (or list of) + :type allow: str or list :param deny: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be excluded (ie. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. - :type deny: a regular expression (or list of) + :type deny: str or list :param allow_domains: a single value or a list of string containing domains which will be considered for extracting the links @@ -97,7 +97,7 @@ LxmlLinkExtractor that the link's text must match in order to be extracted. If not given (or empty), it will match all links. If a list of regular expressions is given, the link will be extracted if it matches at least one. - :type restrict_text: a regular expression (or list of) + :type restrict_text: str or list :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. @@ -115,11 +115,11 @@ LxmlLinkExtractor different for requests with canonicalized and raw URLs. If you're using LinkExtractor to follow links it is more robust to keep the default ``canonicalize=False``. - :type canonicalize: boolean + :type canonicalize: bool :param unique: whether duplicate filtering should be applied to extracted links. - :type unique: boolean + :type unique: bool :param process_value: a function which receives each value extracted from the tag and attributes scanned and can modify the value and return a @@ -141,7 +141,7 @@ LxmlLinkExtractor if m: return m.group(1) - :type process_value: callable + :type process_value: collections.abc.Callable :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces @@ -150,6 +150,6 @@ LxmlLinkExtractor elements, etc., so LinkExtractor strips space chars by default. Set ``strip=False`` to turn it off (e.g. if you're extracting urls from elements or attributes which allow leading/trailing whitespaces). - :type strip: boolean + :type strip: bool .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 1c2f1da4d..4137fdd24 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -320,7 +320,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the given value using :meth:`~scrapy.utils.misc.extract_regex` method, applied before processors - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -365,7 +365,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected XPath region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -408,7 +408,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected CSS region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 727c67482..2f99a72f5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,7 +31,7 @@ Request objects a :class:`Response`. :param url: the URL of this request - :type url: string + :type url: str :param callback: the function that will be called with the response of this request (once its downloaded) as its first parameter. For more information @@ -40,10 +40,10 @@ Request objects :meth:`~scrapy.spiders.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. - :type callback: callable + :type callback: collections.abc.Callable :param method: the HTTP method of this request. Defaults to ``'GET'``. - :type method: string + :type method: str :param meta: the initial values for the :attr:`Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. @@ -54,7 +54,7 @@ Request objects ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). - :type body: str or unicode + :type body: str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -105,7 +105,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to ``str`` (if given as ``unicode``). - :type encoding: string + :type encoding: str :param priority: the priority of this request (defaults to ``0``). The priority is used by the scheduler to define the order used to process @@ -117,7 +117,7 @@ Request objects the scheduler. This is used when you want to perform an identical request multiple times, to ignore the duplicates filter. Use it with care, or you will get into crawling loops. Default to ``False``. - :type dont_filter: boolean + :type dont_filter: bool :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed @@ -125,7 +125,7 @@ Request objects as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. - :type errback: callable + :type errback: collections.abc.Callable :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -407,7 +407,7 @@ fields with form data from :class:`Response` objects. :param formdata: is a dictionary (or iterable of (key, value) tuples) containing HTML Form data which will be url-encoded and assigned to the body of the request. - :type formdata: dict or iterable of tuples + :type formdata: dict or collections.abc.Iterable The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: @@ -439,20 +439,20 @@ fields with form data from :class:`Response` objects. :type response: :class:`Response` object :param formname: if given, the form with name attribute set to this value will be used. - :type formname: string + :type formname: str :param formid: if given, the form with id attribute set to this value will be used. - :type formid: string + :type formid: str :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: string + :type formxpath: str :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: string + :type formcss: str :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. - :type formnumber: integer + :type formnumber: int :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is @@ -470,7 +470,7 @@ fields with form data from :class:`Response` objects. :param dont_click: If True, the form data will be submitted without clicking in any element. - :type dont_click: boolean + :type dont_click: bool The other parameters of this class method are passed directly to the :class:`FormRequest` constructor. @@ -558,7 +558,7 @@ dealing with JSON requests. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be set to ``'POST'`` automatically. - :type data: JSON serializable object + :type data: object :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data into JSON format. @@ -587,10 +587,10 @@ Response objects downloaded (by the Downloader) and fed to the Spiders for processing. :param url: the URL of this response - :type url: string + :type url: str :param status: the HTTP status of the response. Defaults to ``200``. - :type status: integer + :type status: int :param headers: the headers of this response. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). @@ -730,7 +730,7 @@ TextResponse objects body, it will be encoded using this encoding (remember the body attribute is always a string). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. - :type encoding: string + :type encoding: str :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ded3c082b..84acf543f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -292,7 +292,7 @@ class CrawlerProcess(CrawlerRunner): If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. - :param boolean stop_after_crawl: stop or not the reactor when all + :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..7faad308a 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -43,10 +43,10 @@ class RobotParser(with_metaclass(ABCMeta)): """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: string + :type url: str :param user_agent: User agent - :type user_agent: string + :type user_agent: str """ pass diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28c7940d..95c02021e 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -105,10 +105,9 @@ class BaseSettings(MutableMapping): Get a setting value without affecting its original type. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return self[name] if self[name] is not None else default @@ -123,10 +122,9 @@ class BaseSettings(MutableMapping): ``'0'`` will return ``False`` when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ got = self.get(name, default) try: @@ -145,10 +143,9 @@ class BaseSettings(MutableMapping): Get a setting value as an int. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return int(self.get(name, default)) @@ -157,10 +154,9 @@ class BaseSettings(MutableMapping): Get a setting value as a float. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return float(self.get(name, default)) @@ -173,10 +169,9 @@ class BaseSettings(MutableMapping): ``'one,two'`` will return a list ['one', 'two'] when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -194,10 +189,9 @@ class BaseSettings(MutableMapping): and losing all information about priority and mutability. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -209,7 +203,7 @@ class BaseSettings(MutableMapping): counterpart. :param name: name of the dictionary-like setting - :type name: string + :type name: str """ compbs = BaseSettings() compbs.update(self[name + '_BASE']) @@ -222,7 +216,7 @@ class BaseSettings(MutableMapping): the given ``name`` does not exist. :param name: the setting name - :type name: string + :type name: str """ if name not in self: return None @@ -252,14 +246,13 @@ class BaseSettings(MutableMapping): otherwise they won't have any effect. :param name: the setting name - :type name: string + :type name: str :param value: the value to associate with the setting - :type value: any :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() priority = get_settings_priority(priority) @@ -283,11 +276,11 @@ class BaseSettings(MutableMapping): uppercase variable of ``module`` with the provided ``priority``. :param module: the module or the path of the module - :type module: module object or string + :type module: types.ModuleType or str :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(module, six.string_types): @@ -316,7 +309,7 @@ class BaseSettings(MutableMapping): :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(values, six.string_types): diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..c24b16fcb 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -17,7 +17,7 @@ class SignalManager(object): section. :param receiver: the function to be connected - :type receiver: callable + :type receiver: collections.abc.Callable :param signal: the signal to connect to :type signal: object From 5479e7ecc7d30424dd2f3d9bbfb18abca765be92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:44 +0200 Subject: [PATCH 003/127] Indicate that lists of emails may be provided as a single string or as a list of strings --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 73b1bdc3b..284849c56 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -80,13 +80,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. - :param to: the e-mail recipients + :param to: the e-mail recipients as a string or as a list of strings :type to: str or list :param subject: the subject of the e-mail :type subject: str - :param cc: the e-mails to CC + :param cc: the e-mails to CC as a string or as a list of strings :type cc: str or list :param body: the e-mail body From d96b9f860b01256a84bda641b190b423eb4910b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:59 +0200 Subject: [PATCH 004/127] Use object as type for parameters that allow any value --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 95c02021e..d1a5093a6 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -108,6 +108,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return self[name] if self[name] is not None else default @@ -125,6 +126,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ got = self.get(name, default) try: @@ -146,6 +148,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return int(self.get(name, default)) @@ -157,6 +160,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return float(self.get(name, default)) @@ -172,6 +176,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -192,6 +197,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -249,6 +255,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting + :type default: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From 0946eb335a285e1f210ba1185a564699f53b17d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 17:56:21 +0100 Subject: [PATCH 005/127] =?UTF-8?q?Port=20code=20from=20Twisted=E2=80=99s?= =?UTF-8?q?=20deprecated=20HTTPClientFactory=20into=20ScrapyHTTPClientFact?= =?UTF-8?q?ory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/webclient.py | 90 ++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3fe13414a..16fd214a3 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,9 +1,9 @@ from time import time from six.moves.urllib.parse import urlparse, urlunparse, urldefrag -from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached @@ -93,18 +93,30 @@ class ScrapyHTTPPageGetter(HTTPClient): (self.factory.url, self.factory.timeout))) -class ScrapyHTTPClientFactory(HTTPClientFactory): - """Scrapy implementation of the HTTPClientFactory overwriting the - setUrl method to make use of our Url object that cache the parse - result. - """ +class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter + waiting = 1 noisy = False followRedirect = False afterFoundGet = False + def _build_response(self, body, request): + request.meta['download_latency'] = self.headers_time-self.start_time + status = int(self.status) + headers = Headers(self.response_headers) + respcls = responsetypes.from_args(headers=headers, url=self._url) + return respcls(url=self._url, status=status, headers=headers, body=body) + + def _set_connection_attributes(self, request): + parsed = urlparse_cached(request) + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) + proxy = request.meta.get('proxy') + if proxy: + self.scheme, _, self.host, self.port, _ = _parse(proxy) + self.path = self.url + def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface @@ -139,21 +151,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): elif self.method == b'POST': self.headers['Content-Length'] = 0 - def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time-self.start_time - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.url) - def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) - proxy = request.meta.get('proxy') - if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) - self.path = self.url + def _cancelTimeout(self, result, timeoutCall): + if timeoutCall.active(): + timeoutCall.cancel() + return result + + def buildProtocol(self, addr): + p = ClientFactory.buildProtocol(self, addr) + p.followRedirect = self.followRedirect + p.afterFoundGet = self.afterFoundGet + if self.timeout: + timeoutCall = reactor.callLater(self.timeout, p.timeout) + self.deferred.addBoth(self._cancelTimeout, timeoutCall) + return p def gotHeaders(self, headers): self.headers_time = time() self.response_headers = headers + + def gotStatus(self, version, status, message): + """ + Set the status of the request on us. + @param version: The HTTP version. + @type version: L{bytes} + @param status: The HTTP status code, an integer represented as a + bytestring. + @type status: L{bytes} + @param message: The HTTP status message. + @type message: L{bytes} + """ + self.version, self.status, self.message = version, status, message + + def page(self, page): + if self.waiting: + self.waiting = 0 + self.deferred.callback(page) + + def noPage(self, reason): + if self.waiting: + self.waiting = 0 + self.deferred.errback(reason) + + def clientConnectionFailed(self, _, reason): + """ + When a connection attempt fails, the request cannot be issued. If no + result has yet been provided to the result Deferred, provide the + connection failure reason as an error result. + """ + if self.waiting: + self.waiting = 0 + # If the connection attempt failed, there is nothing more to + # disconnect, so just fire that Deferred now. + self._disconnectedDeferred.callback(None) + self.deferred.errback(reason) From 42954d0df9a78e6641996fbf2eb76afdef9c1856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 20 Nov 2019 08:16:33 +0100 Subject: [PATCH 006/127] Mention that ScrapyHTTPClientFactory has Twisted code --- scrapy/core/downloader/webclient.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 16fd214a3..26726afd4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -93,6 +93,10 @@ class ScrapyHTTPPageGetter(HTTPClient): (self.factory.url, self.factory.timeout))) +# This class used to inherit from Twisted’s +# twisted.web.client.HTTPClientFactory. When that class was deprecated in +# Twisted (https://github.com/twisted/twisted/pull/643), we merged its +# non-overriden code into this class. class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter From 286fca733f23fa41165edcdcc7ab7593cc6b074f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 6 May 2020 16:20:33 +0200 Subject: [PATCH 007/127] Fix parameter name, broken by copy-pasting --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 8f6fd3e6a..99ffa0dc9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -248,7 +248,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting - :type default: object + :type value: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From bbd9d05dbff7abb6ab3a5ea575d75cfe88cb2ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 May 2020 11:44:43 +0200 Subject: [PATCH 008/127] request-response.rst: review type references around body mentions --- docs/topics/request-response.rst | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1638c202d..4fec70e13 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,10 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str + :param body: the request body. If a string is passed, it is converted to + bytes using *encoding*, which defaults to ``utf-8``. If not passed or + ``None`` is passed, an empty bytes array is stored. + :type body: bytes :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +104,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes if given as a string. :type encoding: str :param priority: the priority of this request (defaults to ``0``). @@ -159,7 +157,7 @@ Request objects .. attribute:: Request.body - A str that contains the request body. + The request body as bytes. This attribute is read-only. To change the body of a Request use :meth:`replace`. @@ -598,7 +596,7 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. To access the decoded text as str you can use + :param body: the response body. To access the decoded text as a string, use ``response.text`` from an encoding-aware :ref:`Response subclass `, such as :class:`TextResponse`. @@ -646,10 +644,10 @@ Response objects .. attribute:: Response.body - The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use - :attr:`TextResponse.text` (only available in :class:`TextResponse` - and subclasses). + The response body as bytes. + + If you want the body as a string, use :attr:`TextResponse.text` (only + available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -768,10 +766,10 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode - body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the - encoding will be looked up in the response headers and body instead. + response. If you create a :class:`TextResponse` object with a string as + body, it will be converted to bytes encoded using this encoding. If + *encoding* is ``None`` (default), the encoding will be looked up in the + response headers and body instead. :type encoding: str :class:`TextResponse` objects support the following attributes in addition @@ -779,7 +777,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -787,8 +785,8 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding + ``str(response.body)`` is not a correct way to convert the response + body into a string: you would be using the system default encoding (typically ``ascii``) instead of the response encoding. From c6746f0e381a44f3d66efad86a4aca87805138bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 9 May 2020 15:51:11 +0200 Subject: [PATCH 009/127] =?UTF-8?q?bytes=20array=20=E2=86=92=20bytes=20obj?= =?UTF-8?q?ect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4fec70e13..ad6c10b6e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -53,7 +53,7 @@ Request objects :param body: the request body. If a string is passed, it is converted to bytes using *encoding*, which defaults to ``utf-8``. If not passed or - ``None`` is passed, an empty bytes array is stored. + ``None`` is passed, an empty :class:`bytes` object is stored. :type body: bytes :param headers: the headers of this request. The dict values can be strings From e07708e3744fb26fa72042720b33deab59331cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 9 May 2020 15:54:31 +0200 Subject: [PATCH 010/127] request-response: update the consequences of str(b'') --- docs/topics/request-response.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ad6c10b6e..397632932 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -786,8 +786,7 @@ TextResponse objects .. note:: ``str(response.body)`` is not a correct way to convert the response - body into a string: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + body into a string: ``str(b'')`` returns ``"b''"``. .. attribute:: TextResponse.encoding From ece4fa6c7cf7fdce9e5e77f56b209941ad401f53 Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:21:45 +0000 Subject: [PATCH 011/127] Fix ignored testcase: boto is never installed --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 37384081a..21da9fdcd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -182,9 +182,9 @@ class S3FeedStorageTest(unittest.TestCase): create=True) def test_parse_credentials(self): try: - import boto # noqa: F401 + import botocore # noqa: F401 except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto") + raise unittest.SkipTest("S3FeedStorage requires botocore") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) From 234c8b8c50a2db0d6d71455c382957099f4c0dcc Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:31:56 +0000 Subject: [PATCH 012/127] Removing deprecated S3FeedStorage without AWS keys instancing. --- scrapy/extensions/feedexport.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72a34ae0d..e793c12dc 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,23 +94,10 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None): - # BEGIN Backward compatibility for initialising without keys (and - # without using from_crawler) - no_defaults = access_key is None and secret_key is None - if no_defaults: - from scrapy.utils.project import get_project_settings - settings = get_project_settings() - if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: - warnings.warn( - "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " - "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backward compatibility + no_keys = access_key is None and secret_key is None + if no_keys: + raise NotConfigured('%s is missing AWS credentials' % + self.__class__.__name__) u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key From 98e8086d1b3da8906d1395f453b1299b4e3b499e Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:21:09 +0000 Subject: [PATCH 013/127] Adapt S3FeedStorage testcase --- tests/test_feedexport.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 21da9fdcd..34a67fff3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,6 +24,7 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -176,10 +177,6 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - @mock.patch('scrapy.utils.project.get_project_settings', - new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key', - 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}), - create=True) def test_parse_credentials(self): try: import botocore # noqa: F401 @@ -205,12 +202,9 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backward compatibility for initialising without settings - with warnings.catch_warnings(record=True) as w: - storage = S3FeedStorage('s3://mybucket/export.csv') - self.assertEqual(storage.access_key, 'conf_key') - self.assertEqual(storage.secret_key, 'conf_secret') - self.assertTrue('without AWS keys' in str(w[-1].message)) + # Instantiate without credentials + with self.assertRaises(NotConfigured): + S3FeedStorage('s3://mybucket/export.csv') @defer.inlineCallbacks def test_store(self): From 2829cd4268d22a328acd64a6816c479b1fe21f39 Mon Sep 17 00:00:00 2001 From: nyov Date: Tue, 17 Mar 2020 10:19:13 +0000 Subject: [PATCH 014/127] Allow use without credentials --- scrapy/extensions/feedexport.py | 4 ---- tests/test_feedexport.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e793c12dc..68d6533d3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,10 +94,6 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None): - no_keys = access_key is None and secret_key is None - if no_keys: - raise NotConfigured('%s is missing AWS credentials' % - self.__class__.__name__) u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 34a67fff3..656bb515f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,7 +24,6 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -202,9 +201,6 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Instantiate without credentials - with self.assertRaises(NotConfigured): - S3FeedStorage('s3://mybucket/export.csv') @defer.inlineCallbacks def test_store(self): From 430d22e46ebb67aba39005365e2e68d834d8fb86 Mon Sep 17 00:00:00 2001 From: Artur Shellunts Date: Tue, 21 Jul 2020 23:39:04 +0200 Subject: [PATCH 015/127] Remove not used import warnings --- tests/test_feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 656bb515f..e80e07554 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,6 @@ import random import shutil import string import tempfile -import warnings from io import BytesIO from logging import getLogger from pathlib import Path From 6f4ccec5675c00b0ec7877a0b1fcc234d5983490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 14:03:14 +0200 Subject: [PATCH 016/127] Cover our deprecation policy in the documentation --- docs/contributing.rst | 5 +++++ docs/versioning.rst | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7b901dd00..525ad3497 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches diff --git a/docs/versioning.rst b/docs/versioning.rst index 227085f02..57643ea9a 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,7 +1,7 @@ .. _versioning: ============================ -Versioning and API Stability +Versioning and API stability ============================ Versioning @@ -34,7 +34,7 @@ For example: production) -API Stability +API stability ============= API stability was one of the major goals for the *1.0* release. @@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the same way. +.. _deprecation-policy: + +Deprecation policy +================== + +We aim to maintain support for deprecated Scrapy features for at least 1 year. + +For example, if a feature is deprecated in a Scrapy version released on +June 15th 2020, that feature should continue to work in versions released on +June 14th 2021 or before that. + +Any new Scrapy release after a year *may* remove support for that deprecated +feature. + +All deprecated features removed in a Scrapy release are explicitly mentioned in +the :ref:`release notes `. + + .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases From aefd43a6c6b8d2e32dfb9bd0c94529805ce5037e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Aug 2020 12:52:54 +0200 Subject: [PATCH 017/127] Upgrade minimum dependencies for Python 3.6 support --- tests/requirements-py3.txt | 3 +-- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 00c56084d..4fa58d11b 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,7 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.6' -mitmproxy<4.0.0; python_version < '3.6' +mitmproxy >=4, <5 # The latest version does not support some pinned dependencies # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines diff --git a/tox.ini b/tox.ini index 11882c03f..4f5531aea 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -rtests/requirements-py3.txt # Extras boto3>=1.13.0 - botocore>=1.3.23 + botocore>=1.4.87 Pillow>=3.4.2 passenv = S3_TEST_FILE_URI @@ -76,7 +76,7 @@ deps = zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras - botocore==1.3.23 + botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==3.4.2 From 1c4b4cc6b046e121be33a57031a0900ca1347e16 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Tue, 11 Aug 2020 17:42:44 +0530 Subject: [PATCH 018/127] Support defining file path based on item in media pipelines (#4686) --- docs/topics/media-pipeline.rst | 24 ++++--- scrapy/pipelines/files.py | 16 ++--- scrapy/pipelines/images.py | 14 ++-- scrapy/pipelines/media.py | 69 +++++++++++++++--- tests/test_pipeline_files.py | 13 ++++ tests/test_pipeline_media.py | 123 ++++++++++++++++++++++++++++++++- 6 files changed, 225 insertions(+), 34 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..487e26b8e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. @@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6bc5d46eb..5a2184681 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): def _onsuccess(result): if not result: return # returning None force download @@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline): checksum = result.get('checksum', None) return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} - path = self.file_path(request, info=info) + path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) dfd.addCallbacks(_onsuccess, lambda _: None) dfd.addErrback( @@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): referer = referer_str(request) if response.status != 200: @@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, status) try: - path = self.file_path(request, response=response, info=info) - checksum = self.file_downloaded(response, request, info) + path = self.file_path(request, response=response, info=info, item=item) + checksum = self.file_downloaded(response, request, info, item=item) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' @@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline): urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u) for u in urls] - def file_downloaded(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def file_downloaded(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = md5sum(buf) buf.seek(0) @@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_ext = os.path.splitext(request.url)[1] # Handles empty and wild extensions by trying to guess the diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e2dd70215..0a67a0b1d 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info): - return self.image_downloaded(response, request, info) + def file_downloaded(self, response, request, info, *, item=None): + return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info): + def image_downloaded(self, response, request, info, *, item=None): checksum = None - for path, image, buf in self.get_images(response, request, info): + for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) checksum = md5sum(buf) @@ -119,8 +119,8 @@ class ImagesPipeline(FilesPipeline): headers={'Content-Type': 'image/jpeg'}) return checksum - def get_images(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def get_images(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size @@ -166,7 +166,7 @@ class ImagesPipeline(FilesPipeline): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return 'full/%s.jpg' % (image_guid) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index aa65f4f0e..2439de9a5 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,12 +1,16 @@ import functools import logging from collections import defaultdict +from inspect import signature +from warnings import warn + from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -27,6 +31,7 @@ class MediaPipeline: def __init__(self, download_func=None, settings=None): self.download_func = download_func + self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -38,6 +43,9 @@ class MediaPipeline: ) self._handle_statuses(self.allow_redirects) + # Check if deprecated methods are being used and make them compatible + self._make_compatible() + def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -77,11 +85,11 @@ class MediaPipeline: def process_item(self, item, spider): info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) - dlist = [self._process_request(r, info) for r in requests] + dlist = [self._process_request(r, info, item) for r in requests] dfd = DeferredList(dlist, consumeErrors=1) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info): + def _process_request(self, request, info, item): fp = request_fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback @@ -102,34 +110,73 @@ class MediaPipeline: # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info) - dfd.addCallback(self._check_media_to_download, request, info) + dfd = mustbe_deferred(self.media_to_download, request, info, item=item) + dfd.addCallback(self._check_media_to_download, request, info, item=item) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(lambda f: logger.error( f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider}) ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _make_compatible(self): + """Make overridable methods of MediaPipeline and subclasses backwards compatible""" + methods = [ + "file_path", "media_to_download", "media_downloaded", + "file_downloaded", "image_downloaded", "get_images" + ] + + for method_name in methods: + method = getattr(self, method_name, None) + if callable(method): + setattr(self, method_name, self._compatible(method)) + + def _compatible(self, func): + """Wrapper for overridable methods to allow backwards compatibility""" + self._check_signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if self._expects_item[func.__name__]: + return func(*args, **kwargs) + + kwargs.pop('item', None) + return func(*args, **kwargs) + + return wrapper + + def _check_signature(self, func): + sig = signature(func) + self._expects_item[func.__name__] = True + + if 'item' not in sig.parameters: + old_params = str(sig)[1:-1] + new_params = old_params + ", *, item=None" + warn('%s(self, %s) is deprecated, ' + 'please use %s(self, %s)' + % (func.__name__, old_params, func.__name__, new_params), + ScrapyDeprecationWarning, stacklevel=2) + self._expects_item[func.__name__] = False + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - def _check_media_to_download(self, result, request, info): + def _check_media_to_download(self, result, request, info, item): if result is not None: return result if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) return dfd @@ -171,7 +218,7 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): """Check request before starting download""" pass @@ -179,7 +226,7 @@ class MediaPipeline: """Returns the media requests to download""" pass - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" return response @@ -199,3 +246,7 @@ class MediaPipeline: extra={'spider': info.spider} ) return item + + def file_path(self, request, response=None, info=None, *, item=None): + """Returns the path where downloaded media should be stored""" + pass diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a023dfcc8..b19b4ff2a 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase): for p in patchers: p.stop() + def test_file_path_from_item(self): + """ + Custom file path based on item data, overriding default implementation + """ + class CustomFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, item=None): + return 'full/%s' % item.get('path') + + file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(file_path(request, item=item), 'full/path-to-store-file') + class FilesPipelineTestCaseFieldsMixin: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 4f130c0c9..6afd47497 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info @@ -169,7 +171,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('download') return super().download(request, info) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') @@ -179,7 +181,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('get_media_requests') return item.get('requests') - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): self._mockcalled.append('media_downloaded') return super().media_downloaded(response, request, info) @@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ['get_media_requests', 'media_to_download', 'item_completed']) +class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): + + def __init__(self, *args, **kwargs): + super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + self._mockcalled = [] + + def get_media_requests(self, item, info): + item_url = item['image_urls'][0] + return Request( + item_url, + meta={'response': Response(item_url, status=200, body=b'data')} + ) + + def inc_stats(self, *args, **kwargs): + return True + + def media_to_download(self, request, info): + self._mockcalled.append('media_to_download') + return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + + def media_downloaded(self, response, request, info): + self._mockcalled.append('media_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + + def file_downloaded(self, response, request, info): + self._mockcalled.append('file_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + + def file_path(self, request, response=None, info=None): + self._mockcalled.append('file_path') + return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + + def get_images(self, response, request, info): + self._mockcalled.append('get_images') + return [] + + def image_downloaded(self, response, request, info): + self._mockcalled.append('image_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + + +class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + + def setUp(self): + self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe.open_spider(None) + self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) + + def _assert_method_called_with_warnings(self, method, message, warnings): + self.assertIn(method, self.pipe._mockcalled) + warningShown = False + for warning in warnings: + if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning: + warningShown = True + self.assertTrue(warningShown) + + @inlineCallbacks + def test_media_to_download_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_to_download(self, request, info) is deprecated, ' + 'please use media_to_download(self, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_to_download', message, warnings) + + @inlineCallbacks + def test_media_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_downloaded(self, response, request, info) is deprecated, ' + 'please use media_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_downloaded', message, warnings) + + @inlineCallbacks + def test_file_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_downloaded(self, response, request, info) is deprecated, ' + 'please use file_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('file_downloaded', message, warnings) + + @inlineCallbacks + def test_file_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_path(self, request, response=None, info=None) is deprecated, ' + 'please use file_path(self, request, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('file_path', message, warnings) + + @inlineCallbacks + def test_get_images_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'get_images(self, response, request, info) is deprecated, ' + 'please use get_images(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('get_images', message, warnings) + + @inlineCallbacks + def test_image_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'image_downloaded(self, response, request, info) is deprecated, ' + 'please use image_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('image_downloaded', message, warnings) + + class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): From 394631fc0a731a0b8b0108edc1711c0c87e2ca15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:08:09 +0200 Subject: [PATCH 019/127] Restore 3.5 support for mitmproxy-based tests --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 4fa58d11b..b67c08403 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,8 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy >=4, <5 # The latest version does not support some pinned dependencies +mitmproxy >=4, <5; python_version < '3.6' # The latest version does not support some pinned dependencies +mitmproxy <4; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From 8e393a0b218b56cc8a70f7bda277969aa026790d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:29:51 +0200 Subject: [PATCH 020/127] Do not change the mitmproxy version for no-3.6 Python versions --- tests/requirements-py3.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b67c08403..b425ce771 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,9 @@ # Tests requirements attrs -dataclasses; python_version == '3.6' -mitmproxy >=4, <5; python_version < '3.6' # The latest version does not support some pinned dependencies -mitmproxy <4; python_version < '3.6' +dataclasses; python_version ==3.6 +mitmproxy; python_version >=3.7 +mitmproxy >=4, <5; python_version >=3.6, <3.7 +mitmproxy <4; python_version <3.6 # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From b1de55d37d6a1b84e92df2d93d83e247e5d0e0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:34:40 +0200 Subject: [PATCH 021/127] Fix marker syntax --- tests/requirements-py3.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b425ce771..b51177abb 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,9 +1,9 @@ # Tests requirements attrs -dataclasses; python_version ==3.6 -mitmproxy; python_version >=3.7 -mitmproxy >=4, <5; python_version >=3.6, <3.7 -mitmproxy <4; python_version <3.6 +dataclasses; python_version == '3.6' +mitmproxy; python_version >= '3.7' +mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' +mitmproxy < 4; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From 125a058340cf77a70fe605b0317b3c517f637c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 17:07:21 +0200 Subject: [PATCH 022/127] Do not let umask affect the permissions of startproject-generated files --- scrapy/utils/template.py | 6 +++-- tests/test_commands.py | 57 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 96ff4b09b..f068be737 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -12,10 +12,12 @@ def render_templatefile(path, **kwargs): content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path + + if path.endswith('.tmpl'): + os.rename(path, render_path) + with open(render_path, 'wb') as fp: fp.write(content.encode('utf8')) - if path.endswith('.tmpl'): - os.remove(path) CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/tests/test_commands.py b/tests/test_commands.py index 8938156fc..be88e3511 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -126,9 +126,13 @@ class StartprojectTest(ProjectTest): def get_permissions_dict(path, renamings=None, ignore=None): + + def get_permissions(path): + return oct(os.stat(path).st_mode) + renamings = renamings or tuple() permissions_dict = { - '.': os.stat(path).st_mode, + '.': get_permissions(path), } for root, dirs, files in os.walk(path): nodes = list(chain(dirs, files)) @@ -143,13 +147,15 @@ def get_permissions_dict(path, renamings=None, ignore=None): search_string, replacement ) - permissions = os.stat(absolute_path).st_mode + permissions = get_permissions(absolute_path) permissions_dict[relative_path] = permissions return permissions_dict class StartprojectTemplatesTest(ProjectTest): + maxDiff = None + def setUp(self): super().setUp() self.tmpl = join(self.temp_path, 'templates') @@ -311,6 +317,53 @@ class StartprojectTemplatesTest(ProjectTest): self.assertEqual(actual_permissions, expected_permissions) + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = os.path.join( + scrapy_path, + 'templates', + 'project' + ) + project_name = 'umaskproject' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + class CommandTest(ProjectTest): From 4c0afb606c59e6c8eabe4bf97cdb0494cf26dec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 17:45:26 +0200 Subject: [PATCH 023/127] Update permission expectations --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index be88e3511..55088c605 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -297,7 +297,7 @@ class StartprojectTemplatesTest(ProjectTest): path.mkdir(mode=permissions) else: path.touch(mode=permissions) - expected_permissions[node] = path.stat().st_mode + expected_permissions[node] = oct(path.stat().st_mode) process = subprocess.Popen( ( From 5f4df622a17b1d8f9c3b9b693cb2f7119ffec27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 05:41:06 +0200 Subject: [PATCH 024/127] test_utils_iterators.py: support Windows the right way --- tests/__init__.py | 2 +- tests/test_utils_iterators.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..4253017fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb') as f: + with open(path, 'rb', newline='') as f: return f.read() diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 298178f08..50190d4d1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = "foo{}bar".format(os.linesep) - - class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) @@ -267,7 +264,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual(result, [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr @@ -283,7 +280,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): @@ -296,7 +293,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv1], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) @@ -305,7 +302,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv2], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): @@ -327,7 +324,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): @@ -353,7 +350,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): From 24ba5a71aca9b368eee21bd7d9043a7d26dba403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 06:35:09 +0200 Subject: [PATCH 025/127] Maybe the problem is not in the code after all --- .gitattributes | 1 + tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/tests/__init__.py b/tests/__init__.py index 4253017fa..12ce79fa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb', newline='') as f: + with open(path, 'rb') as f: return f.read() From 65e0abaea5eddb1dbb28b2bca9cd00e9b9409471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 09:05:51 +0200 Subject: [PATCH 026/127] Document FEED_URI_PARAMS --- docs/topics/feed-exports.rst | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 37b7096f6..0f0f258dc 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -321,13 +321,14 @@ The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition. * ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. + See :ref:`topics-feed-format` for possible values. Mandatory, no fallback setting +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` * ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING @@ -500,7 +501,7 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. +* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch. Use :ref:`printf-style string formatting ` to alter the number format. For example, to make the batch ID a 5-digit @@ -517,16 +518,74 @@ And your :command:`crawl` command line is:: The command line above can generate a directory tree like:: -->projectname --->dirname ---->1-filename2020-03-28T14-45-08.237134.json ---->2-filename2020-03-28T14-45-09.148903.json ---->3-filename2020-03-28T14-45-10.046092.json + ->projectname + -->dirname + --->1-filename2020-03-28T14-45-08.237134.json + --->2-filename2020-03-28T14-45-09.148903.json + --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains 100 items or fewer. +.. setting:: FEED_URI_PARAMS + +FEED_URI_PARAMS +--------------- + +Default: ``None`` + +A string with the import path of a function to set the parameters to apply with +:ref:`printf-style string formatting ` to the +feed URI. + +The function signature should be as follows: + +.. function:: uri_params(params, spider) + + Return a :class:`dict` of key-value pairs to apply to the feed URI using + :ref:`printf-style string formatting `. + + :param params: default key-value pairs + + Specifically: + + - ``batch_id``: ID of the file batch. See + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` + is always ``1``. + + - ``batch_time``: UTC date and time, in ISO format with ``:`` + replaced with ``-``. + + See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + - ``time``: ``batch_time``, with microseconds set to ``0``. + :type params: dict + + :param spider: source spider of the feed items + :type spider: scrapy.spiders.Spider + +For example, to include the :attr:`name ` of the +source spider in the feed URI: + +#. Define the following function somewhere in your project:: + + # myproject/utils.py + def uri_params(params, spider): + return {**params, 'spider_name': spider.name} + +#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:: + + # myproject/settings.py + FEED_URI_PARAMS = 'myproject.utils.uri_params' + +#. Use ``%(spider_name)s`` in your feed URI:: + + scrapy crawl -o "%(spider_name)s.jl" + + .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore From 756c368a6b0d2eef65f86f8418f9a7fbeff036c7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Aug 2020 22:09:24 +0500 Subject: [PATCH 027/127] Use a longer key in mitmproxy-ca.pem. --- tests/keys/mitmproxy-ca.pem | 78 +++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index 08004feca..cdef75f99 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,32 +1,50 @@ ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt -auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha -QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB -AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF -QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol -Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w -T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll -KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj -58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc -1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT -9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB -dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn -Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB +/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT +NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ +u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH +JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB +eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls +90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ +epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 +VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn +v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT +I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU +oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au +/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG +gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn +YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf +dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 +9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC +QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu +bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW +CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR +OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu +PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK +eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx +BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ +B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy +HzPh2taMnCA0h1fNLFFBkw88 +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT -CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX -DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt -aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u -YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp -w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk -Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB -/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG -CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB -gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG -+EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n -MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D -y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z -e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1 -n95u +MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM +CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX +DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt +aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW +ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 +ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg +z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI +ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE +6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ +0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC +AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD +BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG +CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA +A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd +kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe +gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY +1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj +envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj +wiKjdLjCmJgjGAQjRw5VYJ8JI0XO -----END CERTIFICATE----- From 2aa4f3cbf96ad55aa4a1fa064758338daf16b110 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 17 Aug 2020 05:39:59 -0300 Subject: [PATCH 028/127] Conditional request attribute binding for responses (#4632) --- docs/topics/signals.rst | 5 + scrapy/core/engine.py | 13 +- scrapy/core/scraper.py | 62 ++++---- setup.cfg | 3 + tests/test_request_attribute_binding.py | 202 ++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 tests/test_request_attribute_binding.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 255ba9d3f..1d99d8c28 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -423,6 +423,11 @@ response_received :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: The ``request`` argument might not contain the original request that + reached the downloader, if a :ref:`topics-downloader-middleware` modifies + the :class:`~scrapy.http.Response` object and sets a specific ``request`` + attribute. + response_downloaded ~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 86a6abb23..5e0dfe37c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -243,12 +243,17 @@ class ExecutionEngine: % (type(response), response) ) if isinstance(response, Response): - response.request = request # tie request to response received - logkws = self.logformatter.crawled(request, response, spider) + if response.request is None: + response.request = request + logkws = self.logformatter.crawled(response.request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log( + signal=signals.response_received, + response=response, + request=response.request, + spider=spider, + ) return response def _on_complete(_): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1ef0790a9..20bdb22a1 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,7 +12,7 @@ from scrapy import signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel +from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output @@ -120,40 +120,40 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, response, request, spider): - """Handle the downloaded response or failure through the spider - callback/errback""" - if not isinstance(response, (Response, Failure)): - raise TypeError( - "Incorrect type: expected Response or Failure, got %s: %r" - % (type(response), response) - ) - - dfd = self._scrape2(response, request, spider) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, response, spider) - dfd.addCallback(self.handle_spider_output, request, response, spider) + def _scrape(self, result, request, spider): + """ + Handle the downloaded response or failure through the spider callback/errback + """ + if not isinstance(result, (Response, Failure)): + raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) + dfd = self._scrape2(result, request, spider) # returns spider's processed output + dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, request_result, request, spider): - """Handle the different cases of request's result been a Response or a - Failure""" - if not isinstance(request_result, Failure): - return self.spidermw.scrape_response( - self.call_spider, request_result, request, spider) - else: - dfd = self.call_spider(request_result, request, spider) - return dfd.addErrback( - self._log_download_errors, request_result, request, spider) + def _scrape2(self, result, request, spider): + """ + Handle the different cases of request's result been a Response or a Failure + """ + if isinstance(result, Response): + return self.spidermw.scrape_response(self.call_spider, result, request, spider) + else: # result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result, request, spider): - result.request = request - dfd = defer_result(result) - callback = request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) - warn_on_generator_with_return_value(spider, request.errback) - dfd.addCallbacks(callback=callback, - errback=request.errback, - callbackKeywords=request.cb_kwargs) + if isinstance(result, Response): + if getattr(result, "request", None) is None: + result.request = request + callback = result.request.callback or spider._parse + warn_on_generator_with_return_value(spider, callback) + dfd = defer_succeed(result) + dfd.addCallback(callback, **result.request.cb_kwargs) + else: # result is a Failure + result.request = request + warn_on_generator_with_return_value(spider, request.errback) + dfd = defer_fail(result) + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..3a624ec94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,9 @@ ignore_errors = True [mypy-tests.test_pipelines] ignore_errors = True +[mypy-tests.test_request_attribute_binding] +ignore_errors = True + [mypy-tests.test_request_cb_kwargs] ignore_errors = True diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py new file mode 100644 index 000000000..b60b7c579 --- /dev/null +++ b/tests/test_request_attribute_binding.py @@ -0,0 +1,202 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy import Request, signals +from scrapy.crawler import CrawlerRunner +from scrapy.http.response import Response + +from testfixtures import LogCapture + +from tests.mockserver import MockServer +from tests.spiders import SingleRequestSpider + + +OVERRIDEN_URL = "https://example.org" + + +class ProcessResponseMiddleware: + def process_response(self, request, response, spider): + return response.replace(request=Request(OVERRIDEN_URL)) + + +class RaiseExceptionRequestMiddleware: + def process_request(self, request, spider): + 1 / 0 + return request + + +class CatchExceptionOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + request=Request(OVERRIDEN_URL), + ) + + +class CatchExceptionDoNotOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + ) + + +class AlternativeCallbacksSpider(SingleRequestSpider): + name = "alternative_callbacks_spider" + + def alt_callback(self, response, foo=None): + self.logger.info("alt_callback was invoked with foo=%s", foo) + + +class AlternativeCallbacksMiddleware: + def process_response(self, request, response, spider): + new_request = request.replace( + url=OVERRIDEN_URL, + callback=spider.alt_callback, + cb_kwargs={"foo": "bar"}, + ) + return response.replace(request=new_request) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_response_200(self): + url = self.mockserver.url("/status?n=200") + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_response_error(self): + for status in ("404", "500"): + url = self.mockserver.url("/status?n={}".format(status)) + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + response = failure.value.response + self.assertEqual(failure.request.url, url) + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_raise_exception(self): + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + self.assertEqual(failure.request.url, url) + self.assertIsInstance(failure.value, ZeroDivisionError) + + @defer.inlineCallbacks + def test_downloader_middleware_override_request_in_process_response(self): + """ + Downloader middleware which returns a response with an specific 'request' attribute. + + * The spider callback should receive the overriden response.request + * Handlers listening to the response_received signal should receive the overriden response.request + * The "crawled" log message should show the overriden response.request + """ + signal_params = {} + + def signal_handler(response, request, spider): + signal_params["response"] = response + signal_params["request"] = request + + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".ProcessResponseMiddleware": 595, + } + }) + crawler = runner.create_crawler(SingleRequestSpider) + crawler.signals.connect(signal_handler, signal=signals.response_received) + + with LogCapture() as log: + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, OVERRIDEN_URL) + + self.assertEqual(signal_params["response"].url, url) + self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) + + log.check_present( + ("scrapy.core.engine", "DEBUG", "Crawled (200) (referer: None)".format(OVERRIDEN_URL)), + ) + + @defer.inlineCallbacks + def test_downloader_middleware_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response with a specific 'request' attribute. + + The spider callback should receive the overriden response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, OVERRIDEN_URL) + + @defer.inlineCallbacks + def test_downloader_middleware_do_not_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response without a specific 'request' attribute. + + The spider callback should receive the original response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_alternative_callback(self): + """ + Downloader middleware which returns a response with a + specific 'request' attribute, with an alternative callback + """ + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".AlternativeCallbacksMiddleware": 595, + } + }) + crawler = runner.create_crawler(AlternativeCallbacksSpider) + + with LogCapture() as log: + url = self.mockserver.url("/status?n=200") + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + log.check_present( + ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"), + ) From a8e08d51cd50e8f0b58a60992d310e56d4f71641 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Mon, 17 Aug 2020 14:15:52 +0530 Subject: [PATCH 029/127] Check if file is already present on running `scrapy genspider` and terminate if so (#4623) --- scrapy/commands/genspider.py | 41 ++++++++++++++++------ tests/test_commands.py | 67 +++++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4c7548e9c..74a077d1b 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -66,16 +66,9 @@ class Command(ScrapyCommand): print("Cannot create a spider with the same name as your project") return - try: - spidercls = self.crawler_process.spider_loader.load(name) - except KeyError: - pass - else: - # if spider already exists and not --force then halt - if not opts.force: - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) - return + if not opts.force and self._spider_exists(name): + return + template_file = self._find_template(opts.template) if template_file: self._genspider(module, name, domain, opts.template, template_file) @@ -119,6 +112,34 @@ class Command(ScrapyCommand): if filename.endswith('.tmpl'): print(" %s" % splitext(filename)[0]) + def _spider_exists(self, name): + if not self.settings.get('NEWSPIDER_MODULE'): + # if run as a standalone command and file with same filename already exists + if exists(name + ".py"): + print("%s already exists" % (abspath(name + ".py"))) + return True + return False + + try: + spidercls = self.crawler_process.spider_loader.load(name) + except KeyError: + pass + else: + # if spider with same name exists + print("Spider %r already exists in module:" % name) + print(" %s" % spidercls.__module__) + return True + + # a file with the same name exists in the target directory + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = dirname(spiders_module.__file__) + spiders_dir_abs = abspath(spiders_dir) + if exists(join(spiders_dir_abs, name + ".py")): + print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) + return True + + return False + @property def templates_dir(self): return join( diff --git a/tests/test_commands.py b/tests/test_commands.py index 8938156fc..109f006a8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -8,7 +8,7 @@ import sys import tempfile from contextlib import contextmanager from itertools import chain -from os.path import exists, join, abspath +from os.path import exists, join, abspath, getmtime from pathlib import Path from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION @@ -337,8 +337,11 @@ class GenspiderCommandTest(CommandTest): p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) + modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) + modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): self.test_template('basic') @@ -360,6 +363,40 @@ class GenspiderCommandTest(CommandTest): self.assertEqual(2, self.call('genspider', self.project_name)) assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + def test_same_filename_as_existing_spider(self, force=False): + file_name = 'example' + file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name) + self.assertEqual(0, self.call('genspider', file_name, 'example.com')) + assert exists(file_path) + + # change name of spider but not its file name + with open(file_path, 'r+') as spider_file: + file_data = spider_file.read() + file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = getmtime(file_path) + file_contents_before = file_data + + if force: + p, out, err = self.proc('genspider', '--force', file_name, 'example.com') + self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("%s already exists" % (file_path), out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) + class GenspiderStandaloneCommandTest(ProjectTest): @@ -367,6 +404,34 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.call('genspider', 'example', 'example.com') assert exists(join(self.temp_path, 'example.py')) + def test_same_name_as_existing_file(self, force=False): + file_name = 'example' + file_path = join(self.temp_path, file_name + '.py') + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("Created spider %r using template \'basic\' " % file_name, out) + assert exists(file_path) + modify_time_before = getmtime(file_path) + file_contents_before = open(file_path, 'r').read() + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') + self.assertIn("Created spider %r using template \'crawl\' " % file_name, out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) + class MiscCommandsTest(CommandTest): From 55edf8d3b8885541cdbf9d1c62d9a6bbf634e2a0 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Mon, 17 Aug 2020 18:50:52 +1000 Subject: [PATCH 030/127] Add typing hint to httpcache downloadermiddlewares (#4243) --- scrapy/downloadermiddlewares/httpcache.py | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6db57bd8b..62f1c3a29 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,4 +1,5 @@ from email.utils import formatdate +from typing import Optional, Type, TypeVar from twisted.internet import defer from twisted.internet.error import ( @@ -13,10 +14,19 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http.request import Request +from scrapy.http.response import Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object +HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,7 +34,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) - def __init__(self, settings, stats): + def __init__(self, settings: Settings, stats: StatsCollector) -> None: if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) @@ -33,26 +43,26 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.storage.open_spider(spider) - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Response]: if request.meta.get('dont_cache', False): - return + return None # Skip uncacheable requests if not self.policy.should_cache_request(request): request.meta['_dont_cache'] = True # flag as uncacheable - return + return None # Look for cached response and check if expired cachedresponse = self.storage.retrieve_response(spider, request) @@ -61,7 +71,7 @@ class HttpCacheMiddleware: if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) raise IgnoreRequest("Ignored request not in cache: %s" % request) - return # first time request + return None # first time request # Return cached response only if not expired cachedresponse.flags.append('cached') @@ -73,7 +83,9 @@ class HttpCacheMiddleware: # process_response hook request.meta['cached_response'] = cachedresponse - def process_response(self, request, response, spider): + return None + + def process_response(self, request: Request, response: Response, spider: Spider) -> Response: if request.meta.get('dont_cache', False): return response @@ -85,7 +97,7 @@ class HttpCacheMiddleware: # RFC2616 requires origin server to set Date header, # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: - response.headers['Date'] = formatdate(usegmt=1) + response.headers['Date'] = formatdate(usegmt=True) # Do not validate first-hand responses cachedresponse = request.meta.pop('cached_response', None) @@ -102,13 +114,18 @@ class HttpCacheMiddleware: self._cache_response(spider, response, request, cachedresponse) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Optional[Response]: cachedresponse = request.meta.pop('cached_response', None) if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS): self.stats.inc_value('httpcache/errorrecovery', spider=spider) return cachedresponse + return None - def _cache_response(self, spider, response, request, cachedresponse): + def _cache_response( + self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response] + ) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value('httpcache/store', spider=spider) self.storage.store_response(spider, request, response) From e70975f0bb4e8378bf0e00ae7a7014247cd59441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Aug 2020 15:10:08 +0200 Subject: [PATCH 031/127] Allow overwriting feeds (#4512) Co-authored-by: Yuval Hager --- docs/faq.rst | 6 +- docs/intro/overview.rst | 26 +- docs/intro/tutorial.rst | 13 +- docs/topics/feed-exports.rst | 47 +++- scrapy/commands/__init__.py | 15 +- scrapy/extensions/feedexport.py | 142 +++++++--- scrapy/utils/conf.py | 44 +++- scrapy/utils/ftp.py | 6 +- scrapy/utils/python.py | 3 +- tests/ftpserver.py | 24 ++ tests/mockserver.py | 26 ++ tests/requirements-py3.txt | 1 + tests/test_commands.py | 126 +++++++++ tests/test_feedexport.py | 442 ++++++++++++++++++++++++++++---- tests/test_utils_conf.py | 16 ++ tests/test_utils_python.py | 4 + 16 files changed, 791 insertions(+), 150 deletions(-) create mode 100644 tests/ftpserver.py diff --git a/docs/faq.rst b/docs/faq.rst index ea2c8216f..9346ec358 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -o items.json + scrapy crawl myspider -O items.json To dump into a CSV file:: - scrapy crawl myspider -o items.csv + scrapy crawl myspider -O items.csv To dump into a XML file:: - scrapy crawl myspider -o items.xml + scrapy crawl myspider -O items.xml For more information see :ref:`topics-feed-exports` diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 01986b594..dd80c7bd0 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination:: if next_page is not None: yield response.follow(next_page, self.parse) - Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.json + scrapy runspider quotes_spider.py -o quotes.jl +When this finishes you will have in the ``quotes.jl`` file a list of the +quotes in JSON Lines format, containing text and author, looking like this:: -When this finishes you will have in the ``quotes.json`` file a list of the -quotes in JSON format, containing text and author, looking like this (reformatted -here for better readability):: - - [{ - "author": "Jane Austen", - "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" - }, - { - "author": "Groucho Marx", - "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" - }, - { - "author": "Steve Martin", - "text": "\u201cA day without sunshine is like, you know, night.\u201d" - }, - ...] + {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} + {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} + {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"} + ... What just happened? diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5f35dc936..f96c78887 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -464,16 +464,15 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o quotes.json + scrapy crawl quotes -O quotes.json That will generate an ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -For historic reasons, Scrapy appends to a given file instead of overwriting -its contents. If you run this command twice without removing the file -before the second time, you'll end up with a broken JSON file. - -You can also use other formats, like `JSON Lines`_:: +The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead +to append new content to any existing file. However, appending to a JSON file +makes the file contents invalid JSON. When appending to a file, consider +using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jl @@ -704,7 +703,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o quotes-humor.json -a tag=humor + scrapy crawl quotes -O quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0f0f258dc..cd4f7cf29 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -291,6 +291,7 @@ Default: ``{}`` A dictionary in which every key is a feed URI (or a :class:`pathlib.Path` object) and each value is a nested dictionary containing configuration parameters for the specific feed. + This setting is required for enabling the feed export feature. See :ref:`topics-feed-storage-backends` for supported URI schemes. @@ -318,17 +319,43 @@ For instance:: } The following is a list of the accepted keys and the setting that is used -as a fallback value if that key is not provided for a specific feed definition. +as a fallback value if that key is not provided for a specific feed definition: + +- ``format``: the :ref:`serialization format `. + + This setting is mandatory, there is no fallback value. + +- ``batch_item_count``: falls back to + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + +- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. + +- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. + +- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. + +- ``overwrite``: whether to overwrite the file if it already exists + (``True``) or append to its content (``False``). + + The default value depends on the :ref:`storage backend + `: + + - :ref:`topics-feed-storage-fs`: ``False`` + + - :ref:`topics-feed-storage-ftp`: ``True`` + + .. note:: Some FTP servers may not support appending to files (the + ``APPE`` FTP command). + + - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported + `_) + + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + +- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. + +- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. -* ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. - Mandatory, no fallback setting -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` -* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` -* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` -* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` -* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 57ce4e522..cfd940fe7 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + if opts.output or opts.overwrite_output: + feeds = feed_process_params_from_cli( + self.settings, + opts.output, + opts.output_format, + opts.overwrite_output, + ) self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b7a4e362e..980825499 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import without_none_values +from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): + argument_names = get_func_args(builder) + if 'feed_options' in argument_names: + kwargs['feed_options'] = feed_options + else: + warnings.warn( + "{} does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove this " + "warning. This parameter will become mandatory in a future " + "version of Scrapy." + .format(builder.__qualname__), + category=ScrapyDeprecationWarning + ) + return builder(*preargs, uri, *args, **kwargs) + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" - def __init__(uri): - """Initialize the storage with the parameters given in the URI""" + def __init__(uri, *, feed_options=None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" def open(spider): """Open the storage for the given spider. It must return a file-like @@ -64,10 +81,15 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None): + def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer self._stdout = _stdout + if feed_options and feed_options.get('overwrite', False) is True: + logger.warning('Standard output (stdout) storage does not support ' + 'overwriting. To suppress this warning, remove the ' + 'overwrite option from your FEEDS setting, or set ' + 'it to False.') def open(self, spider): return self._stdout @@ -79,14 +101,16 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri): + def __init__(self, uri, *, feed_options=None): self.path = file_uri_to_path(uri) + feed_options = feed_options or {} + self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' def open(self, spider): dirname = os.path.dirname(self.path) if dirname and not os.path.exists(dirname): os.makedirs(dirname) - return open(self.path, 'ab') + return open(self.path, self.write_mode) def store(self, file): file.close() @@ -94,7 +118,8 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + feed_options=None): u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key @@ -111,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage): else: import boto self.connect_s3 = boto.connect_s3 + if feed_options and feed_options.get('overwrite', True) is False: + logger.warning('S3 does not support appending to files. To ' + 'suppress this warning, remove the overwrite ' + 'option from your FEEDS setting or set it to True.') @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + feed_options=feed_options, ) def _store_in_thread(self, file): @@ -135,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage): kwargs = {'policy': self.acl} if self.acl else {} key.set_contents_from_file(file, **kwargs) key.close() + file.close() class GCSFeedStorage(BlockingFeedStorage): @@ -165,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False): + def __init__(self, uri, use_active_mode=False, *, feed_options=None): u = urlparse(uri) self.host = u.hostname self.port = int(u.port or '21') self.username = u.username - self.password = unquote(u.password) + self.password = unquote(u.password or '') self.path = u.path self.use_active_mode = use_active_mode + self.overwrite = not feed_options or feed_options.get('overwrite', True) @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, - use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE') + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, + crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'), + feed_options=feed_options, ) def _store_in_thread(self, file): ftp_store_file( path=self.path, file=file, host=self.host, port=self.port, username=self.username, - password=self.password, use_active_mode=self.use_active_mode + password=self.password, use_active_mode=self.use_active_mode, + overwrite=self.overwrite, ) @@ -242,32 +278,32 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects - feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed in self.settings.getdict('FEEDS').items(): + for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - for uri, feed in self.feeds.items(): - if not self._storage_supported(uri): + for uri, feed_options in self.feeds.items(): + if not self._storage_supported(uri, feed_options): raise NotConfigured if not self._settings_are_valid(): raise NotConfigured - if not self._exporter_supported(feed['format']): + if not self._exporter_supported(feed_options['format']): raise NotConfigured def open_spider(self, spider): - for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params']) + for uri, feed_options in self.feeds.items(): + uri_params = self._get_uri_params(spider, feed_options['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, - feed=feed, + feed_options=feed_options, spider=spider, uri_template=uri, )) @@ -306,32 +342,32 @@ class FeedExporter: ) return d - def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start - :param feed: dict with parameters of feed + :param feed_options: dict with parameters of feed :param spider: user spider :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ - storage = self._get_storage(uri) + storage = self._get_storage(uri, feed_options) file = storage.open(spider) exporter = self._get_exporter( file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], + format=feed_options['format'], + fields_to_export=feed_options['fields'], + encoding=feed_options['encoding'], + indent=feed_options['indent'], ) slot = _FeedSlot( file=file, exporter=exporter, storage=storage, uri=uri, - format=feed['format'], - store_empty=feed['store_empty'], + format=feed_options['format'], + store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, ) @@ -355,7 +391,7 @@ class FeedExporter: slots.append(self._start_new_batch( batch_id=slot.batch_id + 1, uri=slot.uri_template % uri_params, - feed=self.feeds[slot.uri_template], + feed_options=self.feeds[slot.uri_template], spider=spider, uri_template=slot.uri_template, )) @@ -394,11 +430,11 @@ class FeedExporter: return False return True - def _storage_supported(self, uri): + def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme if scheme in self.storages: try: - self._get_storage(uri) + self._get_storage(uri, feed_options) return True except NotConfigured as e: logger.error("Disabled feed storage scheme: %(scheme)s. " @@ -416,8 +452,30 @@ class FeedExporter: def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri): - return self._get_instance(self.storages[urlparse(uri).scheme], uri) + def _get_storage(self, uri, feed_options): + """Fork of create_instance specific to feed storage classes + + It supports not passing the *feed_options* parameters to classes that + do not support it, and issuing a deprecation warning instead. + """ + feedcls = self.storages[urlparse(uri).scheme] + crawler = getattr(self, 'crawler', None) + + def build_instance(builder, *preargs): + return build_storage(builder, uri, preargs=preargs) + + if crawler and hasattr(feedcls, 'from_crawler'): + instance = build_instance(feedcls.from_crawler, crawler) + method_name = 'from_crawler' + elif hasattr(feedcls, 'from_settings'): + instance = build_instance(feedcls.from_settings, self.settings) + method_name = 'from_settings' + else: + instance = build_instance(feedcls) + method_name = '__new__' + if instance is None: + raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + return instance def _get_uri_params(self, spider, uri_params, slot=None): params = {} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index a83076c47..90a52b25b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output, output_format=None): +def feed_process_params_from_cli(settings, output, output_format=None, + overwrite_output=None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary @@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None): def check_valid_format(output_format): if output_format not in valid_output_formats: - raise UsageError("Unrecognized output format '%s', set one after a" - " colon using the -o option (i.e. -o :)" - " or as a file extension, from the supported list %s" % - (output_format, tuple(valid_output_formats))) + raise UsageError( + "Unrecognized output format '%s'. Set a supported one (%s) " + "after a colon at the end of the output URI (i.e. -o/-O " + ":) or as a file extension." % ( + output_format, + tuple(valid_output_formats), + ) + ) + + overwrite = False + if overwrite_output: + if output: + raise UsageError( + "Please use only one of -o/--output and -O/--overwrite-output" + ) + output = overwrite_output + overwrite = True if output_format: if len(output) == 1: check_valid_format(output_format) - warnings.warn('The -t command line option is deprecated in favor' - ' of specifying the output format within the -o' - ' option, please check the -o option docs for more details', - category=ScrapyDeprecationWarning, stacklevel=2) + message = ( + 'The -t command line option is deprecated in favor of ' + 'specifying the output format within the output URI. See the ' + 'documentation of the -o and -O options for more information.', + ) + warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} else: - raise UsageError('The -t command line option cannot be used if multiple' - ' output files are specified with the -o option') + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result = {} for element in output: @@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None): feed_uri = 'stdout:' check_valid_format(feed_format) result[feed_uri] = {'format': feed_format} + if overwrite: + result[feed_uri]['overwrite'] = True - # FEEDS setting should take precedence over the -o and -t CLI options + # FEEDS setting should take precedence over the matching CLI options result.update(settings.getdict('FEEDS')) return result diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f07bdd748..19d56d6ec 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( *, path, file, host, port, - username, password, use_active_mode=False): + username, password, use_active_mode=False, overwrite=True): """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ @@ -32,4 +32,6 @@ def ftp_store_file( file.seek(0) dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) - ftp.storbinary('STOR %s' % filename, file) + command = 'STOR' if overwrite else 'APPE' + ftp.storbinary('%s %s' % (command, filename), file) + file.close() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 59f1b8371..1f2333264 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -198,7 +198,8 @@ def _getargspec_py23(func): def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = _getargspec_py23(func) + spec = inspect.getfullargspec(func) + func_args = spec.args + spec.kwonlyargs elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): diff --git a/tests/ftpserver.py b/tests/ftpserver.py new file mode 100644 index 000000000..6f0289e08 --- /dev/null +++ b/tests/ftpserver.py @@ -0,0 +1,24 @@ +from argparse import ArgumentParser + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer + + +def main(): + parser = ArgumentParser() + parser.add_argument('-d', '--directory') + args = parser.parse_args() + + authorizer = DummyAuthorizer() + full_permissions = 'elradfmwMT' + authorizer.add_anonymous(args.directory, perm=full_permissions) + handler = FTPHandler + handler.authorizer = authorizer + address = ('127.0.0.1', 2121) + server = FTPServer(address, handler) + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/tests/mockserver.py b/tests/mockserver.py index 1f40473ba..48d7b8d37 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -3,7 +3,10 @@ import json import os import random import sys +from pathlib import Path +from shutil import rmtree from subprocess import Popen, PIPE +from tempfile import mkdtemp from urllib.parse import urlencode from OpenSSL import SSL @@ -256,6 +259,29 @@ class MockDNSServer: self.proc.communicate() +class MockFTPServer: + """Creates an FTP server on port 2121 with a default passwordless user + (anonymous) and a temporary root path that you can read from the + :attr:`path` attribute.""" + + def __enter__(self): + self.path = Path(mkdtemp()) + self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)], + stderr=PIPE, env=get_testenv()) + for line in self.proc.stderr: + if b'starting FTP server' in line: + break + return self + + def __exit__(self, exc_type, exc_value, traceback): + rmtree(str(self.path)) + self.proc.kill() + self.proc.communicate() + + def url(self, path): + return 'ftp://127.0.0.1:2121/' + path + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b51177abb..fe1cbc997 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,6 +4,7 @@ dataclasses; python_version == '3.6' mitmproxy; python_version >= '3.7' mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' mitmproxy < 4; python_version < '3.6' +pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines diff --git a/tests/test_commands.py b/tests/test_commands.py index 109f006a8..f76f851e7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -74,6 +74,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() + p.communicate() assert False, 'Command took too much time to complete' timer = Timer(15, kill_proc) @@ -569,6 +570,55 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + class BenchCommandTest(CommandTest): @@ -577,3 +627,79 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) + + +class CrawlCommandTest(CommandTest): + + def crawl(self, code, args=()): + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(code) + return self.proc('crawl', 'myspider', *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('It works!') + return [] +""" + log = self.get_log(spider_code) + self.assertIn("[myspider] DEBUG: It works!", log) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2afc25a7a..850485b5e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import random import shutil import string import tempfile +import warnings from abc import ABC, abstractmethod from collections import defaultdict from io import BytesIO @@ -25,7 +26,7 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -46,7 +47,7 @@ from scrapy.utils.test import ( mock_google_cloud_storage, ) -from tests.mockserver import MockServer +from tests.mockserver import MockFTPServer, MockServer class FileFeedStorageTest(unittest.TestCase): @@ -75,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase): st = FileFeedStorage(path) verifyObject(IFeedStorage, st) + def _store(self, feed_options=None): + path = os.path.abspath(self.mktemp()) + storage = FileFeedStorage(path, feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + return path + + def test_append(self): + path = self._store() + return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") + + def test_overwrite(self): + path = self._store({"overwrite": True}) + return self._assert_stores( + FileFeedStorage(path, feed_options={"overwrite": True}), + path + ) + @defer.inlineCallbacks - def _assert_stores(self, storage, path): + def _assert_stores(self, storage, path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") @@ -84,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase): self.assertTrue(os.path.exists(path)) try: with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") + self.assertEqual(fp.read(), expected_content) finally: os.unlink(path) @@ -99,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase): spider = TestSpider.from_crawler(crawler) return spider - def test_store(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - st = FTPFeedStorage(uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + return storage.store(file) - def test_store_active_mode(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True} - crawler = get_crawler(settings_dict=use_active_mode) - st = FTPFeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _assert_stored(self, path, content): + self.assertTrue(path.exists()) + try: + with path.open('rb') as fp: + self.assertEqual(fp.read(), content) + finally: + os.unlink(str(path)) + + @defer.inlineCallbacks + def test_append(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options) + yield self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo") + yield self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @defer.inlineCallbacks + def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options, settings=settings) + yield self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo", settings=settings) + yield self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted) + st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted, + {}) self.assertEqual(st.password, string.punctuation) - @defer.inlineCallbacks - def _assert_stores(self, storage, path): - spider = self.get_test_spider() - file = storage.open(spider) - file.write(b"content") - yield storage.store(file) - self.assertTrue(os.path.exists(path)) - try: - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") - # again, to check s3 objects are overwritten - yield storage.store(BytesIO(b"new content")) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"new content") - finally: - os.unlink(path) - class BlockingFeedStorageTest(unittest.TestCase): @@ -190,8 +236,10 @@ class S3FeedStorageTest(unittest.TestCase): 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler - storage = S3FeedStorage.from_crawler(crawler, - 's3://mybucket/export.csv') + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') # Instantiate directly @@ -254,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -269,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -370,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertNotIn('S3 does not support appending to files', str(log)) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl', + feed_options={'overwrite': False}, + ) + self.assertIn('S3 does not support appending to files', str(log)) + class GCSFeedStorageTest(unittest.TestCase): @@ -439,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase): yield storage.store(file) self.assertEqual(out.getvalue(), b"content") + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:') + self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log)) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:', feed_options={'overwrite': True}) + self.assertIn('Standard output (stdout) storage does not support overwriting', str(log)) + class FromCrawlerMixin: init_with_crawler = False @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): cls.init_with_crawler = True return cls(*args, **kwargs) @@ -454,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin): class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): - pass + + @classmethod + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): + cls.init_with_crawler = True + return cls(*args, feed_options=feed_options, **kwargs) class DummyBlockingFeedStorage(BlockingFeedStorage): @@ -588,8 +671,8 @@ class FeedExportTest(FeedExportTestBase): FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - printf_escape(path_to_url(file_path)): feed - for file_path, feed in FEEDS.items() + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() } content = {} @@ -599,12 +682,12 @@ class FeedExportTest(FeedExportTestBase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - for file_path, feed in FEEDS.items(): + for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): continue with open(str(file_path), 'rb') as f: - content[feed['format']] = f.read() + content[feed_options['format']] = f.read() finally: for file_path in FEEDS.keys(): @@ -1542,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase): content = json.loads(content.decode('utf-8')) expected_batch, items = items[:batch_size], items[batch_size:] self.assertEqual(expected_batch, content) + + +class FeedExportInitTest(unittest.TestCase): + + def test_unsupported_storage(self): + settings = { + 'FEEDS': { + 'unsupported://uri': {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + def test_unsupported_format(self): + settings = { + 'FEEDS': { + 'file://path': { + 'format': 'unsupported_format', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + +class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FileFeedStorageWithoutFeedOptions(FileFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class S3FeedStorageWithoutFeedOptions(S3FeedStorage): + + def __init__(self, uri, access_key, secret_key, acl): + super().__init__(uri, access_key, secret_key, acl) + + +class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): + + def __init__(self, uri, use_active_mode=False): + super().__init__(uri) + + +class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f3ef36127..ccc65c4fd 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli(settings, ['-:pickle']) ) + def test_feed_export_config_overwrite(self): + settings = Settings() + self.assertEqual( + {'output.json': {'format': 'json', 'overwrite': True}}, + feed_process_params_from_cli(settings, [], None, ['output.json']) + ) + + def test_output_and_overwrite_output(self): + with self.assertRaises(UsageError): + feed_process_params_from_cli( + Settings(), + ['output1.json'], + None, + ['output2.json'], + ) + def test_feed_complete_default_values_from_settings_empty(self): feed = {} settings = Settings({ diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3f93f509e..c298d0bd2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass + def f3(a, b=None, *, c=None): + pass + class A: def __init__(self, a, b, c): pass @@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) + self.assertEqual(get_func_args(f3), ['a', 'b', 'c']) self.assertEqual(get_func_args(A), ['a', 'b', 'c']) self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) self.assertEqual(get_func_args(partial_f1), ['b', 'c']) From d9e69bfb51d5ed5a08d57878b29a6e7db8ef9d15 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Aug 2020 19:46:24 +0500 Subject: [PATCH 032/127] Re-enable TLS 1.2 in cipher tests. --- tests/mockserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 48d7b8d37..6f0c274b9 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -289,8 +289,8 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c ) if cipher_string: ctx = factory.getContext() - # disabling TLS1.2+ because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3) + # disabling TLS1.3 because it unconditionally enables some strong ciphers + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory From 42383cc267393c3c5fb89c9108759125939ab3e5 Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Wed, 19 Aug 2020 12:48:14 +0530 Subject: [PATCH 033/127] Add a setting to customize the asyncio event loop (#4414) --- docs/topics/asyncio.rst | 12 +++++++++++ docs/topics/settings.rst | 20 ++++++++++++++++++ scrapy/crawler.py | 2 +- scrapy/settings/default_settings.py | 2 ++ scrapy/utils/log.py | 7 +++++++ scrapy/utils/reactor.py | 12 ++++++++--- tests/CrawlerProcess/asyncio_custom_loop.py | 17 +++++++++++++++ tests/requirements-py3.txt | 1 + tests/test_commands.py | 23 +++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 10 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_custom_loop.py diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 038a459fd..bfb430d52 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -26,3 +26,15 @@ reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`:: install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + + + diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 722ae4593..618b9989e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -216,6 +216,26 @@ Default: ``None`` The name of the region associated with the AWS client. +.. setting:: ASYNCIO_EVENT_LOOP + +ASYNCIO_EVENT_LOOP +------------------ + +Default: ``None`` + +Import path of a given asyncio event loop class. + +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the +desired asyncio event loop class. If the setting is set to ``None`` the default asyncio +event loop will be used. + +If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. + +Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. + .. setting:: BOT_NAME BOT_NAME diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d028bea4d..4c6b0e496 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner): def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"]) + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) super()._handle_twisted_reactor() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 896afa995..a0251394b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,6 +19,8 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False +ASYNCIO_EVENT_LOOP = None + AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 1d6a2c39d..e41315738 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -150,6 +150,13 @@ def log_scrapy_info(settings): logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) + from twisted.internet import asyncioreactor + if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): + logger.debug( + "Using asyncio event loop: %s.%s", + reactor._asyncioEventloop.__module__, + reactor._asyncioEventloop.__class__.__name__, + ) class StreamLogger: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 3c705f69b..879d27907 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -50,13 +50,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def install_reactor(reactor_path): +def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified - import path.""" + import path. Also installs the asyncio event loop with the specified import + path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + else: + event_loop = asyncio.new_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") installer_path = module + ["install"] diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..1e4ada722 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop" +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index fe1cbc997..44ddcded8 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -13,6 +13,7 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures +uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython diff --git a/tests/test_commands.py b/tests/test_commands.py index f76f851e7..ee8a92604 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -16,6 +16,7 @@ from tempfile import mkdtemp from threading import Timer from unittest import skipIf +from pytest import mark from twisted.trial import unittest import scrapy @@ -570,6 +571,28 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', + 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor', + '-s', + 'ASYNCIO_EVENT_LOOP=uvloop.Loop', + ]) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + import asyncio + loop = asyncio.new_event_loop() + self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + def test_output(self): spider_code = """ import scrapy diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1a4cfe813..7c2e251a9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_loop_asyncio(self): + log = self.run_script("asyncio_custom_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From a57db9e3024fe1426021b8b692a48f4c8db82a77 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 19 Aug 2020 18:45:24 +0300 Subject: [PATCH 034/127] Bitbucket no longer supports Mercurial repositories (#4738) --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index db720b918..33a920bb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ install: - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" diff --git a/setup.py b/setup.py index d0880051f..52a27c368 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ if has_environment_marker_platform_impl_support(): ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by - # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no, + # https://foss.heptapod.net/pypy/pypy/-/issues/2498, # which was fixed in Cython 0.26, released on 2017-06-19, and used to # generate the C headers of lxml release tarballs published since then, the # first of which was: From d68aab992e435aa1c88061e83d03e57e7d31533d Mon Sep 17 00:00:00 2001 From: Grisha Temchenko Date: Thu, 20 Aug 2020 09:22:07 -0400 Subject: [PATCH 035/127] Smarter generator check for combined return/yield statements (#4721) --- scrapy/utils/misc.py | 19 ++++++++++++++++++- ...t_return_with_argument_inside_generator.py | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d6966be8e..bd400bd30 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -5,6 +5,7 @@ import os import re import hashlib import warnings +from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -184,6 +185,22 @@ def set_environ(**kwargs): os.environ[k] = v +def walk_callable(node): + """Similar to ``ast.walk``, but walks only function body and skips nested + functions defined within the node. + """ + todo = deque([node]) + walked_func_def = False + while todo: + node = todo.popleft() + if isinstance(node, ast.FunctionDef): + if walked_func_def: + continue + walked_func_def = True + todo.extend(ast.iter_child_nodes(node)) + yield node + + _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) @@ -201,7 +218,7 @@ def is_generator_with_return_value(callable): if inspect.isgeneratorfunction(callable): tree = ast.parse(dedent(inspect.getsource(callable))) - for node in ast.walk(tree): + for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True return _generator_callbacks_cache[callable] diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index bdbec1beb..2be38620c 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield 1 yield from g() + def m(): + yield 1 + + def helper(): + return 0 + + yield helper() + + def n(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + assert is_generator_with_return_value(f) assert is_generator_with_return_value(g) assert not is_generator_with_return_value(h) assert not is_generator_with_return_value(i) assert not is_generator_with_return_value(j) assert not is_generator_with_return_value(k) # not recursive + assert not is_generator_with_return_value(m) + assert is_generator_with_return_value(n) From 2fbfe2c21411480c35a99ff5497ee59f2e5e0dbb Mon Sep 17 00:00:00 2001 From: yogendra0sharma Date: Fri, 21 Aug 2020 12:18:15 +0530 Subject: [PATCH 036/127] Removed appveyor.xml no longer needed --- appveyor.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7fd636864..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ -platform: x86 -version: '{branch}-{build}' -environment: - matrix: - - PYTHON: "C:\\Python36" - TOX_ENV: py36 - -branches: - only: - - master - - /d+\.\d+\.\d+[\w\-]*$/ - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" - - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - - "pip install -U tox" - -build: false -skip_tags: true -test_script: - - "tox -e %TOX_ENV%" - -cache: - - '%LOCALAPPDATA%\pip\cache' From e90be0d8a5e3c20ab6aced22ccac59a876db8c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 14:09:52 +0200 Subject: [PATCH 037/127] Mark the new test as xfail for xmliter_lxml --- tests/test_utils_iterators.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index bbdc88dd1..ae64e36cf 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,6 @@ import os +from pytest import mark from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml @@ -149,6 +150,26 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath('id/text()').getall(), []) self.assertEqual(node.xpath('price/text()').getall(), []) + def test_xmliter_namespaced_nodename(self): + body = b""" + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) my_iter = self.xmliter(response, 'g:image_link') node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') @@ -187,6 +208,10 @@ class XmliterTestCase(unittest.TestCase): class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) + @mark.xfail(reason='known bug of the current implementation') + def test_xmliter_namespaced_nodename(self): + super().test_xmliter_namespaced_nodename() + def test_xmliter_iterate_namespace(self): body = b""" From afd3a4d116809ecf4c334657c5150c157aa9465b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 17:04:02 +0200 Subject: [PATCH 038/127] Fix style issue --- scrapy/utils/iterators.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c356ad7f8..6f6b5e337 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -41,7 +41,15 @@ def xmliter(obj, nodename): r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) for match in r.finditer(text): - nodetext = document_header + match.group().replace(nodename, '%s %s' % (nodename, ' '.join(namespaces.values())), 1) + header_end + nodetext = ( + document_header + + match.group().replace( + nodename, + '%s %s' % (nodename, ' '.join(namespaces.values())), + 1 + ) + + header_end + ) yield Selector(text=nodetext, type='xml') From 7c076122ebb16a6db9b85b68c85c354c2a49fd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 17:06:54 +0200 Subject: [PATCH 039/127] Skip checks introduced in Pylint 2.6.0 --- pylintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylintrc b/pylintrc index 129c7bf7d..5b6b9fab0 100644 --- a/pylintrc +++ b/pylintrc @@ -68,6 +68,7 @@ disable=abstract-method, pointless-statement, pointless-string-statement, protected-access, + raise-missing-from, redefined-argument-from-local, redefined-builtin, redefined-outer-name, @@ -75,6 +76,7 @@ disable=abstract-method, signature-differs, singleton-comparison, super-init-not-called, + super-with-arguments, superfluous-parens, too-few-public-methods, too-many-ancestors, From f1250177dc1c486517b9ca8ed136162533014da7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sat, 22 Aug 2020 04:33:35 -0300 Subject: [PATCH 040/127] Remove Python 3.5 from CI (#4743) --- .travis.yml | 12 +++--------- azure-pipelines.yml | 4 +--- tests/test_utils_python.py | 18 +++++++++--------- tox.ini | 4 ++-- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 33a920bb6..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,16 +19,10 @@ matrix: python: 3.8 - env: TOXENV=pinned - python: 3.5.2 + python: 3.6.1 - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + python: 3.6.1 + - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - env: TOXENV=py python: 3.6 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c03e258c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,11 +4,9 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned Python36: python.version: '3.6' + TOXENV: windows-pinned Python37: python.version: '3.7' Python38: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c298d0bd2..3115cc92f 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,8 +3,8 @@ import gc import operator import platform import unittest +from datetime import datetime from itertools import count -from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( @@ -216,15 +216,15 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) - else: - self.assertEqual( - get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - if version_info < (3, 6): - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) - else: + elif platform.python_implementation() == 'PyPy': + self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) + + build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y') + if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1 self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) + else: + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tox.ini b/tox.ini index 4f5531aea..dec0d75e8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = # Extras boto3>=1.13.0 botocore>=1.4.87 - Pillow>=3.4.2 + Pillow>=4.0.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -78,7 +78,7 @@ deps = # Extras botocore==1.4.87 google-cloud-storage==1.29.0 - Pillow==3.4.2 + Pillow==4.0.0 [testenv:pinned] deps = From 2d8ec9d44fa2201986da98c3f5f7c3c7f1ecfb56 Mon Sep 17 00:00:00 2001 From: WinterComes Date: Wed, 17 Jul 2019 22:50:34 +0300 Subject: [PATCH 041/127] Change DOWNLOAD_MAXSIZE logger level from Error to Warning --- scrapy/core/downloader/handlers/http11.py | 21 +++++++++++---------- tests/test_downloader_handlers.py | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index fb04d1fb7..a78b19318 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -394,13 +394,14 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - error_msg = ("Cancelling download of %(url)s: expected response " - "size (%(size)s) larger than download max size (%(maxsize)s).") - error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Expected response size (%(size)s) larger than " + "download max size (%(maxsize)s) in request %(request)s.") + warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize} + + logger.warning(warning_msg, warning_args) - logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_msg % error_args) + raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " @@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: - logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", - {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + logger.warning("Received (%(bytes)s) bytes larger than download " + "max size (%(maxsize)s) in request %(request)s.", + {'bytes': self._bytes_received, + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 13063d106..7059f0892 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('largechunkedfile')) def check(logger): - logger.error.assert_called_once_with(mock.ANY, mock.ANY) + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) d = self.download_request(request, Spider('foo', download_maxsize=1500)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) From 0b3881d65e12e7e0ac5f70c0f33e45dbab546c5d Mon Sep 17 00:00:00 2001 From: drs-11 Date: Mon, 24 Aug 2020 20:26:06 +0530 Subject: [PATCH 042/127] Reverted maxsize warning log message --- scrapy/core/downloader/handlers/http11.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a78b19318..25e800984 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -394,9 +394,9 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - warning_msg = ("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s) in request %(request)s.") - warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} logger.warning(warning_msg, warning_args) From 3e726b9df721f2288f4ae9a685de9bdff0762c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 11:22:05 +0100 Subject: [PATCH 043/127] Support for delegated methods as callbacks It can be useful to structure the spiders code around some helper classes. --- scrapy/utils/reqser.py | 29 ++++++++++++++--------------- tests/test_utils_reqser.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 5ea2aafb8..35a4fc72c 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -73,23 +73,22 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): if obj: try: - func_self = func.__self__ - except AttributeError: # func has no __self__ + func.__func__ + except AttributeError: # func is not a instance method. Not supported. pass else: - if func_self is obj: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError("Function %s is not a method of: %s" % (func, obj)) + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) def _get_method(obj, name): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index de94ec960..c8d1db138 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -102,6 +102,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_delegated_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.delegated_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -131,6 +137,9 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass +class TestSpiderDelegation: + def delegated_callback(self, response): + pass def parse_item(response): pass @@ -155,6 +164,9 @@ class TestSpider(Spider, TestSpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error + def __init__(self): + self.delegated_callback = TestSpiderDelegation().delegated_callback + def parse_item(self, response): pass From a2d6fa5adc17035cebb8dad4a3bd2020236b4f71 Mon Sep 17 00:00:00 2001 From: maranqz Date: Tue, 25 Aug 2020 13:34:43 +0300 Subject: [PATCH 044/127] Add errors parameter for CsvItemExporter with tests --- scrapy/exporters.py | 5 +++-- tests/test_exporters.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..8cd2077b6 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): + def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = 'utf-8' @@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + errors=errors, ) self.csv_writer = csv.writer(self.stream, **self._kwargs) self._headers_not_written = True diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6c25a0064..ebc477e74 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='22,False,3.14,2015-01-01 01:01:01\r\n' ) + def test_errors_default(self): + with self.assertRaises(UnicodeEncodeError): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + expected=None, + encoding='windows-1251', + ) + + def test_errors_xmlcharrefreplace(self): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + include_headers_line=False, + expected='Wɵ​rd\r\n', + encoding='windows-1251', + errors='xmlcharrefreplace', + ) + class CsvItemExporterDataclassTest(CsvItemExporterTest): item_class = TestDataClass From 39affea93c5b60cde89e000486c39c3c0ce87dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 13:57:48 +0200 Subject: [PATCH 045/127] Fix style issues --- tests/test_utils_reqser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index c8d1db138..ee68cf6b1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -137,10 +137,12 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass + class TestSpiderDelegation: def delegated_callback(self, response): pass + def parse_item(response): pass From 0524df866936506ae9438a5565fc4733fa5ba5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 14:36:38 +0100 Subject: [PATCH 046/127] Code simplification. Thanks @victor-torres for the suggestion --- scrapy/utils/reqser.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 35a4fc72c..503d7b133 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,23 +71,19 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): - if obj: - try: - func.__func__ - except AttributeError: # func is not a instance method. Not supported. - pass - else: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) From 2f28cee3ce482a9380c816b689fc6a5dabc60295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 17:49:17 +0200 Subject: [PATCH 047/127] Add a test to cover searching for a missing node name --- tests/test_utils_iterators.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ae64e36cf..2adccebb8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -175,6 +175,30 @@ class XmliterTestCase(unittest.TestCase): node.register_namespace('g', 'http://base.google.com/ns/1.0') self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + def test_xmliter_namespaced_nodename_missing(self): + body = b""" + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) + my_iter = self.xmliter(response, 'g:link_image') + with self.assertRaises(StopIteration): + next(my_iter) + def test_xmliter_exception(self): body = ( '' From 58ca8bbf6d1589bd0c8cc1ebda52299346f55e8a Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 22 Aug 2020 22:32:03 +0200 Subject: [PATCH 048/127] Use f-strings (#4307) --- docs/conf.py | 2 +- docs/intro/tutorial.rst | 6 +- docs/topics/developer-tools.rst | 138 +++++++++--------- docs/topics/exporters.rst | 6 +- docs/topics/item-pipeline.rst | 6 +- docs/topics/leaks.rst | 2 +- docs/topics/selectors.rst | 7 +- docs/topics/settings.rst | 2 +- docs/topics/spiders.rst | 4 +- extras/qps-bench-server.py | 2 +- extras/qpsclient.py | 4 +- scrapy/cmdline.py | 16 +- scrapy/commands/__init__.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 10 +- scrapy/commands/edit.py | 4 +- scrapy/commands/genspider.py | 27 ++-- scrapy/commands/parse.py | 4 +- scrapy/commands/runspider.py | 8 +- scrapy/commands/startproject.py | 12 +- scrapy/commands/version.py | 4 +- scrapy/contracts/__init__.py | 8 +- scrapy/contracts/default.py | 12 +- scrapy/core/downloader/__init__.py | 16 +- scrapy/core/downloader/handlers/__init__.py | 3 +- scrapy/core/downloader/handlers/http11.py | 17 ++- scrapy/core/downloader/handlers/s3.py | 6 +- scrapy/core/downloader/middleware.py | 15 +- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 12 +- scrapy/core/scraper.py | 4 +- scrapy/core/spidermw.py | 20 +-- scrapy/downloadermiddlewares/cookies.py | 12 +- scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 6 +- scrapy/downloadermiddlewares/stats.py | 6 +- scrapy/exporters.py | 2 +- scrapy/extensions/corestats.py | 2 +- scrapy/extensions/debug.py | 2 +- scrapy/extensions/httpcache.py | 10 +- scrapy/extensions/memdebug.py | 2 +- scrapy/extensions/memusage.py | 14 +- scrapy/extensions/statsmailer.py | 10 +- scrapy/http/common.py | 2 +- scrapy/http/headers.py | 2 +- scrapy/http/request/__init__.py | 12 +- scrapy/http/request/form.py | 19 ++- scrapy/http/response/__init__.py | 6 +- scrapy/http/response/text.py | 15 +- scrapy/item.py | 6 +- scrapy/link.py | 6 +- scrapy/logformatter.py | 4 +- scrapy/pipelines/files.py | 23 ++- scrapy/pipelines/images.py | 9 +- scrapy/pipelines/media.py | 7 +- scrapy/pqueues.py | 9 +- scrapy/responsetypes.py | 2 +- scrapy/selector/unified.py | 4 +- scrapy/settings/__init__.py | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 4 +- scrapy/spiderloader.py | 13 +- scrapy/spidermiddlewares/depth.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/offsite.py | 6 +- scrapy/spidermiddlewares/referer.py | 2 +- scrapy/spiders/__init__.py | 11 +- scrapy/spiders/feed.py | 4 +- scrapy/utils/benchserver.py | 6 +- scrapy/utils/conf.py | 22 +-- scrapy/utils/curl.py | 4 +- scrapy/utils/decorators.py | 4 +- scrapy/utils/deprecate.py | 11 +- scrapy/utils/engine.py | 4 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/iterators.py | 13 +- scrapy/utils/log.py | 4 +- scrapy/utils/misc.py | 15 +- scrapy/utils/project.py | 4 +- scrapy/utils/python.py | 12 +- scrapy/utils/reactor.py | 8 +- scrapy/utils/reqser.py | 4 +- scrapy/utils/response.py | 10 +- scrapy/utils/serialize.py | 6 +- scrapy/utils/ssl.py | 4 +- scrapy/utils/test.py | 2 +- scrapy/utils/testproc.py | 6 +- scrapy/utils/testsite.py | 4 +- scrapy/utils/trackref.py | 4 +- scrapy/utils/url.py | 4 +- sep/sep-002.rst | 2 +- sep/sep-004.rst | 4 +- sep/sep-014.rst | 29 ++-- sep/sep-018.rst | 22 +-- tests/CrawlerRunner/ip_address.py | 2 +- tests/mockserver.py | 10 +- tests/py36/_test_crawl.py | 2 +- tests/spiders.py | 8 +- tests/test_cmdline/extensions.py | 2 +- tests/test_command_check.py | 10 +- tests/test_command_parse.py | 16 +- tests/test_command_shell.py | 16 +- tests/test_command_version.py | 2 +- tests/test_commands.py | 32 ++-- tests/test_contracts.py | 2 +- tests/test_crawl.py | 6 +- tests/test_downloader_handlers.py | 22 +-- tests/test_downloadermiddleware.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 10 +- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_engine.py | 23 +-- tests/test_feedexport.py | 13 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 4 +- tests/test_pipeline_images.py | 4 +- tests/test_proxy_connect.py | 6 +- tests/test_request_attribute_binding.py | 4 +- tests/test_responsetypes.py | 12 +- tests/test_selector.py | 2 +- tests/test_signals.py | 2 +- tests/test_spidermiddleware_output_chain.py | 24 +-- tests/test_utils_curl.py | 2 +- tests/test_utils_datatypes.py | 2 +- tests/test_utils_defer.py | 8 +- tests/test_utils_iterators.py | 2 +- tests/test_utils_url.py | 6 +- tests/test_webclient.py | 8 +- 133 files changed, 561 insertions(+), 568 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 427c79481..27d2b5dff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ master_doc = 'index' # General information about the project. project = 'Scrapy' -copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) +copyright = f'2008–{datetime.now().year}, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f96c78887..914b91022 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) - self.log('Saved file %s' % filename) + self.log(f'Saved file {filename}') As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -190,7 +190,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 101aa159c..c83b1a9d9 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping ================================================= Here is a general guide on how to use your browser's Developer Tools -to ease the scraping process. Today almost all browsers come with +to ease the scraping process. Today almost all browsers come with built in `Developer Tools`_ and although we will use Firefox in this -guide, the concepts are applicable to any other browser. +guide, the concepts are applicable to any other browser. In this guide we'll introduce the basic tools to use from a browser's Developer Tools by scraping `quotes.toscrape.com`_. @@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things: Inspecting a website ==================== -By far the most handy feature of the Developer Tools is the `Inspector` -feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's look at the +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's look at the `quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific -tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes -on this page, without any meta-information about authors, tags, etc. +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. -Instead of viewing the whole source code for the page, we can simply right click +Instead of viewing the whole source code for the page, we can simply right click on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. In it you should see something like this: @@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] -Adding ``text()`` at the end we are able to extract the first quote with this +Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is -go down a desired path in the source code starting from ``html``. So let's -see if we can refine our XPath a bit: +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: -If we check the `Inspector` again we'll see that directly beneath our -expanded ``div`` tag we have nine identical ``div`` tags, each with the -same attributes as our first. If we expand any of them, we'll see the same +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div`` tag we have nine identical ``div`` tags, each with the +same attributes as our first. If we expand any of them, we'll see the same structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can -expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and see each quote: .. code-block:: html @@ -121,7 +121,7 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, -we'll simply select all ``span`` tags with the ``class="text"`` by using +we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: >>> response.xpath('//span[has-class("text")]/text()').getall() @@ -130,45 +130,45 @@ the `has-class-extension`_: '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', ...] -And with one simple, cleverer XPath we are able to extract all quotes from -the page. We could have constructed a loop over our first XPath to increase -the number of the last ``div``, but this would have been unnecessarily +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily complex and by simply constructing an XPath with ``has-class("text")`` -we were able to extract all quotes in one line. +we were able to extract all quotes in one line. -The `Inspector` has a lot of other helpful features, such as searching in the +The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate -a use case: +a use case: -Say you want to find the ``Next`` button on the page. Type ``Next`` into the -search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="next"``, the second the text +Say you want to find the ``Next`` button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extractor ` to -follow the pagination. On a simple site such as this, there may not be +we could easily create a :ref:`Link Extractor ` to +follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function -can be quite useful on complex sites. +can be quite useful on complex sites. Note that the search bar can also be used to search for and test CSS -selectors. For example, you could search for ``span.text`` to find -all quote texts. Instead of a full text search, this searches for -exactly the ``span`` tag with the ``class="text"`` in the page. +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page are loaded dynamically through multiple requests. While -this can be quite tricky, the `Network`-tool in the Developer Tools +of the page are loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's -take a look at the page `quotes.toscrape.com/scroll`_. +take a look at the page `quotes.toscrape.com/scroll`_. -The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next`` button, the page -automatically loads new quotes when you scroll to the bottom. We -could go ahead and try out different XPaths directly, but instead +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next`` button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the Scrapy shell: .. skip: next @@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell: (...) >>> view(response) -A browser window should open with the webpage but with one -crucial difference: Instead of the quotes we just see a greenish -bar with the word ``Loading...``. +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. .. image:: _images/network_01.png :width: 777 @@ -189,21 +189,21 @@ bar with the word ``Loading...``. :alt: Response from quotes.toscrape.com/scroll The ``view(response)`` command let's us view the response our -shell or later our spider receives from the server. Here we see -that some basic template is loaded which includes the title, +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request -than ``quotes.toscrape/scroll``. +than ``quotes.toscrape/scroll``. -If you click on the ``Network`` tab, you will probably only see -two entries. The first thing we do is enable persistent logs by -clicking on ``Persist Logs``. If this option is disabled, the +If you click on the ``Network`` tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different -page. Enabling this option is a good default, since it gives us -control on when to clear the logs. +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. If we reload the page now, you'll see the log get populated with six -new requests. +new requests. .. image:: _images/network_02.png :width: 777 @@ -212,31 +212,31 @@ new requests. Here we see every request that has been made when reloading the page and can inspect each request and its response. So let's find out -where our quotes are coming from: +where our quotes are coming from: -First click on the request with the name ``scroll``. On the right +First click on the request with the name ``scroll``. On the right you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Response``. -What you should see in the ``Preview`` pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response)`` in the -shell. Accordingly the ``type`` of the request in the log is ``html``. -The other requests have types like ``css`` or ``js``, but what -interests us is the one request called ``quotes?page=1`` with the -type ``json``. +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. -If we click on this request, we see that the request URL is +If we click on this request, we see that the request URL is ``http://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click -on the request and open ``Open in new tab`` to get a better overview. +on the request and open ``Open in new tab`` to get a better overview. .. image:: _images/network_03.png :width: 777 :height: 375 :alt: JSON-object returned from the quotes.toscrape API -With this response we can now easily parse the JSON-object and +With this response we can now easily parse the JSON-object and also request each page to get every quote on the site:: import scrapy @@ -255,17 +255,17 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) -This spider starts at the first page of the quotes-API. With each -response, we parse the ``response.text`` and assign it to ``data``. -This lets us operate on the JSON-object like on a Python dictionary. +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next`` element is ``true`` (try loading +If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page`` attribute -and ``yield`` a new request, inserting the incremented page-number +page-number greater than 10), we increment the ``page`` attribute +and ``yield`` a new request, inserting the incremented page-number into our ``url``. .. _requests-from-curl: @@ -298,7 +298,7 @@ Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. As you can see, with a few inspections in the `Network`-tool we -were able to easily replicate the dynamic requests of the scrolling +were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..793799a9a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -57,7 +57,7 @@ value of one of their fields:: adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open('{}.xml'.format(year), 'wb') + f = open(f'{year}.xml', 'wb') exporter = XmlItemExporter(f) exporter.start_exporting() self.year_to_exporter[year] = exporter @@ -98,7 +98,7 @@ Example:: import scrapy def serialize_price(value): - return '$ %s' % str(value) + return f'$ {str(value)}' class Product(scrapy.Item): name = scrapy.Field() @@ -122,7 +122,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': - return '$ %s' % str(value) + return f'$ {str(value)}' return super(Product, self).serialize_field(field, name, value) .. _topics-exporters-reference: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index cd6a6d47e..6287ee0ad 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -96,7 +96,7 @@ contain a price:: adapter['price'] = adapter['price'] * self.vat_factor return item else: - raise DropItem("Missing price in %s" % item) + raise DropItem(f"Missing price in {item}") Write items to a JSON file @@ -211,7 +211,7 @@ item. # Save screenshot to file, filename will be hash of url. url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() - filename = "{}.png".format(url_hash) + filename = f"{url_hash}.png" with open(filename, "wb") as f: f.write(response.body) @@ -240,7 +240,7 @@ returns multiples items with the same id:: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter['id'] in self.ids_seen: - raise DropItem("Duplicate item found: %r" % item) + raise DropItem(f"Duplicate item found: {item!r}") else: self.ids_seen.add(adapter['id']) return item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index d2f7edf0a..b895b95cb 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -102,7 +102,7 @@ A real example Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: - return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, + return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}", callback=self.parse, cb_kwargs={'referer': response}) That line is passing a response reference inside a request which effectively diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9e2c6ba42..b576fde91 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -328,8 +328,9 @@ too. Here's an example: 'Name: My image 5
'] >>> for index, link in enumerate(links): -... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) -... print('Link number %d points to url %r and image %r' % args) +... href_xpath = link.xpath('@href').get() +... img_xpath = link.xpath('img/@src').get() +... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}') Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' @@ -822,7 +823,7 @@ with groups of itemscopes and corresponding itemprops:: ... props = scope.xpath(''' ... set:difference(./descendant::*/@itemprop, ... .//*[@itemscope]/*/@itemprop)''') - ... print(" properties: %s" % (props.getall())) + ... print(f" properties: {props.getall()}") ... print("") current scope: ['http://schema.org/Product'] diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 618b9989e..22d60f87c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -110,7 +110,7 @@ In a spider, the settings are available through ``self.settings``:: start_urls = ['http://example.com'] def parse(self, response): - print("Existing settings: %s" % self.settings.attributes.keys()) + print(f"Existing settings: {self.settings.attributes.keys()}") .. note:: The ``settings`` attribute is set in the base Spider class after the spider diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e50e4aa0a..2056664c7 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods:: def __init__(self, category=None, *args, **kwargs): super(MySpider, self).__init__(*args, **kwargs) - self.start_urls = ['http://www.example.com/categories/%s' % category] + self.start_urls = [f'http://www.example.com/categories/{category}'] # ... The default `__init__` method will take any spider arguments @@ -292,7 +292,7 @@ The above example can also be written as follows:: name = 'myspider' def start_requests(self): - yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + yield scrapy.Request(f'http://www.example.com/categories/{self.category}') Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index da7a0022b..a6472b1ba 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -37,7 +37,7 @@ class Root(Resource): if now - self.lastmark >= 3: self.lastmark = now qps = len(self.tail) / sum(self.tail) - print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps)) + print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}') if 'latency' in request.args: latency = float(request.args['latency'][0]) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index fe1f96cbb..f9fb70342 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -37,11 +37,11 @@ class QPSSpider(Spider): def start_requests(self): url = self.benchurl if self.latency is not None: - url += '?latency={0}'.format(self.latency) + url += f'?latency={self.latency}' slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] + urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)] else: urls = [url] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 3e88536e4..91482ce01 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -44,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'): if inspect.isclass(obj): cmds[entry_point.name] = obj() else: - raise Exception("Invalid entry point %s" % entry_point.name) + raise Exception(f"Invalid entry point {entry_point.name}") return cmds @@ -67,11 +67,11 @@ def _pop_command_name(argv): def _print_header(settings, inproject): + version = scrapy.__version__ if inproject: - print("Scrapy %s - project: %s\n" % (scrapy.__version__, - settings['BOT_NAME'])) + print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") else: - print("Scrapy %s - no active project\n" % scrapy.__version__) + print(f"Scrapy {version} - no active project\n") def _print_commands(settings, inproject): @@ -81,7 +81,7 @@ def _print_commands(settings, inproject): print("Available commands:") cmds = _get_commands_dict(settings, inproject) for cmdname, cmdclass in sorted(cmds.items()): - print(" %-13s %s" % (cmdname, cmdclass.short_desc())) + print(f" {cmdname:<13} {cmdclass.short_desc()}") if not inproject: print() print(" [ more ] More commands available when run from project directory") @@ -91,7 +91,7 @@ def _print_commands(settings, inproject): def _print_unknown_command(settings, cmdname, inproject): _print_header(settings, inproject) - print("Unknown command: %s\n" % cmdname) + print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') @@ -133,7 +133,7 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax()) + parser.usage = f"scrapy {cmdname} {cmd.syntax()}" parser.description = cmd.long_desc() settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings @@ -155,7 +155,7 @@ def _run_command(cmd, args, opts): def _run_command_profiled(cmd, args, opts): if opts.profile: - sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx('cmd.run(args, opts)', globals(), loc) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index cfd940fe7..23ccffcd9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -61,7 +61,7 @@ class ScrapyCommand: group.add_option("--logfile", metavar="FILE", help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help=f"log level (default: {self.settings['LOG_LEVEL']})") group.add_option("--nolog", action="store_true", help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index c9f3b38e0..999c987ea 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1)) + url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 09a76ca7a..7e848dc97 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start)) + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}") writeln() infos = [] @@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult): write("FAILED") failed, errored = map(len, (self.failures, self.errors)) if failed: - infos.append("failures=%d" % failed) + infos.append(f"failures={failed}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"errors={errored}") else: write("OK") if infos: - writeln(" (%s)" % (", ".join(infos),)) + writeln(f" ({', '.join(infos)})") else: write("\n") @@ -85,7 +85,7 @@ class Command(ScrapyCommand): continue print(spider) for method in sorted(methods): - print(' * %s' % method) + print(f' * {method}') else: start = time.time() self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 25d843a53..177b20143 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -32,8 +32,8 @@ class Command(ScrapyCommand): try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: - return self._err("Spider not found: %s" % args[0]) + return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') - self.exitcode = os.system('%s "%s"' % (editor, sfile)) + self.exitcode = os.system(f'{editor} "{sfile}"') diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 74a077d1b..72248bded 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -73,17 +73,18 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, domain, opts.template, template_file) if opts.edit: - self.exitcode = os.system('scrapy edit "%s"' % name) + self.exitcode = os.system(f'scrapy edit "{name}"') def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" + capitalized_module = ''.join(s.capitalize() for s in module.split('_')) tvars = { 'project_name': self.settings.get('BOT_NAME'), 'ProjectName': string_camelcase(self.settings.get('BOT_NAME')), 'module': module, 'name': name, 'domain': domain, - 'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_')) + 'classname': f'{capitalized_module}Spider' } if self.settings.get('NEWSPIDER_MODULE'): spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) @@ -91,32 +92,32 @@ class Command(ScrapyCommand): else: spiders_module = None spiders_dir = "." - spider_file = "%s.py" % join(spiders_dir, module) + spider_file = f"{join(spiders_dir, module)}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r " - % (name, template_name), end=('' if spiders_module else '\n')) + print(f"Created spider {name!r} using template {template_name!r} ", + end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n %s.%s" % (spiders_module.__name__, module)) + print("in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): - template_file = join(self.templates_dir, '%s.tmpl' % template) + template_file = join(self.templates_dir, f'{template}.tmpl') if exists(template_file): return template_file - print("Unable to find template: %s\n" % template) + print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') def _list_templates(self): print("Available templates:") for filename in sorted(os.listdir(self.templates_dir)): if filename.endswith('.tmpl'): - print(" %s" % splitext(filename)[0]) + print(f" {splitext(filename)[0]}") def _spider_exists(self, name): if not self.settings.get('NEWSPIDER_MODULE'): # if run as a standalone command and file with same filename already exists if exists(name + ".py"): - print("%s already exists" % (abspath(name + ".py"))) + print(f"{abspath(name + '.py')} already exists") return True return False @@ -126,8 +127,8 @@ class Command(ScrapyCommand): pass else: # if spider with same name exists - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) + print(f"Spider {name!r} already exists in module:") + print(f" {spidercls.__module__}") return True # a file with the same name exists in the target directory @@ -135,7 +136,7 @@ class Command(ScrapyCommand): spiders_dir = dirname(spiders_module.__file__) spiders_dir_abs = abspath(spiders_dir) if exists(join(spiders_dir_abs, name + ".py")): - print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) + print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") return True return False diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index abc8ba9ff..83ee074da 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand): if opts.verbose: for level in range(1, self.max_level + 1): - print('\n>>> DEPTH LEVEL: %s <<<' % level) + print(f'\n>>> DEPTH LEVEL: {level} <<<') if not opts.noitems: self.print_items(level, colour) if not opts.nolinks: self.print_requests(level, colour) else: - print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level) + print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<') if not opts.noitems: self.print_items(colour=colour) if not opts.nolinks: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index befee021b..aedd8c2ce 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,7 +12,7 @@ def _import_file(filepath): dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) if fext != '.py': - raise ValueError("Not a Python source file: %s" % abspath) + raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path try: @@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand): raise UsageError() filename = args[0] if not os.path.exists(filename): - raise UsageError("File not found: %s\n" % filename) + raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError("Unable to load %r: %s\n" % (filename, e)) + raise UsageError(f"Unable to load {filename!r}: {e}\n") spclasses = list(iter_spider_classes(module)) if not spclasses: - raise UsageError("No spider found in file: %s\n" % filename) + raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() self.crawler_process.crawl(spidercls, **opts.spargs) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e5158d993..1d73fa0cb 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -52,7 +52,7 @@ class Command(ScrapyCommand): print('Error: Project names must begin with a letter and contain' ' only\nletters, numbers and underscores') elif _module_exists(project_name): - print('Error: Module %r already exists' % project_name) + print(f'Error: Module {project_name!r} already exists') else: return True return False @@ -100,7 +100,7 @@ class Command(ScrapyCommand): if exists(join(project_dir, 'scrapy.cfg')): self.exitcode = 1 - print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) + print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') return if not self._is_valid_name(project_name): @@ -113,11 +113,11 @@ class Command(ScrapyCommand): path = join(*paths) tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project '%s', using template directory '%s', " - "created in:" % (project_name, self.templates_dir)) - print(" %s\n" % abspath(project_dir)) + print(f"New Scrapy project '{project_name}', using template directory " + f"'{self.templates_dir}', created in:") + print(f" {abspath(project_dir)}\n") print("You can start your first spider with:") - print(" cd %s" % project_dir) + print(f" cd {project_dir}") print(" scrapy genspider example example.com") @property diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index d0ea72a67..dc8087043 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,8 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = "%-{}s : %s".format(width) + patt = f"%-{width}s : %s" for name, version in versions: print(patt % (name, version)) else: - print("Scrapy %s" % scrapy.__version__) + print(f"Scrapy {scrapy.__version__}") diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 5af3831a2..db0a56e56 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -112,8 +112,8 @@ class Contract: request_cls = None def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) - self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name) + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') self.args = args def add_pre_hook(self, request, results): @@ -172,8 +172,8 @@ def _create_testcase(method, desc): class ContractTestCase(TestCase): def __str__(_self): - return "[%s] %s (%s)" % (spider, method.__name__, desc) + return f"[{spider}] {method.__name__} ({desc})" - name = '%s_%s' % (spider, method.__name__) + name = f'{spider}_{method.__name__}' setattr(ContractTestCase, name, lambda x: x) return ContractTestCase(name) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index cfdcc7c25..9704f5253 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -60,8 +60,7 @@ class ReturnsContract(Contract): if len(self.args) not in [1, 2, 3]: raise ValueError( - "Incorrect argument quantity: expected 1, 2 or 3, got %i" - % len(self.args) + f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) self.obj_name = self.args[0] or None self.obj_type_verifier = self.object_type_verifiers[self.obj_name] @@ -88,10 +87,9 @@ class ReturnsContract(Contract): if self.min_bound == self.max_bound: expected = self.min_bound else: - expected = '%s..%s' % (self.min_bound, self.max_bound) + expected = f'{self.min_bound}..{self.max_bound}' - raise ContractFail("Returned %s %s, expected %s" % - (occurrences, self.obj_name, expected)) + raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}") class ScrapesContract(Contract): @@ -106,5 +104,5 @@ class ScrapesContract(Contract): if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] if missing: - missing_str = ", ".join(missing) - raise ContractFail("Missing fields: %s" % missing_str) + missing_fields = ", ".join(missing) + raise ContractFail(f"Missing fields: {missing_fields}") diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index dc5cf1ab8..12a9db6dd 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -41,17 +41,17 @@ class Slot: def __repr__(self): cls_name = self.__class__.__name__ - return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( - cls_name, self.concurrency, self.delay, self.randomize_delay) + return (f"{cls_name}(concurrency={self.concurrency!r}, " + f"delay={self.delay:.2f}, " + f"randomize_delay={self.randomize_delay!r}") def __str__(self): return ( - "" % ( - self.concurrency, self.delay, self.randomize_delay, - len(self.active), len(self.queue), len(self.transferring), - datetime.fromtimestamp(self.lastseen).isoformat() - ) + f"" ) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index e86680978..73aeb2352 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -71,8 +71,7 @@ class DownloadHandlers: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: - raise NotSupported("Unsupported URL scheme '%s': %s" % - (scheme, self._notconfigured[scheme])) + raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}") return handler.download_request(request, spider) @defer.inlineCallbacks diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 25e800984..1b041c8a8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -60,11 +60,11 @@ class HTTP11DownloadHandler: settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = f""" + '{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \ + argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \ + `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') @@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): else: extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( - self._host, self._port, extra))) + TunnelError('Could not open CONNECT tunnel with proxy ' + f'{self._host}:{self._port} [{extra!r}]') + ) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -371,7 +372,7 @@ class ScrapyAgent: if self._txresponse: self._txresponse._transport.stopProducing() - raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result, request, start_time): request.meta['download_latency'] = time() - start_time diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 8f63ad974..0ef977893 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -56,7 +56,7 @@ class S3DownloadHandler: import botocore.credentials kw.pop('anon', None) if kw: - raise TypeError('Unexpected keyword arguments: %s' % kw) + raise TypeError(f'Unexpected keyword arguments: {kw}') if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( @@ -85,14 +85,14 @@ class S3DownloadHandler: scheme = 'https' if request.meta.get('is_secure') else 'http' bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path - url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) + url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) elif self._signer is not None: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, - url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path), + url=f'{scheme}://s3.amazonaws.com/{bucket}{path}', headers=request.headers.to_unicode_dict(), data=request.body) self._signer.add_auth(awsrequest) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 4c2eea522..b0e612e43 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_request must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, response.__class__.__name__) + f"Middleware {method.__self__.__class__.__name__}" + ".process_request must return None, Response or " + f"Request, got {response.__class__.__name__}" ) if response: return response @@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_response must return Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_response must return Response or Request, " + f"got {type(response)}" ) if isinstance(response, Request): return response @@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_exception must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_exception must return None, Response or " + f"Request, got {type(response)}" ) if response: return response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index b2b96f1ea..c13683393 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." - % (self.factory.url, self.factory.timeout))) + defer.TimeoutError(f"Getting {self.factory.url} took longer " + f"than {self.factory.timeout} seconds.")) # This class used to inherit from Twisted’s @@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.headers['Content-Length'] = 0 def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.url) + return f"<{self.__class__.__name__}: {self.url}>" def _cancelTimeout(self, result, timeoutCall): if timeoutCall.active(): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5e0dfe37c..93bcdb49a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -171,8 +171,8 @@ class ExecutionEngine: def _handle_downloader_output(self, response, request, spider): if not isinstance(response, (Request, Response, Failure)): raise TypeError( - "Incorrect type: expected Request, Response or Failure, got %s: %r" - % (type(response), response) + "Incorrect type: expected Request, Response or Failure, got " + f"{type(response)}: {response!r}" ) # downloader middleware can return requests (for example, redirects) if isinstance(response, Request): @@ -214,7 +214,7 @@ class ExecutionEngine: def crawl(self, request, spider): if spider not in self.open_spiders: - raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) + raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") self.schedule(request, spider) self.slot.nextcall.schedule() @@ -239,8 +239,8 @@ class ExecutionEngine: def _on_success(response): if not isinstance(response, (Response, Request)): raise TypeError( - "Incorrect type: expected Response or Request, got %s: %r" - % (type(response), response) + "Incorrect type: expected Response or Request, got " + f"{type(response)}: {response!r}" ) if isinstance(response, Response): if response.request is None: @@ -268,7 +268,7 @@ class ExecutionEngine: @defer.inlineCallbacks def open_spider(self, spider, start_requests=(), close_if_idle=True): if not self.has_capacity(): - raise RuntimeError("No free spider slot when opening %r" % spider.name) + raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 20bdb22a1..0d3e3450f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -125,7 +125,7 @@ class Scraper: Handle the downloaded response or failure through the spider callback/errback """ if not isinstance(result, (Response, Failure)): - raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) + raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}") dfd = self._scrape2(result, request, spider) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) dfd.addCallback(self.handle_spider_output, request, result, spider) @@ -173,7 +173,7 @@ class Scraper: spider=spider ) self.crawler.stats.inc_value( - "spider_exceptions/%s" % _failure.value.__class__.__name__, + f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 5a99b96be..763e0cdf6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,10 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "{}.{}".format( - f.__self__.__class__.__name__, - f.__func__.__name__ - ) + return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" class SpiderMiddlewareManager(MiddlewareManager): @@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = "Middleware {} must return None or raise an exception, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) except _InvalidOutput: raise except Exception: @@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = "Middleware {} must return None or an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) return _failure def process_spider_output(result, start_index=0): @@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = _evaluate_iterable(result, method_index + 1, recovered) else: - msg = "Middleware {} must return an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) return MutableChain(result, recovered) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 77048f389..e2b7dd901 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -54,8 +54,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: - cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) - msg = "Sending cookies to: {}\n{}".format(request, cookies) + cookies = "\n".join(f"Cookie: {c}\n" for c in cl) + msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _debug_set_cookie(self, response, spider): @@ -63,8 +63,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: - cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) - msg = "Received cookies from: {}\n{}".format(response, cookies) + cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) + msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _format_cookie(self, cookie, request): @@ -90,9 +90,9 @@ class CookiesMiddleware: request, cookie) decoded[key] = cookie[key].decode("latin1", errors="replace") - cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain - cookie_str += "; {}={}".format(key.capitalize(), value) + cookie_str += f"; {key.capitalize()}={value}" return cookie_str def _get_request_cookies(self, jar, request): diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index da89d3e9b..04da11311 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -24,7 +24,7 @@ class HttpProxyMiddleware: def _basic_auth_header(self, username, password): user_pass = to_bytes( - '%s:%s' % (unquote(username), unquote(password)), + f'{unquote(username)}:{unquote(password)}', encoding=self.auth_encoding) return base64.b64encode(user_pass) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 67be8c282..51fe59254 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -88,7 +88,7 @@ class RetryMiddleware: reason = global_object_name(reason.__class__) stats.inc_value('retry/count') - stats.inc_value('retry/reason_count/%s' % reason) + stats.inc_value(f'retry/reason_count/{reason}') return retryreq else: stats.inc_value('retry/max_reached') diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f18b2bf2..d6da55535 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -61,7 +61,7 @@ class RobotsTxtMiddleware: if netloc not in self._parsers: self._parsers[netloc] = Deferred() - robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) + robotsurl = f"{url.scheme}://{url.netloc}/robots.txt" robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, @@ -94,7 +94,7 @@ class RobotsTxtMiddleware: def _parse_robots(self, response, netloc, spider): self.crawler.stats.inc_value('robotstxt/response_count') - self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status)) + self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}') rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp @@ -102,7 +102,7 @@ class RobotsTxtMiddleware: def _robots_error(self, failure, netloc): if failure.type is not IgnoreRequest: - key = 'robotstxt/exception_count/{}'.format(failure.type) + key = f'robotstxt/exception_count/{failure.type}' self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 46a2ad397..5479cd0e2 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -17,13 +17,13 @@ class DownloaderStats: def process_request(self, request, spider): self.stats.inc_value('downloader/request_count', spider=spider) - self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) + self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider) reqlen = len(request_httprepr(request)) self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) - self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) + self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) reslen = len(response_httprepr(response)) self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response @@ -31,4 +31,4 @@ class DownloaderStats: def process_exception(self, request, exception, spider): ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) - self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) + self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..54cf5c0b1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -39,7 +39,7 @@ class BaseItemExporter: self.export_empty_fields = options.pop('export_empty_fields', False) self.indent = options.pop('indent', None) if not dont_fail and options: - raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) + raise TypeError(f"Unexpected options: {', '.join(options.keys())}") def export_item(self, item): raise NotImplementedError diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 389cb65bc..675f8276f 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -43,4 +43,4 @@ class CoreStats: def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ self.stats.inc_value('item_dropped_count', spider=spider) - self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) + self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 586399784..fd2a02d8d 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -48,7 +48,7 @@ class StackTraceDump: for id_, frame in sys._current_frames().items(): name = id2name.get(id_, '') dump = ''.join(traceback.format_stack(frame)) - dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump) + dumps += f"# Thread: {name}({id_})\n{dump}\n" return dumps diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 6294a9b52..e0c04b2de 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -223,7 +223,7 @@ class DbmCacheStorage: self.db = None def open_spider(self, spider): - dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) + dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) @@ -251,13 +251,13 @@ class DbmCacheStorage: 'headers': dict(response.headers), 'body': response.body, } - self.db['%s_data' % key] = pickle.dumps(data, protocol=4) - self.db['%s_time' % key] = str(time()) + self.db[f'{key}_data'] = pickle.dumps(data, protocol=4) + self.db[f'{key}_time'] = str(time()) def _read_data(self, spider, request): key = self._request_key(request) db = self.db - tkey = '%s_time' % key + tkey = f'{key}_time' if tkey not in db: return # not found @@ -265,7 +265,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db['%s_data' % key]) + return pickle.loads(db[f'{key}_data']) def _request_key(self, request): return request_fingerprint(request) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index dc8cdbb1d..cee44ea62 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -30,4 +30,4 @@ class MemoryDebugger: for cls, wdict in live_refs.items(): if not wdict: continue - self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) + self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ab2e43e8c..274cbdbfe 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -82,8 +82,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s terminated: memory usage exceeded %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} terminated: " + f"memory usage exceeded {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -105,8 +105,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s warning: memory usage reached %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} warning: " + f"memory usage reached {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) @@ -115,9 +115,9 @@ class MemoryUsage: def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" stats = self.crawler.stats - s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) - s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) - s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 320f13b29..997e74fc9 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -24,11 +24,11 @@ class StatsMailer: o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - + def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.name - body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) + body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats()) + body += f"\n\n{spider.name} stats\n\n" + body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats) + return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) diff --git a/scrapy/http/common.py b/scrapy/http/common.py index ba6ab277c..98699d7fd 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,6 +1,6 @@ def obsolete_setter(setter, attrname): def newsetter(self, value): c = self.__class__.__name__ - msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) + msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) return newsetter diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 6bf9e5346..1a2b99b0a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -33,7 +33,7 @@ class Headers(CaselessDict): elif isinstance(x, int): return str(x).encode(self.encoding) else: - raise TypeError('Unsupported value type: {}'.format(type(x))) + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a98ba9960..ef58deacc 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -25,13 +25,13 @@ class Request(object_ref): self._set_url(url) self._set_body(body) if not isinstance(priority, int): - raise TypeError("Request priority not an integer: %r" % priority) + raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority if callback is not None and not callable(callback): - raise TypeError('callback must be a callable, got %s' % type(callback).__name__) + raise TypeError(f'callback must be a callable, got {type(callback).__name__}') if errback is not None and not callable(errback): - raise TypeError('errback must be a callable, got %s' % type(errback).__name__) + raise TypeError(f'errback must be a callable, got {type(errback).__name__}') self.callback = callback self.errback = errback @@ -60,13 +60,13 @@ class Request(object_ref): def _set_url(self, url): if not isinstance(url, str): - raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) + raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) if ('://' not in self._url) and (not self._url.startswith('data:')): - raise ValueError('Missing scheme in request url: %s' % self._url) + raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -86,7 +86,7 @@ class Request(object_ref): return self._encoding def __str__(self): - return "<%s %s>" % (self.method, self.url) + return f"<{self.method} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 59af81321..c90d68fa1 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath): base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: - raise ValueError("No element found in %s" % response) + raise ValueError(f"No element found in {response}") if formname is not None: - f = root.xpath('//form[@name="%s"]' % formname) + f = root.xpath(f'//form[@name="{formname}"]') if f: return f[0] if formid is not None: - f = root.xpath('//form[@id="%s"]' % formid) + f = root.xpath(f'//form[@id="{formid}"]') if f: return f[0] @@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + raise ValueError(f'No element found with {formxpath}') # If we get here, it means that either formname was None # or invalid @@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): try: form = forms[formnumber] except IndexError: - raise IndexError("Form number %d not found in %s" % - (formnumber, response)) + raise IndexError(f"Form number {formnumber} not found in {response}") else: return form @@ -205,12 +204,12 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') elif len(el) > 1: - raise ValueError("Multiple elements found (%r) matching the criteria " - "in clickdata: %r" % (el, clickdata)) + raise ValueError(f"Multiple elements found ({el!r}) matching the " + f"criteria in clickdata: {clickdata!r}") else: - raise ValueError('No clickable element matching clickdata: %r' % (clickdata,)) + raise ValueError(f'No clickable element matching clickdata: {clickdata!r}') diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c2c37dd1d..c635fde69 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -55,8 +55,8 @@ class Response(object_ref): if isinstance(url, str): self._url = url else: - raise TypeError('%s url must be str, got %s:' % - (type(self).__name__, type(url).__name__)) + raise TypeError(f'{type(self).__name__} url must be str, ' + f'got {type(url).__name__}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -77,7 +77,7 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) def __str__(self): - return "<%d %s>" % (self.status, self.url) + return f"<{self.status} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a7bb34d48..e36e14880 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -47,8 +47,8 @@ class TextResponse(Response): self._body = b'' # used by encoding detection if isinstance(body, str): if self._encoding is None: - raise TypeError('Cannot convert unicode body - %s has no encoding' % - type(self).__name__) + raise TypeError('Cannot convert unicode body - ' + f'{type(self).__name__} has no encoding') self._body = body.encode(self._encoding) else: super()._set_body(body) @@ -92,7 +92,7 @@ class TextResponse(Response): # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: - charset = 'charset=%s' % benc + charset = f'charset={benc}' self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody @@ -255,12 +255,11 @@ def _url_from_selector(sel): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): - raise _InvalidSelector("Unsupported selector: %s" % sel) + raise _InvalidSelector(f"Unsupported selector: {sel}") if sel.root.tag not in ('a', 'link'): - raise _InvalidSelector("Only and elements are supported; got <%s>" % - sel.root.tag) + raise _InvalidSelector("Only and elements are supported; " + f"got <{sel.root.tag}>") href = sel.root.get('href') if href is None: - raise _InvalidSelector("<%s> element has no href attribute: %s" % - (sel.root.tag, sel)) + raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}") return strip_html5_whitespace(href) diff --git a/scrapy/item.py b/scrapy/item.py index c262a153c..af3849302 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -96,19 +96,19 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) + raise KeyError(f"{self.__class__.__name__} does not support field: {key}") def __delitem__(self, key): del self._values[key] def __getattr__(self, name): if name in self.fields: - raise AttributeError("Use item[%r] to get field value" % name) + raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) + raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) def __len__(self): diff --git a/scrapy/link.py b/scrapy/link.py index 1ef50b113..684735f6e 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -14,7 +14,7 @@ class Link: def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): got = url.__class__.__name__ - raise TypeError("Link urls must be str objects, got %s" % got) + raise TypeError(f"Link urls must be str objects, got {got}") self.url = url self.text = text self.fragment = fragment @@ -33,6 +33,6 @@ class Link: def __repr__(self): return ( - 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' - % (self.url, self.text, self.fragment, self.nofollow) + f'Link(url={self.url!r}, text={self.text!r}, ' + f'fragment={self.fragment!r}, nofollow={self.nofollow!r})' ) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 0f9e6f1cb..87568b2d1 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -54,8 +54,8 @@ class LogFormatter: def crawled(self, request, response, spider): """Logs a message when the crawler finds a webpage.""" - request_flags = ' %s' % str(request.flags) if request.flags else '' - response_flags = ' %s' % str(response.flags) if response.flags else '' + request_flags = f' {str(request.flags)}' if request.flags else '' + response_flags = f' {str(response.flags)}' if response.flags else '' return { 'level': logging.DEBUG, 'msg': CRAWLEDMSG, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5a2184681..99a72aa70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -108,7 +108,7 @@ class S3FilesStore: from boto.s3.connection import S3Connection self.S3Connection = S3Connection if not uri.startswith("s3://"): - raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): @@ -133,7 +133,7 @@ class S3FilesStore: return c.get_bucket(self.bucket, validate=False) def _get_boto_key(self, path): - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' if self.is_botocore: return threads.deferToThread( self.s3_client.head_object, @@ -145,7 +145,7 @@ class S3FilesStore: def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' buf.seek(0) if self.is_botocore: extra = self._headers_to_botocore_kwargs(self.HEADERS) @@ -208,8 +208,7 @@ class S3FilesStore: try: kwarg = mapping[key] except KeyError: - raise TypeError( - 'Header "%s" is not supported by botocore' % key) + raise TypeError(f'Header "{key}" is not supported by botocore') else: extra[kwarg] = value return extra @@ -283,7 +282,7 @@ class FTPFilesStore: def __init__(self, uri): if not uri.startswith("ftp://"): - raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) self.port = u.port self.host = u.hostname @@ -293,7 +292,7 @@ class FTPFilesStore: self.basedir = u.path.rstrip('/') def persist_file(self, path, buf, info, meta=None, headers=None): - path = '%s/%s' % (self.basedir, path) + path = f'{self.basedir}/{path}' return threads.deferToThread( ftp_store_file, path=path, file=buf, host=self.host, port=self.port, username=self.username, @@ -308,10 +307,10 @@ class FTPFilesStore: ftp.login(self.username, self.password) if self.USE_ACTIVE_MODE: ftp.set_pasv(False) - file_path = "%s/%s" % (self.basedir, path) - last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip()) + file_path = f"{self.basedir}/{path}" + last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) m = hashlib.md5() - ftp.retrbinary('RETR %s' % file_path, m.update) + ftp.retrbinary(f'RETR {file_path}', m.update) return {'last_modified': last_modified, 'checksum': m.hexdigest()} # The file doesn't exist except Exception: @@ -515,7 +514,7 @@ class FilesPipeline(MediaPipeline): def inc_stats(self, spider, status): spider.crawler.stats.inc_value('file_count', spider=spider) - spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider) # Overridable Interface def get_media_requests(self, item, info): @@ -545,4 +544,4 @@ class FilesPipeline(MediaPipeline): media_type = mimetypes.guess_type(request.url)[0] if media_type: media_ext = mimetypes.guess_extension(media_type) - return 'full/%s%s' % (media_guid, media_ext) + return f'full/{media_guid}{media_ext}' diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 0a67a0b1d..aafd1d8b2 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -125,8 +125,9 @@ class ImagesPipeline(FilesPipeline): width, height = orig_image.size if width < self.min_width or height < self.min_height: - raise ImageException("Image too small (%dx%d < %dx%d)" % - (width, height, self.min_width, self.min_height)) + raise ImageException("Image too small " + f"({width}x{height} < " + f"{self.min_width}x{self.min_height})") image, buf = self.convert_image(orig_image) yield path, image, buf @@ -168,8 +169,8 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'full/%s.jpg' % (image_guid) + return f'full/{image_guid}.jpg' def thumb_path(self, request, thumb_id, response=None, info=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 2439de9a5..0a12f3e2c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -61,7 +61,7 @@ class MediaPipeline: 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - formatted_key = "{}_{}".format(class_name.upper(), key) + formatted_key = f"{class_name.upper()}_{key}" if ( not base_class_name or class_name == base_class_name @@ -151,9 +151,8 @@ class MediaPipeline: if 'item' not in sig.parameters: old_params = str(sig)[1:-1] new_params = old_params + ", *, item=None" - warn('%s(self, %s) is deprecated, ' - 'please use %s(self, %s)' - % (func.__name__, old_params, func.__name__, new_params), + warn(f'{func.__name__}(self, {old_params}) is deprecated, ' + f'please use {func.__name__}(self, {new_params})', ScrapyDeprecationWarning, stacklevel=2) self._expects_item[func.__name__] = False diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e13d389ee..a9aa6c649 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -141,17 +141,16 @@ class DownloaderAwarePriorityQueue: def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: - raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' - % (self.__class__,)) + raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP') if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError("DownloaderAwarePriorityQueue accepts " - "``slot_startprios`` as a dict; %r instance " + "``slot_startprios`` as a dict; " + f"{slot_startprios.__class__!r} instance " "is passed. Most likely, it means the state is" "created by an incompatible priority queue. " "Only a crawl started with the same priority " - "queue class can be resumed." % - slot_startprios.__class__) + "queue class can be resumed.") self._downloader_interface = DownloaderInterface(crawler) self.downstream_queue_cls = downstream_queue_cls diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index d207088e6..6ed9f8b8f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -45,7 +45,7 @@ class ResponseTypes: elif mimetype in self.classes: return self.classes[mimetype] else: - basetype = "%s/*" % mimetype.split('/')[0] + basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index f12c61081..a25871433 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -66,8 +66,8 @@ class Selector(_ParselSelector, object_ref): def __init__(self, response=None, text=None, type=None, root=None, **kwargs): if response is not None and text is not None: - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + raise ValueError(f'{self.__class__.__name__}.__init__() received ' + 'both response and text') st = _st(response, type or self._default_type) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 951fc65e2..1fe1e6fd1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -52,7 +52,7 @@ class SettingsAttribute: self.priority = priority def __str__(self): - return "".format(self=self) + return f"" __repr__ = __str__ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a0251394b..4ef330dd2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -287,7 +287,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/shell.py b/scrapy/shell.py index 10de119ce..c370ccaff 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -140,7 +140,7 @@ class Shell: b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): - b.append(" %-10s %s" % (k, v)) + b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " @@ -150,7 +150,7 @@ class Shell: b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % line for line in b) + return "\n".join(f"[s] {line}" for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) or is_item(value) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index db4193430..04fda311f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader: dupes = [] for name, locations in self._found.items(): dupes.extend([ - " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + f" {cls} named {name!r} (in {mod})" for mod, cls in locations if len(locations) > 1 ]) @@ -36,7 +36,7 @@ class SpiderLoader: dupes_string = "\n\n".join(dupes) warnings.warn( "There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string), + f"{dupes_string}\n\n This can cause unexpected behavior.", category=UserWarning, ) @@ -53,10 +53,9 @@ class SpiderLoader: except ImportError: if self.warn_only: warnings.warn( - "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc() - ), + f"\n{traceback.format_exc()}Could not load spiders " + f"from module '{name}'. " + "See above traceback for details.", category=RuntimeWarning, ) else: @@ -75,7 +74,7 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError("Spider not found: {}".format(spider_name)) + raise KeyError(f"Spider not found: {spider_name}") def find_by_request(self, request): """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index fa7f5bef9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -43,7 +43,7 @@ class DepthMiddleware: return False else: if self.verbose_stats: - self.stats.inc_value('request_depth_count/%s' % depth, + self.stats.inc_value(f'request_depth_count/{depth}', spider=spider) self.stats.max_value('request_depth_max', depth, spider=spider) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index db9d0f2ae..ae5c258df 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -48,7 +48,7 @@ class HttpErrorMiddleware: if isinstance(exception, HttpError): spider.crawler.stats.inc_value('httperror/response_ignored_count') spider.crawler.stats.inc_value( - 'httperror/response_ignored_status_count/%s' % response.status + f'httperror/response_ignored_status_count/{response.status}' ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a006f3177..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -61,15 +61,15 @@ class OffsiteMiddleware: continue elif url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " - "Ignoring URL entry %s in allowed_domains." % domain) + f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) elif port_pattern.search(domain): message = ("allowed_domains accepts only domains without ports. " - "Ignoring entry %s in allowed_domains." % domain) + f"Ignoring entry {domain} in allowed_domains.") warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 434067b00..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -278,7 +278,7 @@ def _load_policy_class(policy, warning_only=False): try: return _policy_classes[policy.lower()] except KeyError: - msg = "Could not load referrer policy %r" % policy + msg = f"Could not load referrer policy {policy!r}" if not warning_only: raise RuntimeError(msg) else: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 12b4fba09..3da0a11db 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -25,7 +25,7 @@ class Spider(object_ref): if name is not None: self.name = name elif not getattr(self, 'name', None): - raise ValueError("%s must have a name" % type(self).__name__) + raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, 'start_urls'): self.start_urls = [] @@ -66,9 +66,8 @@ class Spider(object_ref): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead (see %s.%s)." % ( - cls.__module__, cls.__name__ - ), + "override Spider.start_requests method instead " + f"(see {cls.__module__}.{cls.__name__}).", ) for url in self.start_urls: yield self.make_requests_from_url(url) @@ -90,7 +89,7 @@ class Spider(object_ref): return self.parse(response, **kwargs) def parse(self, response, **kwargs): - raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) + raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined') @classmethod def update_settings(cls, settings): @@ -107,7 +106,7 @@ class Spider(object_ref): return closed(reason) def __str__(self): - return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) + return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" __repr__ = __str__ diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index cf658aec4..6ed17e4dd 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -71,11 +71,11 @@ class XMLFeedSpider(Spider): elif self.iterator == 'xml': selector = Selector(response, type='xml') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') elif self.iterator == 'html': selector = Selector(response, type='html') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f595a1acb..86238c4cd 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -21,8 +21,8 @@ class Root(Resource): for nl in nlist: args['n'] = nl argstr = urlencode(args, doseq=True) - request.write("follow {1}
" - .format(argstr, nl).encode('utf8')) + request.write(f"follow {nl}
" + .encode('utf8')) request.write(b"") return b'' @@ -39,6 +39,6 @@ if __name__ == '__main__': def _print_listening(): httpHost = httpPort.getHost() - print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port)) + print(f"Bench server at http://{httpHost.host}:{httpHost.port}") reactor.callWhenRunning(_print_listening) reactor.run() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 90a52b25b..05cd5f25c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,8 +17,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {!r} convert to the same object, ' - 'please update your settings'.format(complist)) + raise ValueError('Some paths in {complist!r} convert to the same object, ' + 'please update your settings') def _map_keys(compdict): if isinstance(compdict, BaseSettings): @@ -26,9 +26,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: - raise ValueError('Some paths in {!r} convert to the same ' + raise ValueError(f'Some paths in {list(compdict.keys())!r} ' + 'convert to the same ' 'object, please update your settings' - ''.format(list(compdict.keys()))) + ) else: compbs.set(convert(k), v, priority=prio) return compbs @@ -40,8 +41,9 @@ def build_component_list(compdict, custom=None, convert=update_classpath): """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError('Invalid value {} for component {}, please provide ' - 'a real number or None instead'.format(value, name)) + raise ValueError(f'Invalid value {value} for component {name}, ' + 'please provide a real number or None instead' + ) # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): @@ -141,12 +143,10 @@ def feed_process_params_from_cli(settings, output, output_format=None, def check_valid_format(output_format): if output_format not in valid_output_formats: raise UsageError( - "Unrecognized output format '%s'. Set a supported one (%s) " + f"Unrecognized output format '{output_format}'. " + f"Set a supported one ({tuple(valid_output_formats)}) " "after a colon at the end of the output URI (i.e. -o/-O " - ":) or as a file extension." % ( - output_format, - tuple(valid_output_formats), - ) + ":) or as a file extension." ) overwrite = False diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c0efcec4..6660b9dc0 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = 'There was an error parsing the curl command: {}'.format(message) + error_msg = f'There was an error parsing the curl command: {message}' raise ValueError(error_msg) @@ -52,7 +52,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = 'Unrecognized options: {}'.format(', '.join(argv)) + msg = f'Unrecognized options: {", ".join(argv)}' if ignore_unknown_options: warnings.warn(msg) else: diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2e2c7adc1..fef3882cb 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -14,9 +14,9 @@ def deprecated(use_instead=None): def deco(func): @wraps(func) def wrapped(*args, **kwargs): - message = "Call to deprecated function %s." % func.__name__ + message = f"Call to deprecated function {func.__name__}." if use_instead: - message += " Use %s instead." % use_instead + message += f" Use {use_instead} instead." warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) return func(*args, **kwargs) return wrapped diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3c8e3c8b5..fb7e69889 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -8,9 +8,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning def attribute(obj, oldattr, newattr, version='0.12'): cname = obj.__class__.__name__ warnings.warn( - "%s.%s attribute is deprecated and will be no longer supported " - "in Scrapy %s, use %s.%s attribute instead" - % (cname, oldattr, version, cname, newattr), + f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " + f"in Scrapy {version}, use {cname}.{newattr} attribute instead", ScrapyDeprecationWarning, stacklevel=3) @@ -116,7 +115,7 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn("Error detecting parent module: %r" % e) + warnings.warn(f"Error detecting parent module: {e!r}") return deprecated_cls @@ -124,7 +123,7 @@ def create_deprecated_class( def _clspath(cls, forced=None): if forced is not None: return forced - return '{}.{}'.format(cls.__module__, cls.__name__) + return f'{cls.__module__}.{cls.__name__}' DEPRECATION_RULES = [ @@ -137,7 +136,7 @@ def update_classpath(path): for prefix, replacement in DEPRECATION_RULES: if path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) - warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), + warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning) return new_path return path diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 267c7ecd1..0c1cee1a0 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -29,7 +29,7 @@ def get_engine_status(engine): try: checks += [(test, eval(test))] except Exception as e: - checks += [(test, "%s (exception)" % type(e).__name__)] + checks += [(test, f"{type(e).__name__} (exception)")] return checks @@ -38,7 +38,7 @@ def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: - s += "%-47s : %s\n" % (test, result) + s += f"{test:<47} : {result}\n" s += "\n" return s diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 19d56d6ec..6cace4f07 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -33,5 +33,5 @@ def ftp_store_file( dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) command = 'STOR' if overwrite else 'APPE' - ftp.storbinary('%s %s' % (command, filename), file) + ftp.storbinary(f'{command} {filename}', file) file.close() diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 5e15bf0c8..789da1392 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -22,8 +22,8 @@ def xmliter(obj, nodename): """ nodename_patt = re.escape(nodename) - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) + HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S) + HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -31,7 +31,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) + r = re.compile(fr'<{nodename_patt}[\s>].*?', re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -40,9 +40,9 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = '{%s}%s' % (namespace, nodename) if namespace else nodename + tag = f'{{{namespace}}}{nodename}'if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) - selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) + selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding='unicode') node.clear() @@ -131,8 +131,7 @@ def _body_or_str(obj, unicode=True): if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) raise TypeError( - "Object %r must be %s, not %s" - % (obj, expected_types_str, type(obj).__name__) + f"Object {obj!r} must be {expected_types_str}, not {type(obj).__name__}" ) if isinstance(obj, Response): if not unicode: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index e41315738..62df7a6ab 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ - "%s %s" % (name, version) + f"{name} {version}" for name, version in scrapy_components_versions() if name != "Scrapy" ] @@ -187,7 +187,7 @@ class LogCounterHandler(logging.Handler): self.crawler = crawler def emit(self, record): - sname = 'log_count/{}'.format(record.levelname) + sname = f'log_count/{record.levelname}' self.crawler.stats.inc_value(sname) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index bd400bd30..9107f30ef 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -46,7 +46,7 @@ def load_object(path): try: dot = path.rindex('.') except ValueError: - raise ValueError("Error loading object '%s': not a full path" % path) + raise ValueError(f"Error loading object '{path}': not a full path") module, name = path[:dot], path[dot + 1:] mod = import_module(module) @@ -54,7 +54,7 @@ def load_object(path): try: obj = getattr(mod, name) except AttributeError: - raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name)) + raise NameError(f"Module '{module}' doesn't define any object named '{name}'") return obj @@ -163,7 +163,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): instance = objcls(*args, **kwargs) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") return instance @@ -234,9 +234,10 @@ def warn_on_generator_with_return_value(spider, callable): """ if is_generator_with_return_value(callable): warnings.warn( - 'The "{}.{}" method is a generator and includes a "return" statement with a ' - 'value different than None. This could lead to unexpected behaviour. Please see ' + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators' - .format(spider.__class__.__name__, callable.__name__), stacklevel=2, + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, ) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index b8d3ebf9d..fd13d85e3 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -20,7 +20,7 @@ def inside_project(): try: import_module(scrapy_module) except ImportError as exc: - warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) + warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}") else: return True return bool(closest_scrapy_cfg()) @@ -90,7 +90,7 @@ def get_project_settings(): warnings.warn( 'Use of environment variables prefixed with SCRAPY_ to override ' 'settings is deprecated. The following environment variables are ' - 'currently defined: {}'.format(setting_envvar_list), + f'currently defined: {setting_envvar_list}', ScrapyDeprecationWarning ) settings.setdict(scrapy_envvars, priority='project') diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1f2333264..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -91,7 +91,7 @@ def to_unicode(text, encoding=None, errors='strict'): return text if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.decode(encoding, errors) @@ -104,7 +104,7 @@ def to_bytes(text, encoding=None, errors='strict'): return text if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) @@ -174,7 +174,7 @@ def binary_is_text(data): does not contain unprintable control characters. """ if not isinstance(data, bytes): - raise TypeError("data must be bytes, got '%s'" % type(data).__name__) + raise TypeError(f"data must be bytes, got '{type(data).__name__}'") return all(c not in _BINARYCHARS for c in data) @@ -217,7 +217,7 @@ def get_func_args(func, stripself=False): else: return get_func_args(func.__call__, True) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') if stripself: func_args.pop(0) return func_args @@ -250,7 +250,7 @@ def get_spec(func): elif hasattr(func, '__call__'): spec = _getargspec_py23(func.__call__) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') defaults = spec.defaults or [] @@ -322,7 +322,7 @@ def global_object_name(obj): >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return "%s.%s" % (obj.__module__, obj.__name__) + return f"{obj.__module__}.{obj.__name__}" if hasattr(sys, "pypy_version_info"): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 879d27907..831d29462 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,7 +10,7 @@ def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor if len(portrange) > 2: - raise ValueError("invalid portrange: %s" % portrange) + raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) if not hasattr(portrange, '__iter__'): @@ -78,9 +78,9 @@ def verify_installed_reactor(reactor_path): from twisted.internet import reactor reactor_class = load_object(reactor_path) if not isinstance(reactor, reactor_class): - msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( - reactor.__module__, reactor.__class__.__name__, reactor_path - ) + msg = ("The installed reactor " + f"({reactor.__module__}.{reactor.__class__.__name__}) does not " + f"match the requested one ({reactor_path})") raise Exception(msg) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 503d7b133..d38b1bc4d 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -84,7 +84,7 @@ def _find_method(obj, func): # https://docs.python.org/3/reference/datamodel.html if obj_func.__func__ is func.__func__: return name - raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) + raise ValueError(f"Function {func} is not an instance method in: {obj}") def _get_method(obj, name): @@ -92,4 +92,4 @@ def _get_method(obj, name): try: return getattr(obj, name) except AttributeError: - raise ValueError("Method %r not found in: %s" % (name, obj)) + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c29b619ce..99b089b6f 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -39,7 +39,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_unicode(message)) + return f'{status} {to_unicode(message)}' def response_httprepr(response): @@ -69,15 +69,15 @@ def open_in_browser(response, _openfunc=webbrowser.open): body = response.body if isinstance(response, HtmlResponse): if b'' body = body.replace(b'', to_bytes(repl)) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' else: - raise TypeError("Unsupported response type: %s" % - response.__class__.__name__) + raise TypeError("Unsupported response type: " + f"{response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) - return _openfunc("file://%s" % fname) + return _openfunc(f"file://{fname}") diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index cc3263602..a73cf03c5 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -17,7 +17,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): if isinstance(o, set): return list(o) elif isinstance(o, datetime.datetime): - return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) + return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") elif isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) elif isinstance(o, datetime.time): @@ -29,9 +29,9 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif is_item(o): return ItemAdapter(o).asdict() elif isinstance(o, Request): - return "<%s %s %s>" % (type(o).__name__, o.method, o.url) + return f"<{type(o).__name__} {o.method} {o.url}>" elif isinstance(o, Response): - return "<%s %s %s>" % (type(o).__name__, o.status, o.url) + return f"<{type(o).__name__} {o.status} {o.url}>" else: return super().default(o) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index c3c5e329b..ea4dde882 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -50,7 +50,7 @@ def get_temp_key_info(ssl_object): key_info.append(ffi_buf_to_string(cname)) else: key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) - key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) + key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits') return ', '.join(key_info) @@ -58,4 +58,4 @@ def get_openssl_version(): system_openssl = OpenSSL.SSL.SSLeay_version( OpenSSL.SSL.SSLEAY_VERSION ).decode('ascii', errors='replace') - return '{} ({})'.format(OpenSSL.version.__version__, system_openssl) + return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7442a2f33..f54942ffb 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -79,7 +79,7 @@ def get_ftp_content_and_delete( def buffer_data(data): ftp_data.append(data) - ftp.retrbinary('RETR %s' % path, buffer_data) + ftp.retrbinary(f'RETR {path}', buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index a63c9a942..a54c7db95 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -23,10 +23,10 @@ class ProcessTest: def _process_finished(self, pp, cmd, check_code): if pp.exitcode and check_code: - msg = "process %s exit with code %d" % (cmd, pp.exitcode) - msg += "\n>>> stdout <<<\n%s" % pp.out + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out}" msg += "\n" - msg += "\n>>> stderr <<<\n%s" % pp.err + msg += f"\n>>> stderr <<<\n{pp.err}" raise RuntimeError(msg) return pp.exitcode, pp.out, pp.err diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 397e54703..fce77be32 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -9,7 +9,7 @@ class SiteTest: from twisted.internet import reactor super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = "http://localhost:%d/" % self.site.getHost().port + self.baseurl = f"http://localhost:{self.site.getHost().port}/" def tearDown(self): super().tearDown() @@ -40,5 +40,5 @@ def test_site(): if __name__ == '__main__': from twisted.internet import reactor port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - print("http://localhost:%d/" % port.getHost().port) + print(f"http://localhost:{port.getHost().port}/") reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index baed5c536..3e40acd69 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -41,9 +41,7 @@ def format_live_refs(ignore=NoneType): if issubclass(cls, ignore): continue oldest = min(wdict.values()) - s += "%-30s %6d oldest: %ds ago\n" % ( - cls.__name__, len(wdict), now - oldest - ) + s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n" return s diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b23ddb459..a6a2a9e8b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains): if not host: return False domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith('.%s' % d)) for d in domains) + return any((host == d) or (host.endswith(f'.{d}')) for d in domains) def url_is_from_spider(url, spider): @@ -153,7 +153,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443), ('ftp', 21)): - netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + netloc = netloc.replace(f':{parsed_url.port}', '') return urlunparse(( parsed_url.scheme, netloc, diff --git a/sep/sep-002.rst b/sep/sep-002.rst index c467cb402..2e8a28340 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError("Expected iterable, got %s" % type(value).__name__) + raise TypeError(f"Expected iterable, got {type(value).__name__}") def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index 05b0eb99c..b9f5e556f 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print "%d items scraped" % len(scraped_items) + print(f"{len(scraped_items)} items scraped") # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..4e3340521 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given %r in %r' \ - % (rule.matcher, rule)) + raise ValueError('Not valid matcher given ' + f'{rule.matcher!r} in {rule!r}') # prepare callback if callable(rule.callback): @@ -553,8 +553,7 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError('Invalid callback %r can not be resolved' \ - % callback) + raise AttributeError(f'Invalid callback {callback!r} can not be resolved') else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index fe707923a..d0169b81e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) + robotsurl = f"{url.scheme}://{netloc}/robots.txt" meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 3f9738798..f545de39f 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -38,7 +38,7 @@ class LocalhostSpider(Spider): if __name__ == "__main__": with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port - url = "http://not.a.real.domain:{port}/echo".format(port=port) + url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] reactor.installResolver(createResolver(servers=servers)) diff --git a/tests/mockserver.py b/tests/mockserver.py index 6f0c274b9..ab9aec6a6 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -73,7 +73,7 @@ class Follow(LeafResource): for nl in nlist: args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) - s += "follow %d
" % (argstr, nl) + s += f"follow {nl}
" s += """""" request.write(to_bytes(s)) request.finish() @@ -91,7 +91,7 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) + request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() @@ -310,8 +310,8 @@ if __name__ == "__main__": def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) - httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + httpAddress = f'http://{httpHost.host}:{httpHost.port}' + httpsAddress = f'https://{httpsHost.host}:{httpsHost.port}' print(httpAddress) print(httpsAddress) @@ -323,7 +323,7 @@ if __name__ == "__main__": def print_listening(): host = listener.getHost() - print("%s:%s" % (host.host, host.port)) + print(f"{host.host}:{host.port}") reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py index 162a53760..e49340284 100644 --- a/tests/py36/_test_crawl.py +++ b/tests/py36/_test_crawl.py @@ -33,7 +33,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), + return Request(self.mockserver.url(f"/status?n=200&request={index}"), meta={'index': index}, dont_filter=True, callback=cb) diff --git a/tests/spiders.py b/tests/spiders.py index 63bd726fb..3e525e62f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") self.start_urls = [url] def parse(self, response): @@ -67,7 +67,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -192,7 +192,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -239,7 +239,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = self.mockserver.url("/echo?headers=1&body=test%d" % i) + url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 6504b4d2c..005e45214 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -4,7 +4,7 @@ class TestExtension: def __init__(self, settings): - settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) + settings.set('TEST1', f"{settings['TEST1']} + started") @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_command_check.py b/tests/test_command_check.py index f27f526a3..34f5e59dd 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(""" + file.write(f""" import scrapy class CheckSpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://example.com - {1} + {contracts} \"\"\" - {2} - """.format(self.spider_name, contracts, parse_def)) + {parse_def} + """) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index e115f420f..ed3848d88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -21,14 +21,14 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: - f.write(""" + f.write(f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule class MySpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' def parse(self, response): if getattr(self, 'test_arg', None): @@ -58,7 +58,7 @@ class MySpider(scrapy.Spider): self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): - name = 'goodcrawl{0}' + name = 'goodcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -74,7 +74,7 @@ class MyGoodCrawlSpider(CrawlSpider): class MyBadCrawlSpider(CrawlSpider): '''Spider which doesn't define a parse_item callback while using it in a rule.''' - name = 'badcrawl{0}' + name = 'badcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -82,7 +82,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) +""") fname = abspath(join(self.proj_mod_path, 'pipelines.py')) with open(fname, 'w') as f: @@ -99,9 +99,9 @@ class MyPipeline: fname = abspath(join(self.proj_mod_path, 'settings.py')) with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) + f.write(f""" +ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} +""") @defer.inlineCallbacks def test_spider_arguments(self): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 66c293c00..16c9559b5 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -65,8 +65,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}')" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}')" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Redirecting (302)' in errout assert b'Crawled (200)' in errout @@ -75,23 +75,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}', redirect=False)" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Crawled (302)' in errout @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') - code = "fetch('{0}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks def test_scrapy_import(self): url = self.url('/text') - code = "fetch(scrapy.Request('{0}'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch(scrapy.Request('{url}'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 99c01c2b7..00d998388 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -16,7 +16,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), - "Scrapy %s" % scrapy.__version__, + f"Scrapy {scrapy.__version__}", ) @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index ee8a92604..5faaca738 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -42,7 +42,7 @@ class CommandSettings(unittest.TestCase): def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py']) + opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) @@ -163,10 +163,10 @@ class StartprojectTemplatesTest(ProjectTest): pass assert exists(join(self.tmpl_proj, 'root_template')) - args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] + args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn("New Scrapy project '%s', using template directory" - % self.project_name, out) + self.assertIn(f"New Scrapy project '{self.project_name}', " + "using template directory", out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -247,7 +247,7 @@ class StartprojectTemplatesTest(ProjectTest): 'startproject', project_name, '--set', - 'TEMPLATES_DIR={}'.format(read_only_templates_dir), + f'TEMPLATES_DIR={read_only_templates_dir}', ), cwd=destination, env=self.env, @@ -320,7 +320,7 @@ class CommandTest(ProjectTest): super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) - self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name + self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' class GenspiderCommandTest(CommandTest): @@ -334,14 +334,14 @@ class GenspiderCommandTest(CommandTest): assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) def test_template(self, tplname='crawl'): - args = ['--template=%s' % tplname] if tplname else [] + args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Spider %r already exists in module" % spname, out) + self.assertIn(f"Spider {spname!r} already exists in module", out) modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) self.assertEqual(modify_time_after, modify_time_before) @@ -363,11 +363,11 @@ class GenspiderCommandTest(CommandTest): def test_same_name_as_project(self): self.assertEqual(2, self.call('genspider', self.project_name)) - assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) def test_same_filename_as_existing_spider(self, force=False): file_name = 'example' - file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name) + file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') self.assertEqual(0, self.call('genspider', file_name, 'example.com')) assert exists(file_path) @@ -383,14 +383,14 @@ class GenspiderCommandTest(CommandTest): if force: p, out, err = self.proc('genspider', '--force', file_name, 'example.com') - self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) modify_time_after = getmtime(file_path) self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("%s already exists" % (file_path), out) + self.assertIn(f"{file_path} already exists", out) modify_time_after = getmtime(file_path) self.assertEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() @@ -410,7 +410,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): file_name = 'example' file_path = join(self.temp_path, file_name + '.py') p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("Created spider %r using template \'basic\' " % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) assert exists(file_path) modify_time_before = getmtime(file_path) file_contents_before = open(file_path, 'r').read() @@ -418,14 +418,14 @@ class GenspiderStandaloneCommandTest(ProjectTest): if force: # use different template to ensure contents were changed p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') - self.assertIn("Created spider %r using template \'crawl\' " % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) modify_time_after = getmtime(file_path) self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out) + self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) modify_time_after = getmtime(file_path) self.assertEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 2e7e3ccc4..d0f4a68c2 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -393,7 +393,7 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + contract_doc = f'@url {mockserver.url("/status?n=200")}' TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..e703f45de 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): total_time = times[-1] - times[0] average = total_time / (len(times) - 1) self.assertTrue(average > delay * tolerance, - "download delay too small: %s" % average) + f"download delay too small: {average}") # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -204,7 +204,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) + yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks @@ -465,7 +465,7 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) for req_id in range(3): - self.assertIn("Got response 200, req_id %d" % req_id, str(log)) + self.assertIn(f"Got response 200, req_id {req_id}", str(log)) @defer.inlineCallbacks def test_response_ssl_certificate_none(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7059f0892..0d3c42797 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -121,7 +121,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_non_existent(self): - request = Request('file://%s' % self.mktemp()) + request = Request(f'file://{self.mktemp()}') d = self.download_request(request, Spider('foo')) return self.assertFailure(d, IOError) @@ -249,7 +249,7 @@ class HttpTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -300,7 +300,7 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): self.assertEqual( - response.body, to_bytes('%s:%d' % (self.host, self.portno))) + response.body, to_bytes(f'{self.host}:{self.portno}')) self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) @@ -583,7 +583,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -678,7 +678,7 @@ class HttpProxyTestCase(unittest.TestCase): yield self.download_handler.close() def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def test_download_with_proxy(self): def _test(response): @@ -696,7 +696,7 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'https://example.com') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f'{self.getURL("")}?noconnect' request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): @@ -977,7 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): return deferred def test_ftp_download_success(self): - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -989,7 +989,7 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_download_path_with_spaces(self): request = Request( - url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, + url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", meta=self.req_meta ) d = self.download_handler.download_request(request, None) @@ -1001,7 +1001,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): - request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -1015,7 +1015,7 @@ class BaseFTPTestCase(unittest.TestCase): os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) @@ -1037,7 +1037,7 @@ class FTPTestCase(BaseFTPTestCase): meta = dict(self.req_meta) meta.update({"ftp_password": 'invalid'}) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index a9190c62b..79f24c8a1 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -84,7 +84,7 @@ class DefaultsTest(ManagerTestCase): }) ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), - "Not redirected: {0!r}".format(ret)) + f"Not redirected: {ret!r}") self.assertEqual(to_bytes(ret.url), resp.headers['Location'], "Not redirected to location header") diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index dbae4d3ae..b2b5ce77d 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,7 +28,7 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = f'Failed {fmt}, response type {type(new).__name__}' assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 299fb0eb8..0c6dcf2aa 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -324,7 +324,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (shouldcache, status, headers) in enumerate(responses): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -343,7 +343,7 @@ class RFC2616PolicyTest(DefaultStorageTest): with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw: for idx, (_, status, headers) in enumerate(responses): shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304 - req0 = Request('http://example2-%d.com' % idx) + req0 = Request(f'http://example2-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -386,7 +386,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) @@ -423,7 +423,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0a = Response(req0.url, status=status, headers=headers) # cache expired response res1 = self._process_requestresponse(mw, req0, res0a) @@ -490,7 +490,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 131332131..816ac1440 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,7 +22,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_redirect_3xx_permanent(self): def _test(method, status=301): - url = 'http://www.example.com/{}'.format(status) + url = f'http://www.example.com/{status}' url2 = 'http://www.example.com/redirected' req = Request(url, method=method) rsp = Response(url, headers={'Location': url2}, status=status) @@ -79,7 +79,7 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" # response without Location header but with status code is 3XX should be ignored del rsp.headers['Location'] @@ -207,8 +207,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = """""" - return html.format(interval, url).encode('utf-8') + html = f"""""" + return html.encode('utf-8') def test_priority_adjust(self): req = Request('http://a.com') @@ -243,7 +243,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): self.mw.max_redirect_times = 1 diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 29357ba94..364ce0c89 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -94,7 +94,7 @@ class RetryTest(unittest.TestCase): ] for exc in exceptions: - req = Request('http://www.scrapytest.org/%s' % exc.__name__) + req = Request(f'http://www.scrapytest.org/{exc.__name__}') self._test_retry_exception(req, exc('foo')) stats = self.crawler.stats diff --git a/tests/test_engine.py b/tests/test_engine.py index 1b848ac72..3629aa1aa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -127,8 +127,8 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." - % port.getHost().port) + print(f"Test server running at http://localhost:{port.getHost().port}/ " + "- hit Ctrl-C to finish.") return port @@ -185,7 +185,7 @@ class CrawlerRun: self.deferred.callback(None) def geturl(self, path): - return "http://localhost:%s%s" % (self.portno, path) + return f"http://localhost:{self.portno}{path}" def getpath(self, url): u = urlparse(url) @@ -265,7 +265,7 @@ class EngineTest(unittest.TestCase): "/item1.html", "/item2.html", "/item999.html"] urls_visited = {rp[0].url for rp in self.run.respplug} urls_expected = {self.run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" def _assert_scheduled_requests(self, urls_to_visit=None): self.assertEqual(urls_to_visit, len(self.run.reqplug)) @@ -413,16 +413,19 @@ class StopDownloadEngineTest(EngineTest): yield self.run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) self._assert_visited_urls() self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 850485b5e..94568581a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -184,8 +184,7 @@ class FTPFeedStorageTest(unittest.TestCase): def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted, - {}) + st = FTPFeedStorage(f'ftp://foo:{pw_quoted}@example.com/some_path', {}) self.assertEqual(st.password, string.punctuation) @@ -1230,7 +1229,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Stored %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Stored {fmt} feed (2 items)', str(log)) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): @@ -1251,7 +1250,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Error storing %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) class BatchDeliveriesTest(FeedExportTestBase): @@ -1582,10 +1581,8 @@ class BatchDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) - prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( - bucket_name=s3_test_bucket_name, prefix=prefix - ) + prefix = f'tmp/{filename}' + s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) settings = Settings({ 'FEEDS': { diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 624dd9ab8..41afa2896 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -657,7 +657,7 @@ class SelectJmesTestCase(unittest.TestCase): self.assertEqual( test, expected, - msg='test "{}" got {} expected {}'.format(tl, test, expected) + msg=f'test "{tl}" got {test} expected {expected}' ) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 41ff3651d..dc5be398f 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -20,7 +20,7 @@ class CustomItem(Item): name = Field() def __str__(self): - return "name: %s" % self['name'] + return f"name: {self['name']}" class LogFormatterTestCase(unittest.TestCase): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b2b75ef20..e3e46db07 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [f'tests.test_middleware.{x}' for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): super()._add_middleware(mw) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9af5affec..55fcfa7ba 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -123,10 +123,10 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) - self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3) # check that logs do show the failure on the file downloads - file_dl_failure = 'File (code: %d): Error downloading file from' % code + file_dl_failure = f'File (code: {code}): Error downloading file from' self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b19b4ff2a..1dd7031fe 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -167,7 +167,7 @@ class FilesPipelineTestCase(unittest.TestCase): """ class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): - return 'full/%s' % item.get('path') + return f'full/{item.get("path")}' file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path item = dict(path='path-to-store-file') @@ -495,7 +495,7 @@ class TestFTPFileStore(unittest.TestCase): self.assertIn('last_modified', stat) self.assertIn('checksum', stat) self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6') - path = '%s/%s' % (store.basedir, path) + path = f'{store.basedir}/{path}' content = get_ftp_content_and_delete( path, store.host, store.port, store.username, store.password, store.USE_ACTIVE_MODE) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 082e9ee21..ad138a2dc 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -128,11 +128,11 @@ class DeprecatedImagesPipeline(ImagesPipeline): def image_key(self, url): image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'empty/%s.jpg' % (image_guid) + return f'empty/{image_guid}.jpg' def thumb_key(self, url, thumb_id): thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbsup/{thumb_id}/{thumb_guid}.jpg' class ImagesPipelineTestCaseFieldsMixin: diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..d3f58634e 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -37,14 +37,14 @@ sys.exit(mitmdump()) '-c', script, '--listen-host', '127.0.0.1', '--listen-port', '0', - '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--proxyauth', f'{self.auth_user}:{self.auth_pass}', '--certs', cert_path, '--ssl-insecure', ], stdout=PIPE, env=get_testenv()) line = self.proc.stdout.readline().decode('utf-8') host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) - address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}' return address def stop(self): @@ -118,7 +118,7 @@ class ProxyConnectTestCase(TestCase): def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count('Crawled (%d)' % code), 1) + self.assertEqual(str(log).count(f'Crawled ({code})'), 1) def _assert_got_tunnel_error(self, log): print(log) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index b60b7c579..907117468 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_response_error(self): for status in ("404", "500"): - url = self.mockserver.url("/status?n={}".format(status)) + url = self.mockserver.url(f"/status?n={status}") crawler = CrawlerRunner().create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] @@ -135,7 +135,7 @@ class CrawlTestCase(TestCase): self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) log.check_present( - ("scrapy.core.engine", "DEBUG", "Crawled (200) (referer: None)".format(OVERRIDEN_URL)), + ("scrapy.core.engine", "DEBUG", f"Crawled (200) (referer: None)"), ) @defer.inlineCallbacks diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index a175f88ca..c07d3a99c 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -17,7 +17,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_filename(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_disposition(self): mappings = [ @@ -32,7 +32,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_type(self): mappings = [ @@ -47,7 +47,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_body(self): mappings = [ @@ -58,7 +58,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_body(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_headers(self): mappings = [ @@ -70,7 +70,7 @@ class ResponseTypesTest(unittest.TestCase): for source, cls in mappings: source = Headers(source) retcls = responsetypes.from_headers(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_args(self): # TODO: add more tests that check precedence between the different arguments @@ -86,7 +86,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_args(**source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_custom_mime_types_loaded(self): # check that mime.types files shipped with scrapy are loaded diff --git a/tests/test_selector.py b/tests/test_selector.py index 62036ad8c..cff8d0393 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -88,7 +88,7 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__ + assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__" def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): diff --git a/tests/test_signals.py b/tests/test_signals.py index d6ae526be..a43f00b27 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -13,7 +13,7 @@ class ItemSpider(Spider): def start_requests(self): for index in range(10): - yield Request(self.mockserver.url('/status?n=200&id=%d' % index), + yield Request(self.mockserver.url(f'/status?n=200&id={index}'), meta={'index': index}) def parse(self, response): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 79eda35b3..2f454addc 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -163,11 +163,11 @@ class GeneratorOutputChainSpider(Spider): class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -175,12 +175,12 @@ class _GeneratorDoNothingMiddleware: class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r raise LookupError() def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -192,11 +192,11 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -229,12 +229,12 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -243,13 +243,13 @@ class NotGeneratorFailMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) raise ReferenceError() return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] @@ -262,12 +262,12 @@ class NotGeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 6b05c8771..f5d684d3f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -16,7 +16,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): try: Request(**result) except TypeError as e: - self.fail("Request kwargs are not correct {}".format(e)) + self.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index aa18ef1f3..e4bccf30e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -299,7 +299,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache = LocalWeakReferencedCache() refs = [] for x in range(max): - refs.append(Request('https://example.org/{}'.format(x))) + refs.append(Request(f'https://example.org/{x}')) cache[refs[-1]] = x self.assertEqual(len(cache), max) for i, r in enumerate(refs): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 8c84331b9..e60242a3b 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -40,15 +40,15 @@ class MustbeDeferredTest(unittest.TestCase): def cb1(value, arg1, arg2): - return "(cb1 %s %s %s)" % (value, arg1, arg2) + return f"(cb1 {value} {arg1} {arg2})" def cb2(value, arg1, arg2): - return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + return defer.succeed(f"(cb2 {value} {arg1} {arg2})") def cb3(value, arg1, arg2): - return "(cb3 %s %s %s)" % (value, arg1, arg2) + return f"(cb3 {value} {arg1} {arg2})" def cb_fail(value, arg1, arg2): @@ -56,7 +56,7 @@ def cb_fail(value, arg1, arg2): def eb1(failure, arg1, arg2): - return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) + return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" class DeferUtilsTest(unittest.TestCase): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 50190d4d1..79f5a2bbe 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -409,7 +409,7 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue(type(a) is type(b), - 'Got {}, expected {} for {!r}'.format(type(a), type(b), obj)) + f'Got {type(a)}, expected {type(b)} for { obj!r}') self.assertEqual(a, b) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 2f885a0e8..144c7bd76 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -213,7 +213,7 @@ def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1]) + f'Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`' return do_expected @@ -254,7 +254,7 @@ for k, args in enumerate( start=1, ): t_method = create_guess_scheme_t(args) - t_method.__name__ = 'test_uri_%03d' % k + t_method.__name__ = f'test_uri_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation @@ -269,7 +269,7 @@ for k, args in enumerate( start=1, ): t_method = create_skipped_scheme_t(args) - t_method.__name__ = 'test_uri_skipped_%03d' % k + t_method.__name__ = f'test_uri_skipped_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index ee64d455c..a60181a3a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -253,7 +253,7 @@ class WebClientTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def testPayload(self): s = "0123456789" * 10 @@ -265,7 +265,7 @@ class WebClientTestCase(unittest.TestCase): # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( self.assertEqual, to_bytes("www.example.com"))]) @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) return d def test_timeoutTriggering(self): @@ -376,7 +376,7 @@ class WebClientSSLTestCase(unittest.TestCase): interface="127.0.0.1") def getURL(self, path): - return "https://127.0.0.1:%d/%s" % (self.portno, path) + return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): self.tmpname = self.mktemp() From 560c335c0782ec9a834a83abfa9dfbaa8fd67fed Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:00:51 +0300 Subject: [PATCH 049/127] Add errors parameter in documentation. --- docs/topics/exporters.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..3f8906326 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -292,7 +292,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs) Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -311,6 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str + :param errors: The optional string that specifies how encoding and decoding errors are to be handled. + For more information see `documentation `_. + :type errors: str + The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :func:`csv.writer` function, so you can use any :func:`csv.writer` function From cf50561b8696b645940ae33ba9b168ef97580477 Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 26 Aug 2020 11:08:14 +0000 Subject: [PATCH 050/127] Allow passing classes directly in Settings (#3873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 26 +++++++++++++++++++++ scrapy/utils/deprecate.py | 2 +- scrapy/utils/misc.py | 14 +++++++++-- tests/test_crawler.py | 2 +- tests/test_downloader_handlers.py | 6 ++--- tests/test_feedexport.py | 22 +++++++++--------- tests/test_middleware.py | 2 +- tests/test_settings/__init__.py | 32 ++++++++++++++++++++++++++ tests/test_spidermiddleware_referer.py | 2 +- tests/test_utils_misc/__init__.py | 15 ++++++++++-- 10 files changed, 101 insertions(+), 22 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 618b9989e..2924c0566 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,32 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. + +Import paths and classes +======================== + +.. versionadded:: VERSION + +When a setting references a callable object to be imported by Scrapy, such as a +class or a function, there are two different ways you can specify that object: + +- As a string containing the import path of that object + +- As the object itself + +For example:: + + from mybot.pipelines.validate import ValidateMyItem + ITEM_PIPELINES = { + # passing the classname... + ValidateMyItem: 300, + # ...equals passing the class path + 'mybot.pipelines.validate.ValidateMyItem': 300, + } + +.. note:: Passing non-callable objects is not supported. + + How to access settings ====================== diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3c8e3c8b5..8277a3c8f 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -135,7 +135,7 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: - if path.startswith(prefix): + if isinstance(path, str) and path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), ScrapyDeprecationWarning) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index bd400bd30..d5d1f301c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -39,10 +39,20 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be the import path of a class, function, variable or an - instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' + The object can be the import path of a class, function, variable or an + instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'. + + If ``path`` is not a string, but is a callable object, such as a class or + a function, then return it as is. """ + if not isinstance(path, str): + if callable(path): + return path + else: + raise TypeError("Unexpected argument type, expected string " + "or object, got: %s" % type(path)) + try: dot = path.rindex('.') except ValueError: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7c2e251a9..85035a220 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -142,7 +142,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): def test_spider_manager_verify_interface(self): settings = Settings({ - 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' + 'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface, }) with warnings.catch_warnings(record=True) as w: self.assertRaises(AttributeError, CrawlerRunner, settings) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7059f0892..e50bdb391 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -61,7 +61,7 @@ class OffDH: class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} + handlers = {'scheme': DummyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -69,7 +69,7 @@ class LoadTestCase(unittest.TestCase): self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} + handlers = {'scheme': OffDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -87,7 +87,7 @@ class LoadTestCase(unittest.TestCase): self.assertIn('scheme', dh._notconfigured) def test_lazy_handlers(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + handlers = {'scheme': DummyLazyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 850485b5e..689d25fef 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -845,7 +845,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'}, + 'FEED_STORAGES': {'file': LogOnStoreFileStorage}, 'FEED_STORE_EMPTY': False } @@ -1189,8 +1189,8 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { - 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'}, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'}, + 'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter}, + 'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage}, 'FEEDS': { self._random_temp_filename(): {'format': 'csv'}, }, @@ -1219,7 +1219,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': DummyBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1240,7 +1240,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': FailingBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1667,7 +1667,7 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions' + 'file': StdoutFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1708,7 +1708,7 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions' + 'file': FileFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1756,7 +1756,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions' + 'file': S3FeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1784,7 +1784,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler' + 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1833,7 +1833,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions' + 'file': FTPFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1861,7 +1861,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler' + 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler }, } crawler = get_crawler(settings_dict=settings_dict) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b2b75ef20..8651431b5 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [M1, MOff, M3] def _add_middleware(self, mw): super()._add_middleware(mw) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 6e56a28f5..916fe012a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -385,6 +385,38 @@ class SettingsTest(unittest.TestCase): self.assertIn('key', mydict) self.assertEqual(mydict['key'], 'val') + def test_passing_objects_as_values(self): + from scrapy.core.downloader.handlers.file import FileDownloadHandler + from scrapy.utils.misc import create_instance + from scrapy.utils.test import get_crawler + + class TestPipeline(): + def process_item(self, i, s): + return i + + settings = Settings({ + 'ITEM_PIPELINES': { + TestPipeline: 800, + }, + 'DOWNLOAD_HANDLERS': { + 'ftp': FileDownloadHandler, + }, + }) + + self.assertIn('ITEM_PIPELINES', settings.attributes) + + mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem() + self.assertEqual(priority, 800) + self.assertEqual(mypipeline, TestPipeline) + self.assertIsInstance(mypipeline(), TestPipeline) + self.assertEqual(mypipeline().process_item('item', None), 'item') + + myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp') + self.assertEqual(myhandler, FileDownloadHandler) + myhandler_instance = create_instance(myhandler, None, get_crawler()) + self.assertIsInstance(myhandler_instance, FileDownloadHandler) + self.assertTrue(hasattr(myhandler_instance, 'download_request')) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5141f47af..9456b01d4 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -385,7 +385,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': CustomPythonOrgPolicy} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 9bb996d27..e95a3a316 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -12,11 +12,22 @@ __doctests__ = ['scrapy.utils.misc'] class UtilsMiscTestCase(unittest.TestCase): - def test_load_object(self): + def test_load_object_class(self): + obj = load_object(Field) + self.assertIs(obj, Field) + obj = load_object('scrapy.item.Field') + self.assertIs(obj, Field) + + def test_load_object_function(self): + obj = load_object(load_object) + self.assertIs(obj, load_object) obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + self.assertRaises(TypeError, load_object, dict()) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') From 29725e4b58bfe417388b5ce2c2b4f1c587a465da Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:38:37 +0300 Subject: [PATCH 051/127] Fix tabulation of rst and change documentation link. --- docs/topics/exporters.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 3f8906326..0203def74 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -311,9 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str - :param errors: The optional string that specifies how encoding and decoding errors are to be handled. - For more information see `documentation `_. - :type errors: str + :param errors: The optional string that specifies how encoding and decoding + errors are to be handled. For more information see + :class:`io.TextIOWrapper`. + :type errors: str The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the From c77450990d9f78e68380d724b4a8c15e7b99c995 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:41:24 +0000 Subject: [PATCH 052/127] Update scrapy/commands/version.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/commands/version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index dc8087043..1237610cb 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,7 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = f"%-{width}s : %s" for name, version in versions: - print(patt % (name, version)) + print(f"{name:<{width}} : {version}") else: print(f"Scrapy {scrapy.__version__}") From 92dfa7176dbadcb92e4aae9d3f2c2b02c6e52a4a Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:43:40 +0000 Subject: [PATCH 053/127] Update scrapy/extensions/statsmailer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/extensions/statsmailer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 997e74fc9..bcdbaff24 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -28,7 +28,7 @@ class StatsMailer: def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats()) + body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) body += f"\n\n{spider.name} stats\n\n" - body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats) + body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items()) return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) From ea03e4254fbaff71ce584a461344eb0eb9aa7e09 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:43:52 +0000 Subject: [PATCH 054/127] Update scrapy/http/request/form.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c90d68fa1..2815303a2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -204,7 +204,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata) + xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') From 9aaddcde0af5910e33fbba253777149e04f021f8 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:44:20 +0000 Subject: [PATCH 055/127] Update scrapy/utils/conf.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 05cd5f25c..afd8f5374 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -42,8 +42,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): raise ValueError(f'Invalid value {value} for component {name}, ' - 'please provide a real number or None instead' - ) + 'please provide a real number or None instead') # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): From 2ca8dfb4b08ad416716dc18d03983cc159cc5634 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 13:49:39 +0200 Subject: [PATCH 056/127] revert f-string changes for files under sep/ Issue #4324 --- sep/sep-002.rst | 2 +- sep/sep-004.rst | 4 ++-- sep/sep-014.rst | 29 +++++++++++++++-------------- sep/sep-018.rst | 22 +++++++++++----------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/sep/sep-002.rst b/sep/sep-002.rst index 2e8a28340..c467cb402 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError(f"Expected iterable, got {type(value).__name__}") + raise TypeError("Expected iterable, got %s" % type(value).__name__) def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b9f5e556f..05b0eb99c 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print(f"{len(scraped_items)} items scraped") + print "%d items scraped" % len(scraped_items) # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 4e3340521..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given ' - f'{rule.matcher!r} in {rule!r}') + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) # prepare callback if callable(rule.callback): @@ -553,7 +553,8 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError(f'Invalid callback {callback!r} can not be resolved') + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index d0169b81e..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = f"{url.scheme}://{netloc}/robots.txt" + robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = fr'^(.*\.)?({"|".join(domains)})$' + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed From a8114d3731cfd1ef8fdb808b1563a7288b8a2e90 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 26 Aug 2020 09:00:36 -0300 Subject: [PATCH 057/127] Typing: annotate a few Spider attributes --- scrapy/spiders/__init__.py | 5 ++-- scrapy/spiders/crawl.py | 3 ++- setup.cfg | 51 -------------------------------------- tests/spiders.py | 2 +- 4 files changed, 6 insertions(+), 55 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 12b4fba09..a66d65846 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst """ import logging import warnings +from typing import Optional from scrapy import signals from scrapy.http import Request @@ -18,8 +19,8 @@ class Spider(object_ref): class. """ - name = None - custom_settings = None + name: Optional[str] = None + custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): if name is not None: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index c9fbce08d..bc4551a54 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst import copy import warnings +from typing import Sequence from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse @@ -72,7 +73,7 @@ class Rule: class CrawlSpider(Spider): - rules = () + rules: Sequence[Rule] = () def __init__(self, *a, **kw): super().__init__(*a, **kw) diff --git a/setup.cfg b/setup.cfg index 3a624ec94..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,6 @@ ignore_errors = True [mypy-scrapy.commands] ignore_errors = True -[mypy-scrapy.commands.bench] -ignore_errors = True - [mypy-scrapy.commands.parse] ignore_errors = True @@ -28,9 +25,6 @@ ignore_errors = True [mypy-scrapy.contracts] ignore_errors = True -[mypy-scrapy.core.spidermw] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True @@ -70,15 +64,6 @@ ignore_errors = True [mypy-tests.mocks.dummydbm] ignore_errors = True -[mypy-tests.spiders] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal] -ignore_errors = True - [mypy-tests.test_command_fetch] ignore_errors = True @@ -94,9 +79,6 @@ ignore_errors = True [mypy-tests.test_contracts] ignore_errors = True -[mypy-tests.test_crawler] -ignore_errors = True - [mypy-tests.test_downloader_handlers] ignore_errors = True @@ -127,53 +109,20 @@ ignore_errors = True [mypy-tests.test_pipeline_images] ignore_errors = True -[mypy-tests.test_pipelines] -ignore_errors = True - -[mypy-tests.test_request_attribute_binding] -ignore_errors = True - [mypy-tests.test_request_cb_kwargs] ignore_errors = True -[mypy-tests.test_request_left] -ignore_errors = True - [mypy-tests.test_scheduler] ignore_errors = True -[mypy-tests.test_signals] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.nested.spider4] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider1] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider2] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider3] -ignore_errors = True - [mypy-tests.test_spidermiddleware_httperror] ignore_errors = True -[mypy-tests.test_spidermiddleware_output_chain] -ignore_errors = True - [mypy-tests.test_spidermiddleware_referer] ignore_errors = True -[mypy-tests.test_utils_reqser] -ignore_errors = True - [mypy-tests.test_utils_serialize] ignore_errors = True -[mypy-tests.test_utils_spider] -ignore_errors = True - [mypy-tests.test_utils_url] ignore_errors = True diff --git a/tests/spiders.py b/tests/spiders.py index 63bd726fb..8a0ee44b7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -255,7 +255,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): A CrawlSpider which overrides the 'parse' method """ name = 'crawl_spider_with_parse_method' - custom_settings = { + custom_settings: dict = { 'RETRY_HTTP_CODES': [], # no need to retry } rules = ( From 5ab1a318e8a8245cad694edc297d94ac80a90760 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 15:11:46 +0200 Subject: [PATCH 058/127] test: list appears in ValueError Exception message Issue #4324 --- scrapy/utils/conf.py | 2 +- tests/test_utils_conf.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index afd8f5374..4e7a9967e 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,7 +17,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {complist!r} convert to the same object, ' + raise ValueError(f'Some paths in {complist!r} convert to the same object, ' 'please update your settings') def _map_keys(compdict): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index ccc65c4fd..061bc8c7c 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -50,8 +50,9 @@ class BuildComponentListTest(unittest.TestCase): def test_duplicate_components_in_list(self): duplicate_list = ['a', 'b', 'a'] - self.assertRaises(ValueError, build_component_list, None, - duplicate_list, convert=lambda x: x) + with self.assertRaises(ValueError) as cm: + build_component_list(None, duplicate_list, convert=lambda x: x) + self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence From dd378b4bb1499b9e8da7b193f19ae6a8d81b5a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 11:07:58 +0200 Subject: [PATCH 059/127] Generate localhost keys for tests on the fly --- .gitignore | 2 ++ conftest.py | 6 ++++ tests/keys/__init__.py | 63 ++++++++++++++++++++++++++++++++++++++++ tests/keys/localhost.crt | 20 ------------- tests/keys/localhost.key | 28 ------------------ 5 files changed, 71 insertions(+), 48 deletions(-) create mode 100644 tests/keys/__init__.py delete mode 100644 tests/keys/localhost.crt delete mode 100644 tests/keys/localhost.key diff --git a/.gitignore b/.gitignore index 83a2569dd..795e2605e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ htmlcov/ .coverage.* .cache/ .mypy_cache/ +/tests/keys/localhost.crt +/tests/keys/localhost.key # Windows Thumbs.db diff --git a/conftest.py b/conftest.py index b39d644a5..95d4eaef6 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,8 @@ from pathlib import Path import pytest +from tests.keys import generate_keys + def _py_files(folder): return (str(p) for p in Path(folder).rglob('*.py')) @@ -53,3 +55,7 @@ def reactor_pytest(request): def only_asyncio(request, reactor_pytest): if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio': pytest.skip('This test is only run with --reactor=asyncio') + + +# Generate localhost certificate files, needed by some tests +generate_keys() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py new file mode 100644 index 000000000..da202be4d --- /dev/null +++ b/tests/keys/__init__.py @@ -0,0 +1,63 @@ +import os +from datetime import datetime, timedelta + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.hashes import SHA256 +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) +from cryptography.x509 import ( + CertificateBuilder, + DNSName, + Name, + NameAttribute, + random_serial_number, + SubjectAlternativeName, +) +from cryptography.x509.oid import NameOID + + +# https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate +def generate_keys(): + folder = os.path.dirname(__file__) + + key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend(), + ) + with open(os.path.join(folder, 'localhost.key'), "wb") as f: + f.write( + key.private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=NoEncryption(), + ) + ) + + subject = issuer = Name( + [ + NameAttribute(NameOID.COUNTRY_NAME, u"IE"), + NameAttribute(NameOID.ORGANIZATION_NAME, u"Scrapy"), + NameAttribute(NameOID.COMMON_NAME, u"localhost"), + ] + ) + cert = ( + CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .add_extension( + SubjectAlternativeName([DNSName(u"localhost")]), + critical=False, + ) + .sign(key, SHA256(), default_backend()) + ) + with open(os.path.join(folder, 'localhost.crt'), "wb") as f: + f.write(cert.public_bytes(Encoding.PEM)) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt deleted file mode 100644 index 0cf5256d8..000000000 --- a/tests/keys/localhost.crt +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL -BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j -YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE -BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN -BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK -2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe -pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH -wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7 -56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN -cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB -o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU -VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B -AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ -VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX -FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf -2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE -W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb -dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA== ------END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key deleted file mode 100644 index 8fc373bdd..000000000 --- a/tests/keys/localhost.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w -mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM -39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT -yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2 -XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM -DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc -niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT -S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi -X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1 -QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN -HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W -birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p -hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis -vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o -8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq -12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi -M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo -gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8 -kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM -kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy -kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl -xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l -VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX -abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV -SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv -PWuPk5VX+qF78JhUhPO3nfnu ------END PRIVATE KEY----- From 195f738bbaa7c090df795161390056c3c6c7f1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 12:43:43 +0200 Subject: [PATCH 060/127] Update Python version references after dropping support for 3.5 (#4742) * Update Python version references after dropping support for 3.5 * Remove outdated test * Undo change affecting collect_asyncgen * Undo change to be handled by #4743 * Remove unused import * Remove unused import * Update tests/requirements-py3.txt Co-authored-by: Mikhail Korobov Co-authored-by: Mikhail Korobov --- README.rst | 2 +- docs/intro/install.rst | 4 ++-- docs/intro/tutorial.rst | 2 -- docs/topics/coroutines.rst | 15 ++++----------- scrapy/__init__.py | 4 ++-- scrapy/utils/defer.py | 11 ++--------- scrapy/utils/gz.py | 7 +++---- setup.py | 3 +-- tests/requirements-py3.txt | 3 +-- tests/test_crawl.py | 4 ---- tests/test_item.py | 12 ------------ tests/test_proxy_connect.py | 11 ----------- tox.ini | 2 +- 13 files changed, 17 insertions(+), 63 deletions(-) diff --git a/README.rst b/README.rst index 0e3939e9b..a8f2ba52b 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ including a list of features. Requirements ============ -* Python 3.5.2+ +* Python 3.6+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6d65ae2ee..8b4240bf6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,8 +9,8 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.5.2+, either the CPython implementation (default) or -the PyPy 5.9+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.6+, either the CPython implementation (default) or +the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). Installing Scrapy diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f96c78887..b3b8b4706 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -405,8 +405,6 @@ to get all of them: from sys import version_info -.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") - Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a0952d323..3b1549bd3 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - The following are known caveats of the current implementation that we aim - to address in future versions of Scrapy: - - - The callback output is not processed until the whole callback finishes. + .. note:: The callback output is not processed until the whole callback + finishes. As a side effect, if the callback raises an exception, none of its output is processed. - - Because `asynchronous generators were introduced in Python 3.6`_, you - can only use ``yield`` if you are using Python 3.6 or later. - - If you need to output multiple items or requests and you are using - Python 3.5, return an iterable (e.g. a list) instead. + This is a known caveat of the current implementation that we aim to + address in a future version of Scrapy. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/ - Usage ===== diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f0259a9b7..4326ca4aa 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 5, 2): - print("Scrapy %s requires Python 3.5.2" % __version__) +if sys.version_info < (3, 6): + print("Scrapy %s requires Python 3.6+" % __version__) sys.exit(1) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index a3950db75..21ba02a0b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -124,18 +124,11 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def _isfuture(o): - # workaround for Python before 3.5.3 not having asyncio.isfuture - if hasattr(asyncio, 'isfuture'): - return asyncio.isfuture(o) - return isinstance(o, asyncio.Future) - - def deferred_from_coro(o): """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o - if _isfuture(o) or inspect.isawaitable(o): + if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" @@ -167,7 +160,7 @@ def maybeDeferred_coro(f, *args, **kw): if isinstance(result, defer.Deferred): return result - elif _isfuture(result) or inspect.isawaitable(result): + elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) elif isinstance(result, failure.Failure): return defer.fail(result) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index fbd7bd18f..11d433cf5 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -6,11 +6,10 @@ import struct from scrapy.utils.decorators import deprecated -# - Python>=3.5 GzipFile's read() has issues returning leftover -# uncompressed data when input is corrupted -# (regression or bug-fix compared to Python 3.4) +# - GzipFile's read() has issues returning leftover uncompressed data when +# input is corrupted # - read1(), which fetches data before raising EOFError on next call -# works here but is only available from Python>=3.3 +# works here @deprecated('GzipFile.read1') def read1(gzf, size=-1): return gzf.read1(size) diff --git a/setup.py b/setup.py index 52a27c368..0c2281400 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,6 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -92,7 +91,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5.2', + python_requires='>=3.6', install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 44ddcded8..2247ed917 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,8 +2,7 @@ attrs dataclasses; python_version == '3.6' mitmproxy; python_version >= '3.7' -mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' -mitmproxy < 4; python_version < '3.6' +mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..c1b918baf 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import sys from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -405,7 +404,6 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): @@ -417,7 +415,6 @@ class CrawlSpiderTestCase(TestCase): itemcount = crawler.stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): @@ -437,7 +434,6 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({'foo': i}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): diff --git a/tests/test_item.py b/tests/test_item.py index 66fa761f0..78d204e34 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,3 @@ -import sys import unittest from unittest import mock from warnings import catch_warnings @@ -7,9 +6,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta -PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) - - class ItemTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): @@ -280,14 +276,6 @@ class ItemMetaTest(unittest.TestCase): with mock.patch.object(base, '__new__', new_mock): class MyItem(Item): - if not PY36_PLUS: - # This attribute is an internal attribute in Python 3.6+ - # and must be propagated properly. See - # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object - # In <3.6, we add a dummy attribute just to ensure the - # __new__ method propagates it correctly. - __classcell__ = object() - def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..6f70c4267 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -7,7 +7,6 @@ from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit from unittest import skipIf -import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -58,8 +57,6 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf(sys.version_info < (3, 5, 4), - "requires mitmproxy < 3.0.0, which these tests do not support") @skipIf("pypy" in sys.executable, "mitmproxy does not support PyPy") @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), @@ -88,14 +85,6 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, log) - @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6)) - @defer.inlineCallbacks - def test_https_connect_tunnel_error(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(log) - @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) diff --git a/tox.ini b/tox.ini index dec0d75e8..12e40295c 100644 --- a/tox.ini +++ b/tox.ini @@ -89,7 +89,7 @@ deps = basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.5, so we do + # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 From 3f0a677c0496da3d4d4294a182aa5e5b297a7cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 20:56:58 +0200 Subject: [PATCH 061/127] Cover version directive usage in the documentation policy (#4310) * Cover version directives in the documentation policy * Remove version directives in preparation for Scrapy 2.0 * Update the policy based on the deprecation policy * Only remove version directives after 3 years --- docs/contributing.rst | 11 ++++++++ docs/topics/api.rst | 2 -- docs/topics/autothrottle.rst | 2 -- docs/topics/benchmarking.rst | 2 -- docs/topics/commands.rst | 4 --- docs/topics/contracts.rst | 2 -- docs/topics/downloader-middleware.rst | 36 --------------------------- docs/topics/extensions.rst | 4 --- docs/topics/feed-exports.rst | 2 -- docs/topics/request-response.rst | 12 --------- docs/topics/settings.rst | 4 --- docs/topics/spider-middleware.rst | 6 ----- 12 files changed, 11 insertions(+), 76 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 525ad3497..675f55c38 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -199,6 +199,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +Documentation updates that cover new or modified features must use Sphinx’s +:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use +``VERSION`` as version, we will replace it with the actual version right before +the corresponding release. When we release a new major or minor version of +Scrapy, we remove these directives if they are older than 3 years. + +Documentation about deprecated features must be removed as those features are +deprecated, so that new readers do not run into it. New deprecations and +deprecation removals are documented in the :ref:`release notes `. + + Tests ===== diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 52509ffdf..445b2979f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -4,8 +4,6 @@ Core API ======== -.. versionadded:: 0.15 - This section documents the Scrapy core API, and it's intended for developers of extensions and middlewares. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4317019fc..8e6aae65c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 1.1 - Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 99469ebf1..b01a66188 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -4,8 +4,6 @@ Benchmarking ============ -.. versionadded:: 0.17 - Scrapy comes with a simple benchmarking suite that spawns a local HTTP server and crawls it at the maximum possible speed. The goal of this benchmarking is to get an idea of how Scrapy performs in your hardware, in order to have a diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9638a2322..7de5e8121 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,8 +6,6 @@ Command line tool ================= -.. versionadded:: 0.10 - Scrapy is controlled through the ``scrapy`` command-line tool, to be referred here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -566,8 +564,6 @@ and Platform info, which is useful for bug reports. bench ----- -.. versionadded:: 0.17 - * Syntax: ``scrapy bench`` * Requires project: *no* diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 430720fe3..e61421bf1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -4,8 +4,6 @@ Spiders Contracts ================= -.. versionadded:: 0.15 - Testing spiders can get particularly annoying and while nothing prevents you from writing unit tests the task gets cumbersome quickly. Scrapy offers an integrated way of testing your spiders by the means of contracts. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 323e553e5..06e614941 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -217,8 +217,6 @@ The following settings can be used to configure the cookie middleware: Multiple cookie sessions per spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.15 - There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. @@ -475,8 +473,6 @@ DBM storage backend .. class:: DbmCacheStorage - .. versionadded:: 0.13 - A DBM_ storage backend is also available for the HTTP cache middleware. By default, it uses the :mod:`dbm`, but you can change it with the @@ -549,15 +545,10 @@ settings: HTTPCACHE_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.11 - Default: ``False`` Whether the HTTP cache will be enabled. -.. versionchanged:: 0.11 - Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -570,9 +561,6 @@ Expiration time for cached requests, in seconds. Cached requests older than this time will be re-downloaded. If zero, cached requests will never expire. -.. versionchanged:: 0.11 - Before 0.11, zero meant cached requests always expire. - .. setting:: HTTPCACHE_DIR HTTPCACHE_DIR @@ -589,8 +577,6 @@ project data dir. For more info see: :ref:`topics-project-structure`. HTTPCACHE_IGNORE_HTTP_CODES ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``[]`` Don't cache response with these HTTP codes. @@ -609,8 +595,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade HTTPCACHE_IGNORE_SCHEMES ^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``['file']`` Don't cache responses with these URI schemes. @@ -629,8 +613,6 @@ The class which implements the cache storage backend. HTTPCACHE_DBM_MODULE ^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend @@ -641,8 +623,6 @@ The database module to use in the :ref:`DBM storage backend HTTPCACHE_POLICY ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.18 - Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. @@ -652,8 +632,6 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 1.0 - Default: ``False`` If enabled, will compress all cached data with gzip. @@ -664,8 +642,6 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``False`` If enabled, will cache pages unconditionally. @@ -684,8 +660,6 @@ responses you feed to the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``[]`` List of Cache-Control directives in responses to be ignored. @@ -735,8 +709,6 @@ HttpProxyMiddleware .. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware -.. versionadded:: 0.8 - .. reqmeta:: proxy .. class:: HttpProxyMiddleware @@ -817,8 +789,6 @@ RedirectMiddleware settings REDIRECT_ENABLED ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Redirect middleware will be enabled. @@ -860,8 +830,6 @@ MetaRefreshMiddleware settings METAREFRESH_ENABLED ^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 - Default: ``True`` Whether the Meta Refresh middleware will be enabled. @@ -924,8 +892,6 @@ RetryMiddleware Settings RETRY_ENABLED ^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Retry middleware will be enabled. @@ -1179,8 +1145,6 @@ AjaxCrawlMiddleware Settings AJAXCRAWL_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.21 - Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0fc83e645..14096ada4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -288,8 +288,6 @@ If zero (or non set), spiders won't be closed by number of passed items. CLOSESPIDER_PAGECOUNT """"""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of responses to crawl. If the spider @@ -302,8 +300,6 @@ number of crawled responses. CLOSESPIDER_ERRORCOUNT """""""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of errors to receive before diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cd4f7cf29..9fb2189e8 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -4,8 +4,6 @@ Feed exports ============ -.. versionadded:: 0.10 - One of the most frequently required features when implementing scrapers is being able to store the scraped data properly and, quite often, that means generating an "export file" with the scraped data (commonly called "export diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d0136137f..30b1945d0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -553,18 +553,6 @@ fields with form data from :class:`Response` objects. The other parameters of this class method are passed directly to the :class:`FormRequest` ``__init__`` method. - .. versionadded:: 0.10.3 - The ``formname`` parameter. - - .. versionadded:: 0.17 - The ``formxpath`` parameter. - - .. versionadded:: 1.1.0 - The ``formcss`` parameter. - - .. versionadded:: 1.1.0 - The ``formid`` parameter. - Request usage examples ---------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2924c0566..d010a0023 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1076,8 +1076,6 @@ See :ref:`topics-extensions-ref-memusage`. MEMUSAGE_CHECK_INTERVAL_SECONDS ------------------------------- -.. versionadded:: 1.1 - Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` @@ -1358,8 +1356,6 @@ The class that will be used for loading spiders, which must implement the SPIDER_LOADER_WARN_ONLY ----------------------- -.. versionadded:: 1.3.3 - Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..fc114a63f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_start_requests(start_requests, spider) - .. versionadded:: 0.15 - This method is called with the start requests of the spider, and works similarly to the :meth:`process_spider_output` method, except that it doesn't have a response associated and must return only requests (not @@ -341,8 +339,6 @@ RefererMiddleware settings REFERER_ENABLED ^^^^^^^^^^^^^^^ -.. versionadded:: 0.15 - Default: ``True`` Whether to enable referer middleware. @@ -352,8 +348,6 @@ Whether to enable referer middleware. REFERRER_POLICY ^^^^^^^^^^^^^^^ -.. versionadded:: 1.4 - Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. reqmeta:: referrer_policy From 0e579182319504ba7bfd0c09333fe92f70c6d312 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Fri, 28 Aug 2020 13:58:32 +0200 Subject: [PATCH 062/127] test(Slot): cover __repr__ Issue: #4324 --- scrapy/core/downloader/__init__.py | 2 +- tests/test_core_downloader.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/test_core_downloader.py diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 12a9db6dd..4f7ab594f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -43,7 +43,7 @@ class Slot: cls_name = self.__class__.__name__ return (f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r}") + f"randomize_delay={self.randomize_delay!r})") def __str__(self): return ( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py new file mode 100644 index 000000000..113ea8f19 --- /dev/null +++ b/tests/test_core_downloader.py @@ -0,0 +1,10 @@ +from twisted.trial import unittest + +from scrapy.core.downloader import Slot + + +class SlotTest(unittest.TestCase): + + def test_repr(self): + slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) + self.assertEqual(repr(slot), 'Slot(concurrency=8, delay=0.10, randomize_delay=True)') From de640f41ecfa1a87b39d9e9268c396fc97353fc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 28 Aug 2020 18:27:36 +0500 Subject: [PATCH 063/127] Merge back tests/py36/_test_crawl.py. --- conftest.py | 2 -- tests/py36/_test_crawl.py | 57 --------------------------------------- tests/spiders.py | 53 ++++++++++++++++++++++++++++++++++++ tests/test_crawl.py | 6 ++--- 4 files changed, 56 insertions(+), 62 deletions(-) delete mode 100644 tests/py36/_test_crawl.py diff --git a/conftest.py b/conftest.py index b39d644a5..be97b7714 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ collect_ignore = [ *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), ] for line in open('tests/ignores.txt'): diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py deleted file mode 100644 index 162a53760..000000000 --- a/tests/py36/_test_crawl.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio - -from scrapy import Request -from tests.spiders import SimpleSpider - - -class AsyncDefAsyncioGenSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen' - - async def parse(self, response): - await asyncio.sleep(0.2) - yield {'foo': 42} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_loop' - - async def parse(self, response): - for i in range(10): - await asyncio.sleep(0.1) - yield {'foo': i} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_complex' - initial_reqs = 4 - following_reqs = 3 - depth = 2 - - def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}, - dont_filter=True, - callback=cb) - - def start_requests(self): - for i in range(1, self.initial_reqs + 1): - yield self._get_req(i) - - async def parse(self, response): - index = response.meta['index'] - yield {'index': index} - if index < 10 ** self.depth: - for new_index in range(10 * index, 10 * index + self.following_reqs): - yield self._get_req(new_index) - yield self._get_req(index, cb=self.parse2) - await asyncio.sleep(0.1) - yield {'index': index + 5} - - async def parse2(self, response): - await asyncio.sleep(0.1) - yield {'index2': response.meta['index']} diff --git a/tests/spiders.py b/tests/spiders.py index 8a0ee44b7..8a85d2b51 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index, cb=None): + return Request(self.mockserver.url("/status?n=200&request=%d" % index), + meta={'index': index}, + dont_filter=True, + callback=cb) + + def start_requests(self): + for i in range(1, self.initial_reqs + 1): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) + await asyncio.sleep(0.1) + yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c1b918baf..ba8c3fd3c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -19,6 +19,9 @@ from scrapy.http.response import Response from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( + AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenLoopSpider, + AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, @@ -407,7 +410,6 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - from tests.py36._test_crawl import AsyncDefAsyncioGenSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -423,7 +425,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: @@ -442,7 +443,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) From ffdf6fe100cf0ac99a726eab9d927e65f4c01364 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 29 Aug 2020 07:21:48 +0200 Subject: [PATCH 064/127] use f-strings for the newly merged code from master Issue: #4324 --- tests/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 3c01c0217..106392ea6 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -177,7 +177,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), + return Request(self.mockserver.url(f"/status?n=200&request={index}"), meta={'index': index}, dont_filter=True, callback=cb) From 8123c427373778398036bb375bf03b3cd6b240de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 28 Aug 2020 18:31:09 +0500 Subject: [PATCH 065/127] Simplify running spiders in CrawlSpiderTestCase. --- tests/test_crawl.py | 56 +++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index ba8c3fd3c..a8c9b6d7f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -335,6 +335,19 @@ class CrawlSpiderTestCase(TestCase): def tearDown(self): self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks + def _run_spider(self, spider_cls): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(spider_cls) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + return log, items, crawler.stats + @defer.inlineCallbacks def test_crawlspider_with_parse(self): self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) @@ -379,15 +392,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) self.assertIn("Got response 200", str(log)) self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) @@ -410,27 +415,17 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 10) for i in range(10): self.assertIn({'foo': i}, items) @@ -438,15 +433,8 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) - itemcount = crawler.stats.get_value('item_scraped_count') + _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 156) # some random items for i in [1, 4, 21, 22, 207, 311]: @@ -457,9 +445,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): self.assertIn("Got response 200, req_id %d" % req_id, str(log)) From 90ca9350f56c8c450c3385590d77390ad1d77365 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 29 Aug 2020 08:03:03 +0000 Subject: [PATCH 066/127] Update scrapy/commands/check.py Co-authored-by: Mikhail Korobov --- scrapy/commands/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 7e848dc97..ae21d86e6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln(f"Ran {run} contract{plural} in {stop - start:.3f}") + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s") writeln() infos = [] From a8e895e684184e967a751ea28a7102f21dd74834 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 10:57:22 +0300 Subject: [PATCH 067/127] kwargs for Item exporters classes test docs --- docs/topics/feed-exports.rst | 5 +++++ scrapy/extensions/feedexport.py | 1 + scrapy/utils/conf.py | 1 + tests/test_feedexport.py | 37 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9fb2189e8..e69a64195 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -303,6 +303,9 @@ For instance:: 'store_empty': False, 'fields': None, 'indent': 4, + 'item_export_kwargs': { + 'export_empty_fields': True, + }, }, '/home/user/documents/items.xml': { 'format': 'xml', @@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. +- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 980825499..f32d48fa5 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -360,6 +360,7 @@ class FeedExporter: fields_to_export=feed_options['fields'], encoding=feed_options['encoding'], indent=feed_options['indent'], + **feed_options['item_export_kwargs'], ) slot = _FeedSlot( file=file, diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 4e7a9967e..b904c4a03 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings): out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) + out.setdefault("item_export_kwargs", dict()) if settings["FEED_EXPORT_INDENT"] is None: out.setdefault("indent", None) else: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 840e0f87b..e88e4f5ce 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1252,6 +1252,43 @@ class FeedExportTest(FeedExportTestBase): for fmt in ['json', 'xml', 'csv']: self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) + @defer.inlineCallbacks + def test_extend_kwargs(self): + items = [{'foo': 'FOO', 'bar': 'BAR'}] + + expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8') + expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8') + test_cases = [ + # with title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=True), + }, + 'expected': expected_with_title_csv, + }, + # without title + { + 'options': { + 'format': 'csv', + 'item_export_kwargs': dict(include_headers_line=False), + }, + 'expected': expected_without_title_csv, + }, + ] + + for row in test_cases: + feed_options = row['options'] + settings = { + 'FEEDS': { + self._random_temp_filename(): feed_options, + }, + 'FEED_EXPORT_INDENT': None, + } + + data = yield self.exported_data(items, settings) + self.assertEqual(row['expected'], data[feed_options['format']]) + class BatchDeliveriesTest(FeedExportTestBase): __test__ = True From fc3c66ce950e945e7ac6e19fea31e8454147ac29 Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 11:44:48 +0300 Subject: [PATCH 068/127] fix tests: * FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_empty * FeedExportConfigTestCase.test_feed_complete_default_values_from_settings_non_empty --- tests/test_utils_conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 061bc8c7c..dc2560add 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": (1, 2, 3, 4), "batch_item_count": 2, + "item_export_kwargs": dict(), }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "store_empty": True, "uri_params": None, "batch_item_count": 2, + "item_export_kwargs": dict(), }) From 71d2c2f1a319b570eb2026707d49c4f62d59296e Mon Sep 17 00:00:00 2001 From: maranqz Date: Sun, 30 Aug 2020 12:43:44 +0300 Subject: [PATCH 069/127] improve view of dict --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e88e4f5ce..5738e36f1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1263,7 +1263,7 @@ class FeedExportTest(FeedExportTestBase): { 'options': { 'format': 'csv', - 'item_export_kwargs': dict(include_headers_line=True), + 'item_export_kwargs': {'include_headers_line': True}, }, 'expected': expected_with_title_csv, }, @@ -1271,7 +1271,7 @@ class FeedExportTest(FeedExportTestBase): { 'options': { 'format': 'csv', - 'item_export_kwargs': dict(include_headers_line=False), + 'item_export_kwargs': {'include_headers_line': False}, }, 'expected': expected_without_title_csv, }, From d10464ca96a24d37c854992e2485d622e8eec2b6 Mon Sep 17 00:00:00 2001 From: Ilia Sergunin Date: Tue, 1 Sep 2020 10:13:40 +0300 Subject: [PATCH 070/127] Update docs/topics/feed-exports.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e69a64195..1744cfd74 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -335,7 +335,7 @@ as a fallback value if that key is not provided for a specific feed definition: - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. -- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters ` classes. +- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). From 307e35c6641c0d9cf4b251996ebefd1347c2c1fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 1 Sep 2020 06:04:00 -0300 Subject: [PATCH 071/127] Improve check for invalid cookie in CookiesMiddleware (#4772) --- scrapy/downloadermiddlewares/cookies.py | 2 +- tests/test_downloadermiddleware_cookies.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index e2b7dd901..87f8152a4 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -74,7 +74,7 @@ class CookiesMiddleware: """ decoded = {} for key in ("name", "value", "path", "domain"): - if not cookie.get(key): + if cookie.get(key) is None: if key in ("name", "value"): msg = "Invalid cookie found in request {}: {} ('{}' is missing)" logger.warning(msg.format(request, cookie, key)) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 010577415..a3de307ee 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -322,6 +322,9 @@ class CookiesMiddlewareTest(TestCase): cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] req2 = Request('http://example.org/2', cookies=cookies2) assert self.mw.process_request(req2, self.spider) is None + cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}] + req3 = Request('http://example.org/3', cookies=cookies3) + assert self.mw.process_request(req3, self.spider) is None lc.check( ("scrapy.downloadermiddlewares.cookies", "WARNING", @@ -331,6 +334,11 @@ class CookiesMiddlewareTest(TestCase): "WARNING", "Invalid cookie found in request :" " {'name': 'foo'} ('value' is missing)"), + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None} ('value' is missing)"), ) self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') + self.assertCookieValEqual(req3.headers['Cookie'], 'key=') From 959222df7e80334810636166a8adb60ef35728de Mon Sep 17 00:00:00 2001 From: drs-11 Date: Sat, 5 Sep 2020 21:32:05 +0530 Subject: [PATCH 072/127] check for unparseable no_proxy values --- scrapy/downloadermiddlewares/httpproxy.py | 7 ++++++- tests/test_downloadermiddleware_httpproxy.py | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 04da11311..d2665b655 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -13,7 +13,12 @@ class HttpProxyMiddleware: self.auth_encoding = auth_encoding self.proxies = {} for type_, url in getproxies().items(): - self.proxies[type_] = self._get_proxy(url, type_) + try: + self.proxies[type_] = self._get_proxy(url, type_) + # some values such as '/var/run/docker.sock' can't be parsed + # by _parse_proxy and as such should be skipped + except ValueError: + continue @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 351631eb8..81d6cc335 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,7 +123,11 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' + os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') From 7a83474cc534642e8daf581d4b427408343377f2 Mon Sep 17 00:00:00 2001 From: KAILASA's Sri Nithya Priyeshananda <68758690+sripriyesha@users.noreply.github.com> Date: Tue, 8 Sep 2020 17:16:31 +0200 Subject: [PATCH 073/127] add mention of FTP server storage in media storage intro At the beginning of this doc, in "Specifying where to store the media" feature details, FTP server storage mention was missing --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..c2255ea79 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -15,7 +15,7 @@ typically you'll either use the Files Pipeline or the Images Pipeline. Both pipelines implement these features: * Avoid re-downloading media that was downloaded recently -* Specifying where to store the media (filesystem directory, Amazon S3 bucket, +* Specifying where to store the media (filesystem directory, FTP server, Amazon S3 bucket, Google Cloud Storage bucket) The Images Pipeline has a few extra functions for processing images: From 82ba7c8b529e004a62db4681f16a482ae4e769f1 Mon Sep 17 00:00:00 2001 From: drs-11 Date: Thu, 10 Sep 2020 20:56:39 +0530 Subject: [PATCH 074/127] created separate test for invalid no-proxy values --- tests/test_downloadermiddleware_httpproxy.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 81d6cc335..7c97bf32a 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -123,11 +123,7 @@ class TestHttpProxyMiddleware(TestCase): def test_no_proxy(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' - os.environ['no_proxy'] = '/var/run/docker.sock' mw = HttpProxyMiddleware() - # '/var/run/docker.sock' may be used by the user for - # no_proxy value but is not parseable and should be skipped - assert 'no' not in mw.proxies os.environ['no_proxy'] = '*' req = Request('http://noproxy.com') @@ -149,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase): req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) + + def test_no_proxy_invalid_values(self): + os.environ['no_proxy'] = '/var/run/docker.sock' + mw = HttpProxyMiddleware() + # '/var/run/docker.sock' may be used by the user for + # no_proxy value but is not parseable and should be skipped + assert 'no' not in mw.proxies From 5e997587d9b13344a0afa9bb4cf781829a66ce23 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Sun, 20 Sep 2020 18:06:46 +0500 Subject: [PATCH 075/127] Remove dead boto2 code, deprecate is_botocore() (#4776) --- scrapy/core/downloader/handlers/s3.py | 55 ++++----------- scrapy/extensions/feedexport.py | 35 ++++------ scrapy/pipelines/files.py | 97 +++++++++------------------ scrapy/utils/boto.py | 23 ++++++- scrapy/utils/test.py | 29 +++----- tests/test_feedexport.py | 69 ++----------------- tests/test_pipeline_files.py | 15 ++--- 7 files changed, 96 insertions(+), 227 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 0ef977893..1966570d4 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,41 +2,20 @@ from urllib.parse import unquote from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.exceptions import NotConfigured -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import create_instance -def _get_boto_connection(): - from boto.s3.connection import S3Connection - - class _v19_S3Connection(S3Connection): - """A dummy S3Connection wrapper that doesn't do any synchronous download""" - def _mexe(self, method, bucket, key, headers, *args, **kwargs): - return headers - - class _v20_S3Connection(S3Connection): - """A dummy S3Connection wrapper that doesn't do any synchronous download""" - def _mexe(self, http_request, *args, **kwargs): - http_request.authorize(connection=self) - return http_request.headers - - try: - import boto.auth # noqa: F401 - except ImportError: - _S3Connection = _v19_S3Connection - else: - _S3Connection = _v20_S3Connection - - return _S3Connection - - class S3DownloadHandler: def __init__(self, settings, *, crawler=None, aws_access_key_id=None, aws_secret_access_key=None, httpdownloadhandler=HTTPDownloadHandler, **kw): + if not is_botocore_available(): + raise NotConfigured('missing botocore library') + if not aws_access_key_id: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: @@ -51,23 +30,15 @@ class S3DownloadHandler: self.anon = kw.get('anon') self._signer = None - if is_botocore(): - import botocore.auth - import botocore.credentials - kw.pop('anon', None) - if kw: - raise TypeError(f'Unexpected keyword arguments: {kw}') - if not self.anon: - SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] - self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) - else: - _S3Connection = _get_boto_connection() - try: - self.conn = _S3Connection( - aws_access_key_id, aws_secret_access_key, **kw) - except Exception as ex: - raise NotConfigured(str(ex)) + import botocore.auth + import botocore.credentials + kw.pop('anon', None) + if kw: + raise TypeError(f'Unexpected keyword arguments: {kw}') + if not self.anon: + SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] + self._signer = SignerCls(botocore.credentials.Credentials( + aws_access_key_id, aws_secret_access_key)) _http_handler = create_instance( objcls=httpdownloadhandler, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 980825499..9f712285f 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -19,7 +19,7 @@ from zope.interface import implementer, Interface from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -120,22 +120,19 @@ class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, feed_options=None): + if not is_botocore_available(): + raise NotConfigured('missing botocore library') u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key - self.is_botocore = is_botocore() self.keyname = u.path[1:] # remove first "/" self.acl = acl - if self.is_botocore: - import botocore.session - session = botocore.session.get_session() - self.s3_client = session.create_client( - 's3', aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key) - else: - import boto - self.connect_s3 = boto.connect_s3 + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' 'suppress this warning, remove the overwrite ' @@ -154,18 +151,10 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - if self.is_botocore: - kwargs = {'ACL': self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, - **kwargs) - else: - conn = self.connect_s3(self.access_key, self.secret_key) - bucket = conn.get_bucket(self.bucketname, validate=False) - key = bucket.new_key(self.keyname) - kwargs = {'policy': self.acl} if self.acl else {} - key.set_contents_from_file(file, **kwargs) - key.close() + kwargs = {'ACL': self.acl} if self.acl else {} + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file, + **kwargs) file.close() diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 99a72aa70..13ecd4e6c 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -11,7 +11,6 @@ import os import time from collections import defaultdict from contextlib import suppress -from email.utils import mktime_tz, parsedate_tz from ftplib import FTP from io import BytesIO from urllib.parse import urlparse @@ -23,7 +22,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaselessDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -91,86 +90,54 @@ class S3FilesStore: } def __init__(self, uri): - self.is_botocore = is_botocore() - if self.is_botocore: - import botocore.session - session = botocore.session.get_session() - self.s3_client = session.create_client( - 's3', - aws_access_key_id=self.AWS_ACCESS_KEY_ID, - aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, - endpoint_url=self.AWS_ENDPOINT_URL, - region_name=self.AWS_REGION_NAME, - use_ssl=self.AWS_USE_SSL, - verify=self.AWS_VERIFY - ) - else: - from boto.s3.connection import S3Connection - self.S3Connection = S3Connection + if not is_botocore_available(): + raise NotConfigured('missing botocore library') + import botocore.session + session = botocore.session.get_session() + self.s3_client = session.create_client( + 's3', + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + endpoint_url=self.AWS_ENDPOINT_URL, + region_name=self.AWS_REGION_NAME, + use_ssl=self.AWS_USE_SSL, + verify=self.AWS_VERIFY + ) if not uri.startswith("s3://"): raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): def _onsuccess(boto_key): - if self.is_botocore: - checksum = boto_key['ETag'].strip('"') - last_modified = boto_key['LastModified'] - modified_stamp = time.mktime(last_modified.timetuple()) - else: - checksum = boto_key.etag.strip('"') - last_modified = boto_key.last_modified - modified_tuple = parsedate_tz(last_modified) - modified_stamp = int(mktime_tz(modified_tuple)) + checksum = boto_key['ETag'].strip('"') + last_modified = boto_key['LastModified'] + modified_stamp = time.mktime(last_modified.timetuple()) return {'checksum': checksum, 'last_modified': modified_stamp} return self._get_boto_key(path).addCallback(_onsuccess) - def _get_boto_bucket(self): - # disable ssl (is_secure=False) because of this python bug: - # https://bugs.python.org/issue5103 - c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) - return c.get_bucket(self.bucket, validate=False) - def _get_boto_key(self, path): key_name = f'{self.prefix}{path}' - if self.is_botocore: - return threads.deferToThread( - self.s3_client.head_object, - Bucket=self.bucket, - Key=key_name) - else: - b = self._get_boto_bucket() - return threads.deferToThread(b.get_key, key_name) + return threads.deferToThread( + self.s3_client.head_object, + Bucket=self.bucket, + Key=key_name) def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" key_name = f'{self.prefix}{path}' buf.seek(0) - if self.is_botocore: - extra = self._headers_to_botocore_kwargs(self.HEADERS) - if headers: - extra.update(self._headers_to_botocore_kwargs(headers)) - return threads.deferToThread( - self.s3_client.put_object, - Bucket=self.bucket, - Key=key_name, - Body=buf, - Metadata={k: str(v) for k, v in (meta or {}).items()}, - ACL=self.POLICY, - **extra) - else: - b = self._get_boto_bucket() - k = b.new_key(key_name) - if meta: - for metakey, metavalue in meta.items(): - k.set_metadata(metakey, str(metavalue)) - h = self.HEADERS.copy() - if headers: - h.update(headers) - return threads.deferToThread( - k.set_contents_from_string, buf.getvalue(), - headers=h, policy=self.POLICY) + extra = self._headers_to_botocore_kwargs(self.HEADERS) + if headers: + extra.update(self._headers_to_botocore_kwargs(headers)) + return threads.deferToThread( + self.s3_client.put_object, + Bucket=self.bucket, + Key=key_name, + Body=buf, + Metadata={k: str(v) for k, v in (meta or {}).items()}, + ACL=self.POLICY, + **extra) def _headers_to_botocore_kwargs(self, headers): """ Convert headers to botocore keyword agruments. diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 12321caa5..3374c57c7 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,11 +1,32 @@ """Boto/botocore helpers""" +import warnings -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning def is_botocore(): + """ Returns True if botocore is available, otherwise raises NotConfigured. Never returns False. + + Previously, when boto was supported in addition to botocore, this returned False if boto was available + but botocore wasn't. + """ + message = ( + 'is_botocore() is deprecated and always returns True or raises an Exception, ' + 'so it cannot be used for checking if boto is available instead of botocore. ' + 'You can use scrapy.utils.boto.is_botocore_available() to check if botocore ' + 'is available.' + ) + warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) try: import botocore # noqa: F401 return True except ImportError: raise NotConfigured('missing botocore library') + + +def is_botocore_available(): + try: + import botocore # noqa: F401 + return True + except ImportError: + return False diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index f54942ffb..94d0ae2d3 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -10,8 +10,7 @@ from unittest import mock from importlib import import_module from twisted.trial.unittest import SkipTest -from scrapy.exceptions import NotConfigured -from scrapy.utils.boto import is_botocore +from scrapy.utils.boto import is_botocore_available def assert_aws_environ(): @@ -29,29 +28,19 @@ def assert_gcs_environ(): def skip_if_no_boto(): - try: - is_botocore() - except NotConfigured as e: - raise SkipTest(e) + if not is_botocore_available(): + raise SkipTest('missing botocore library') def get_s3_content_and_delete(bucket, path, with_key=False): """ Get content from s3 key, and delete key afterwards. """ - if is_botocore(): - import botocore.session - session = botocore.session.get_session() - client = session.create_client('s3') - key = client.get_object(Bucket=bucket, Key=path) - content = key['Body'].read() - client.delete_object(Bucket=bucket, Key=path) - else: - import boto - # assuming boto=2.2.2 - bucket = boto.connect_s3().get_bucket(bucket, validate=False) - key = bucket.get_key(path) - content = key.get_contents_as_string() - bucket.delete_key(path) + import botocore.session + session = botocore.session.get_session() + client = session.create_client('s3') + key = client.get_object(Bucket=bucket, Key=path) + content = key['Body'].read() + client.delete_object(Bucket=bucket, Key=path) return (content, key) if with_key else content diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 840e0f87b..33ac51712 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -45,6 +45,7 @@ from scrapy.utils.test import ( get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, + skip_if_no_boto, ) from tests.mockserver import MockFTPServer, MockServer @@ -227,10 +228,7 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): def test_parse_credentials(self): - try: - import botocore # noqa: F401 - except ImportError: - raise unittest.SkipTest("S3FeedStorage requires botocore") + skip_if_no_boto() aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) @@ -324,11 +322,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): - try: - import botocore # noqa: F401 - except ImportError: - raise unittest.SkipTest('botocore is required') - + skip_if_no_boto() storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -344,11 +338,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): - try: - import botocore # noqa: F401 - except ImportError: - raise unittest.SkipTest('botocore is required') - + skip_if_no_boto() storage = S3FeedStorage( 's3://mybucket/export.csv', 'access_key', @@ -366,57 +356,6 @@ class S3FeedStorageTest(unittest.TestCase): 'custom-acl' ) - @defer.inlineCallbacks - def test_store_not_botocore_without_acl(self): - storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, None) - - storage.is_botocore = False - storage.connect_s3 = mock.MagicMock() - self.assertFalse(storage.is_botocore) - - yield storage.store(BytesIO(b'test file')) - - conn = storage.connect_s3(*storage.connect_s3.call_args) - bucket = conn.get_bucket(*conn.get_bucket.call_args) - key = bucket.new_key(*bucket.new_key.call_args) - self.assertNotIn( - dict(policy='custom-acl'), - key.set_contents_from_file.call_args - ) - - @defer.inlineCallbacks - def test_store_not_botocore_with_acl(self): - storage = S3FeedStorage( - 's3://mybucket/export.csv', - 'access_key', - 'secret_key', - 'custom-acl' - ) - self.assertEqual(storage.access_key, 'access_key') - self.assertEqual(storage.secret_key, 'secret_key') - self.assertEqual(storage.acl, 'custom-acl') - - storage.is_botocore = False - storage.connect_s3 = mock.MagicMock() - self.assertFalse(storage.is_botocore) - - yield storage.store(BytesIO(b'test file')) - - conn = storage.connect_s3(*storage.connect_s3.call_args) - bucket = conn.get_bucket(*conn.get_bucket.call_args) - key = bucket.new_key(*bucket.new_key.call_args) - self.assertIn( - dict(policy='custom-acl'), - key.set_contents_from_file.call_args - ) - def test_overwrite_default(self): with LogCapture() as log: S3FeedStorage( diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 1dd7031fe..d5b0bb3d8 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -22,7 +22,6 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.settings import Settings -from scrapy.utils.boto import is_botocore from scrapy.utils.test import ( assert_aws_environ, assert_gcs_environ, @@ -437,16 +436,10 @@ class TestS3FilesStore(unittest.TestCase): content, key = get_s3_content_and_delete( u.hostname, u.path[1:], with_key=True) self.assertEqual(content, data) - if is_botocore(): - self.assertEqual(key['Metadata'], {'foo': 'bar'}) - self.assertEqual( - key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key['ContentType'], 'image/png') - else: - self.assertEqual(key.metadata, {'foo': 'bar'}) - self.assertEqual( - key.cache_control, S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key.content_type, 'image/png') + self.assertEqual(key['Metadata'], {'foo': 'bar'}) + self.assertEqual( + key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) + self.assertEqual(key['ContentType'], 'image/png') class TestGCSFilesStore(unittest.TestCase): From 7f1e74daa28caa481b4a484392a7dd2a18fc290e Mon Sep 17 00:00:00 2001 From: Mirwaisse Djanbaz Date: Mon, 21 Sep 2020 14:38:16 +0200 Subject: [PATCH 076/127] =?UTF-8?q?dependencides=20=E2=86=92=20dependencie?= =?UTF-8?q?s=20(#4800)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 8b4240bf6..fe7bc0a2a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -211,7 +211,7 @@ PyPy We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. -Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy. +Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. This means that these dependecies will be built during installation. On macOS, you are likely to face an issue with building Cryptography dependency, solution to this problem is described From 3989f64baa39f7e42b0f798dec15cd250e0fba21 Mon Sep 17 00:00:00 2001 From: Mirwaisse Djanbaz Date: Mon, 21 Sep 2020 14:40:00 +0200 Subject: [PATCH 077/127] =?UTF-8?q?dependecies=20=E2=86=92=20dependencies?= =?UTF-8?q?=20(#4801)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index fe7bc0a2a..3bfd3bc3b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -212,7 +212,7 @@ We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. -This means that these dependecies will be built during installation. +This means that these dependencies will be built during installation. On macOS, you are likely to face an issue with building Cryptography dependency, solution to this problem is described `here `_, From 008cf1c75ebe72a607149d7efd8f902f8763bc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 20:45:21 +0200 Subject: [PATCH 078/127] Remove a test that has never been executed in Python 3 --- tests/test_downloader_handlers.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0a374c161..3e8d7e6b9 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -868,29 +868,6 @@ class S3TestCase(unittest.TestCase): self.assertEqual(httpreq.headers['Authorization'], b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') - def test_request_signing5(self): - try: - import botocore # noqa: F401 - except ImportError: - pass - else: - raise unittest.SkipTest( - 'botocore does not support overriding date with x-amz-date') - # deletes an object from the 'johnsmith' bucket using the - # path-style and Date alternative. - date = 'Tue, 27 Mar 2007 21:20:27 +0000' - req = Request( - 's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={ - 'Date': date, - 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', - }) - with self._mocked_date(date): - httpreq = self.download_request(req, self.spider) - # botocore does not override Date with x-amz-date - self.assertEqual( - httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') - def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. date = 'Tue, 27 Mar 2007 21:06:08 +0000' From 56f05fb16476b40f773edc9e93d41c8893a349c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:01:09 +0200 Subject: [PATCH 079/127] Use mocking for tests/test_feedexport.py::S3FeedStorageTest::test_store --- tests/test_feedexport.py | 50 +++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 33ac51712..a8c0cf686 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,6 +13,7 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock +from unittest.mock import call from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url @@ -254,21 +255,42 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - assert_aws_environ() - uri = os.environ.get('S3_TEST_FILE_URI') - if not uri: - raise unittest.SkipTest("No S3 URI available for testing") - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - storage = S3FeedStorage(uri, access_key, secret_key) + skip_if_no_boto() + + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + } + crawler = get_crawler(settings_dict=settings) + bucket = 'mybucket' + key = 'export.csv' + storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}') verifyObject(IFeedStorage, storage) - file = storage.open(scrapy.Spider("default")) - expected_content = b"content: \xe2\x98\x83" - file.write(expected_content) - yield storage.store(file) - u = urlparse(uri) - content = get_s3_content_and_delete(u.hostname, u.path[1:]) - self.assertEqual(content, expected_content) + + file = mock.MagicMock() + from botocore.stub import Stubber + with Stubber(storage.s3_client) as stub: + stub.add_response( + 'put_object', + expected_params={ + 'Body': file, + 'Bucket': bucket, + 'Key': key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + call.seek(0), + # The call to read does not happen with Stubber + call.close(), + ] + ) def test_init_without_acl(self): storage = S3FeedStorage( From 17e135377a564152e82cd2ad177a6a2749c15832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:54:39 +0200 Subject: [PATCH 080/127] Use mocking for tests/test_feedexport.py::BatchDeliveriesTest::test_s3_export --- tests/test_feedexport.py | 99 +++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index a8c0cf686..ac0b3ccaa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,6 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock -from unittest.mock import call from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url @@ -42,7 +41,6 @@ from scrapy.extensions.feedexport import ( from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import ( - assert_aws_environ, get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, @@ -286,9 +284,9 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual( file.method_calls, [ - call.seek(0), + mock.call.seek(0), # The call to read does not happen with Stubber - call.close(), + mock.call.close(), ] ) @@ -1518,46 +1516,53 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): - """ - Test export of items into s3 bucket. - S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini - to perform this test: - [testenv] - setenv = - AWS_SECRET_ACCESS_KEY = ABCD - AWS_ACCESS_KEY_ID = EFGH - S3_TEST_BUCKET_NAME = IJKL - """ - try: - import boto3 - except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto3") + skip_if_no_boto() - assert_aws_environ() - s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - if not s3_test_bucket_name: - raise unittest.SkipTest("No S3 BUCKET available for testing") - - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - prefix = f'tmp/{filename}' - s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' - storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) - settings = Settings({ - 'FEEDS': { - s3_test_file_uri: { - 'format': 'json', - }, - }, - 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, - }) + bucket = 'mybucket' items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), ] + + class CustomS3FeedStorage(S3FeedStorage): + + stubs = [] + + def open(self, *args, **kwargs): + from botocore.stub import ANY, Stubber + stub = Stubber(self.s3_client) + stub.activate() + CustomS3FeedStorage.stubs.append(stub) + stub.add_response( + 'put_object', + expected_params={ + 'Body': ANY, + 'Bucket': bucket, + 'Key': ANY, + }, + service_response={}, + ) + return super().open(*args, **kwargs) + + key = 'export.csv' + uri = f's3://{bucket}/{key}/%(batch_time)s.json' + batch_item_count = 1 + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + 'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count, + 'FEED_STORAGES': { + 's3': CustomS3FeedStorage, + }, + 'FEEDS': { + uri: { + 'format': 'json', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, uri) verifyObject(IFeedStorage, storage) class TestSpider(scrapy.Spider): @@ -1567,22 +1572,14 @@ class BatchDeliveriesTest(FeedExportTestBase): for item in items: yield item - s3 = boto3.resource('s3') - my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') - - with MockServer() as s: + with MockServer() as server: runner = CrawlerRunner(Settings(settings)) - TestSpider.start_urls = [s.url('/')] + TestSpider.start_urls = [server.url('/')] yield runner.crawl(TestSpider) - for file_uri in my_bucket.objects.filter(Prefix=prefix): - content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) - if not content and not items: - break - content = json.loads(content.decode('utf-8')) - expected_batch, items = items[:batch_size], items[batch_size:] - self.assertEqual(expected_batch, content) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1) + for stub in CustomS3FeedStorage.stubs[:-1]: + stub.assert_no_pending_responses() class FeedExportInitTest(unittest.TestCase): From 35726da434a9bc61dcb1ad5ef475c7d030023c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 22:55:25 +0200 Subject: [PATCH 081/127] tests/test_feedexport.py: remove unused import --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ac0b3ccaa..18f7f8458 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,7 +13,7 @@ from logging import getLogger from pathlib import Path from string import ascii_letters, digits from unittest import mock -from urllib.parse import urljoin, urlparse, quote +from urllib.parse import urljoin, quote from urllib.request import pathname2url import lxml.etree From c3b740f07814107e643bb0d073ea116049e37cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:25:37 +0200 Subject: [PATCH 082/127] Use mocking for tests/test_pipeline_files.py::TestS3FilesStore::test_persist --- scrapy/utils/test.py | 9 ---- tests/test_pipeline_files.py | 100 +++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 94d0ae2d3..cf251442f 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest from scrapy.utils.boto import is_botocore_available -def assert_aws_environ(): - """Asserts the current environment is suitable for running AWS testsi. - Raises SkipTest with the reason if it's not. - """ - skip_if_no_boto() - if 'AWS_ACCESS_KEY_ID' not in os.environ: - raise SkipTest("AWS keys not found") - - def assert_gcs_environ(): if 'GCS_PROJECT_ID' not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index d5b0bb3d8..1b14e3b1e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import os import random import time +from datetime import datetime from io import BytesIO from shutil import rmtree from tempfile import mkdtemp @@ -23,11 +24,11 @@ from scrapy.pipelines.files import ( ) from scrapy.settings import Settings from scrapy.utils.test import ( - assert_aws_environ, assert_gcs_environ, get_ftp_content_and_delete, get_gcs_content_and_delete, get_s3_content_and_delete, + skip_if_no_boto, ) @@ -414,32 +415,87 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): class TestS3FilesStore(unittest.TestCase): + @defer.inlineCallbacks def test_persist(self): - assert_aws_environ() - uri = os.environ.get('S3_TEST_FILE_URI') - if not uri: - raise unittest.SkipTest("No S3 URI available for testing") - data = b"TestS3FilesStore: \xe2\x98\x83" - buf = BytesIO(data) + skip_if_no_boto() + + bucket = 'mybucket' + key = 'export.csv' + uri = f's3://{bucket}/{key}' + buffer = mock.MagicMock() meta = {'foo': 'bar'} path = '' + content_type = 'image/png' + store = S3FilesStore(uri) - yield store.persist_file( - path, buf, info=None, meta=meta, - headers={'Content-Type': 'image/png'}) - s = yield store.stat_file(path, info=None) - self.assertIn('last_modified', s) - self.assertIn('checksum', s) - self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8') - u = urlparse(uri) - content, key = get_s3_content_and_delete( - u.hostname, u.path[1:], with_key=True) - self.assertEqual(content, data) - self.assertEqual(key['Metadata'], {'foo': 'bar'}) - self.assertEqual( - key['CacheControl'], S3FilesStore.HEADERS['Cache-Control']) - self.assertEqual(key['ContentType'], 'image/png') + from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: + stub.add_response( + 'put_object', + expected_params={ + 'ACL': S3FilesStore.POLICY, + 'Body': buffer, + 'Bucket': bucket, + 'CacheControl': S3FilesStore.HEADERS['Cache-Control'], + 'ContentType': content_type, + 'Key': key, + 'Metadata': meta, + }, + service_response={}, + ) + + yield store.persist_file( + path, + buffer, + info=None, + meta=meta, + headers={'Content-Type': content_type}, + ) + + stub.assert_no_pending_responses() + self.assertEqual( + buffer.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + ] + ) + + @defer.inlineCallbacks + def test_stat(self): + skip_if_no_boto() + + bucket = 'mybucket' + key = 'export.csv' + uri = f's3://{bucket}/{key}' + checksum = '3187896a9657a28163abb31667df64c8' + + store = S3FilesStore(uri) + from botocore.stub import Stubber + with Stubber(store.s3_client) as stub: + stub.add_response( + 'head_object', + expected_params={ + 'Bucket': bucket, + 'Key': key, + }, + service_response={ + 'ETag': f'"{checksum}"', + 'LastModified': datetime(2019, 12, 1), + }, + ) + + file_stats = yield store.stat_file('', info=None) + self.assertEqual( + file_stats, + { + 'checksum': checksum, + 'last_modified': 1575154800, + }, + ) + + stub.assert_no_pending_responses() class TestGCSFilesStore(unittest.TestCase): From 8f46e845190b7affb7a02c6842b465f0c5c3b76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:28:16 +0200 Subject: [PATCH 083/127] Fix style issues --- scrapy/utils/test.py | 12 ------------ tests/test_feedexport.py | 3 +-- tests/test_pipeline_files.py | 1 - 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index cf251442f..24c38283a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -23,18 +23,6 @@ def skip_if_no_boto(): raise SkipTest('missing botocore library') -def get_s3_content_and_delete(bucket, path, with_key=False): - """ Get content from s3 key, and delete key afterwards. - """ - import botocore.session - session = botocore.session.get_session() - client = session.create_client('s3') - key = client.get_object(Bucket=bucket, Key=path) - content = key['Body'].read() - client.delete_object(Bucket=bucket, Key=path) - return (content, key) if with_key else content - - def get_gcs_content_and_delete(bucket, path): from google.cloud import storage client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 18f7f8458..3ed35bec5 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -41,7 +41,6 @@ from scrapy.extensions.feedexport import ( from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import ( - get_s3_content_and_delete, get_crawler, mock_google_cloud_storage, skip_if_no_boto, @@ -1577,7 +1576,7 @@ class BatchDeliveriesTest(FeedExportTestBase): TestSpider.start_urls = [server.url('/')] yield runner.crawl(TestSpider) - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)+1) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 1b14e3b1e..e840298d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -27,7 +27,6 @@ from scrapy.utils.test import ( assert_gcs_environ, get_ftp_content_and_delete, get_gcs_content_and_delete, - get_s3_content_and_delete, skip_if_no_boto, ) From 07c1d9c25b6f499fbe2cc0f0139d78cf6c1d204a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Sep 2020 23:32:55 +0200 Subject: [PATCH 084/127] Remove boto3 as a dependency for tests --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 12e40295c..ea356c56a 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras - boto3>=1.13.0 botocore>=1.4.87 Pillow>=4.0.0 passenv = From 6ef7c44061f74281e7a977c9b1de4892cf84caf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Sep 2020 12:45:21 +0200 Subject: [PATCH 085/127] Fix timezone test issue --- tests/test_pipeline_files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e840298d9..4e1b90787 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -469,6 +469,7 @@ class TestS3FilesStore(unittest.TestCase): key = 'export.csv' uri = f's3://{bucket}/{key}' checksum = '3187896a9657a28163abb31667df64c8' + last_modified = datetime(2019, 12, 1) store = S3FilesStore(uri) from botocore.stub import Stubber @@ -481,7 +482,7 @@ class TestS3FilesStore(unittest.TestCase): }, service_response={ 'ETag': f'"{checksum}"', - 'LastModified': datetime(2019, 12, 1), + 'LastModified': last_modified, }, ) @@ -490,7 +491,7 @@ class TestS3FilesStore(unittest.TestCase): file_stats, { 'checksum': checksum, - 'last_modified': 1575154800, + 'last_modified': last_modified.timestamp(), }, ) From eff96038c7a488860c1c58c2c7e37f888264c3dc Mon Sep 17 00:00:00 2001 From: madeny <7504281+madeny@users.noreply.github.com> Date: Sat, 26 Sep 2020 22:50:38 +0200 Subject: [PATCH 086/127] Correct some typos This won't be an issue if **your** spider doesn't rely on cookies. --- docs/topics/jobs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 58601824a..d855d0133 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -65,7 +65,7 @@ Cookies expiration ------------------ Cookies may expire. So, if you don't resume your spider quickly the requests -scheduled may no longer work. This won't be an issue if you spider doesn't rely +scheduled may no longer work. This won't be an issue if your spider doesn't rely on cookies. From 894b509d7a4262cfc7548d78e9ff4831c6551c34 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 29 Sep 2020 23:37:28 -0300 Subject: [PATCH 087/127] Crawl rule: remove deprecated code Remove the compatibility layer that handles 'process_request' methods that do not receive a 'response' parameter --- scrapy/spiders/crawl.py | 34 ++++++++++--------------------- tests/test_spider.py | 45 +++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index bc4551a54..1dcf2e6ab 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst """ import copy -import warnings from typing import Sequence -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider -from scrapy.utils.python import get_func_args from scrapy.utils.spider import iterate_spider_output @@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor() class Rule: - def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, - process_links=None, process_request=None, errback=None): + def __init__( + self, + link_extractor=None, + callback=None, + cb_kwargs=None, + follow=None, + process_links=None, + process_request=None, + errback=None, + ): self.link_extractor = link_extractor or _default_link_extractor self.callback = callback self.errback = errback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links or _identity self.process_request = process_request or _identity_process_request - self.process_request_argcount = None self.follow = follow if follow is not None else not callback def _compile(self, spider): @@ -53,22 +57,6 @@ class Rule: self.errback = _get_method(self.errback, spider) self.process_links = _get_method(self.process_links, spider) self.process_request = _get_method(self.process_request, spider) - self.process_request_argcount = len(get_func_args(self.process_request)) - if self.process_request_argcount == 1: - warnings.warn( - "Rule.process_request should accept two arguments " - "(request, response), accepting only one is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - def _process_request(self, request, response): - """ - Wrapper around the request processing function to maintain backward - compatibility with functions that do not take a Response object - """ - args = [request] if self.process_request_argcount == 1 else [request, response] - return self.process_request(*args) class CrawlSpider(Spider): @@ -111,7 +99,7 @@ class CrawlSpider(Spider): for link in rule.process_links(links): seen.add(link) request = self._build_request(rule_index, link) - yield rule._process_request(request, response) + yield rule.process_request(request, response) def _callback(self, response): rule = self._rules[response.meta['rule']] diff --git a/tests/test_spider.py b/tests/test_spider.py index 78157a9b9..d23543f6a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,8 +1,8 @@ import gzip import inspect -from unittest import mock import warnings from io import BytesIO +from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest @@ -20,7 +20,6 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.linkextractors import LinkExtractor -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.test import get_crawler @@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) - def process_request_change_domain(request): + def process_request_change_domain(request, response): return request.replace(url=request.url.replace('.org', '.com')) class _CrawlSpider(self.spider_class): @@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request=process_request_change_domain), ) - with warnings.catch_warnings(record=True) as cw: - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.com/somepage/item/12.html', - 'http://example.com/about.html', - 'http://example.com/nofollow.html']) - self.assertEqual(len(cw), 1) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) def test_process_request_with_response(self): @@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request='process_request_upper'), ) - def process_request_upper(self, request): + def process_request_upper(self, request, response): return request.replace(url=request.url.upper()) - with warnings.catch_warnings(record=True) as cw: - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) - self.assertEqual(len(cw), 1) - self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) def test_process_request_instance_method_with_response(self): From 9661a8dcfc79249e8e3e117eb70682ebc8e57a92 Mon Sep 17 00:00:00 2001 From: Sashreek Shankar <45600974+sashreek1@users.noreply.github.com> Date: Thu, 1 Oct 2020 06:46:12 +0530 Subject: [PATCH 088/127] removed datatype specification for *args & ** kwargs --- scrapy/crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4c6b0e496..578016536 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -180,9 +180,9 @@ class CrawlerRunner: :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, :class:`~scrapy.spiders.Spider` subclass or string - :param list args: arguments to initialize the spider + :param args: arguments to initialize the spider - :param dict kwargs: keyword arguments to initialize the spider + :param kwargs: keyword arguments to initialize the spider """ if isinstance(crawler_or_spidercls, Spider): raise ValueError( From 744f352d09a9ec519c84ec5243e62eef78c66f08 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Oct 2020 14:52:23 -0300 Subject: [PATCH 089/127] Do not process cookies from headers --- scrapy/downloadermiddlewares/cookies.py | 41 ++++++---------------- tests/test_downloadermiddleware_cookies.py | 5 +++ 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 87f8152a4..d95ed3d38 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -97,35 +97,14 @@ class CookiesMiddleware: def _get_request_cookies(self, jar, request): """ - Extract cookies from a Request. Values from the `Request.cookies` attribute - take precedence over values from the `Cookie` request header. + Extract cookies from the Request.cookies attribute """ - def get_cookies_from_header(jar, request): - cookie_header = request.headers.get("Cookie") - if not cookie_header: - return [] - cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";")) - cookie_list_unicode = [] - for cookie_bytes in cookie_gen_bytes: - try: - cookie_unicode = cookie_bytes.decode("utf8") - except UnicodeDecodeError: - logger.warning("Non UTF-8 encoded cookie found in request %s: %s", - request, cookie_bytes) - cookie_unicode = cookie_bytes.decode("latin1", errors="replace") - cookie_list_unicode.append(cookie_unicode) - response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode}) - return jar.make_cookies(response, request) - - def get_cookies_from_attribute(jar, request): - if not request.cookies: - return [] - elif isinstance(request.cookies, dict): - cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) - else: - cookies = request.cookies - formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) - response = Response(request.url, headers={"Set-Cookie": formatted}) - return jar.make_cookies(response, request) - - return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request) + if not request.cookies: + return [] + elif isinstance(request.cookies, dict): + cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + else: + cookies = request.cookies + formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) + response = Response(request.url, headers={"Set-Cookie": formatted}) + return jar.make_cookies(response, request) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index a3de307ee..aff8542e9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -2,6 +2,8 @@ import logging from testfixtures import LogCapture from unittest import TestCase +import pytest + from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.exceptions import NotConfigured @@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase): self.assertIn('Cookie', request.headers) self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) @@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_header(self): # keep only cookies from 'Cookie' request header req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) @@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) From 44f0fde9057256dc16f24f22cfebc61626a94450 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 1 Oct 2020 23:21:09 +0500 Subject: [PATCH 090/127] Simplify TLS logging for the modern pyOpenSSL. (#4822) --- scrapy/core/downloader/tls.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index d9f3750d5..b5c6cc895 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -55,18 +55,11 @@ class ScrapyClientTLSOptions(ClientTLSOptions): set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: - if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 - if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 - logger.debug('SSL connection to %s using protocol %s, cipher %s', - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) - else: - logger.debug('SSL connection to %s using cipher %s', - self._hostnameASCII, - connection.get_cipher_name(), - ) + logger.debug('SSL connection to %s using protocol %s, cipher %s', + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) server_cert = connection.get_peer_certificate() logger.debug('SSL connection certificate: issuer "%s", subject "%s"', x509name_to_string(server_cert.get_issuer()), From e42d82526a1a519ff1305b189acde064a40fbd8d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 1 Oct 2020 23:22:17 +0500 Subject: [PATCH 091/127] Drop the conditional code for old Twisted (#4820) --- scrapy/core/downloader/tls.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index b5c6cc895..2b8990b75 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError from twisted.internet.ssl import AcceptableCiphers -from scrapy import twisted_version from scrapy.utils.ssl import x509name_to_string, get_temp_key_info @@ -28,13 +27,6 @@ openssl_methods = { } -if twisted_version < (17, 0, 0): - from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name -else: - def set_tlsext_host_name(connection, hostNameBytes): - connection.set_tlsext_host_name(hostNameBytes) - - class ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors @@ -52,7 +44,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL.SSL_CB_HANDSHAKE_START: - set_tlsext_host_name(connection, self._hostnameBytes) + connection.set_tlsext_host_name(self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: if self.verbose_logging: logger.debug('SSL connection to %s using protocol %s, cipher %s', From f47b120e2b67be7821a06ff77786c0bab2cfece8 Mon Sep 17 00:00:00 2001 From: Habeeb Shopeju Date: Thu, 1 Oct 2020 19:50:11 +0100 Subject: [PATCH 092/127] Documentation of link extractor usage (#4775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added description when using link extractor outside crawlspiders and created reference documentation for scrapy.link.Link class * Added link.rst to toctree * Corrected spelling errors, moved docs to Link doctstring to use autoclass * Moved link docs to link_extractors * Update docs/topics/link-extractors.rst Co-authored-by: Adrián Chaves * Update link.py Improvements to URL description * Update link.py * Update link.py Fixed line length Flake issue * Update link.py Fixed trailing whitespace Co-authored-by: Adrián Chaves --- docs/index.rst | 1 - docs/topics/link-extractors.rst | 21 ++++++++++++++++++--- scrapy/link.py | 17 ++++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 11aa5c9be..da264fb34 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,6 @@ Basic concepts topics/settings topics/exceptions - :doc:`topics/commands` Learn about the command-line tool used to manage your Scrapy project. diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index ed32411b0..e12ad45e0 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -10,12 +10,19 @@ The ``__init__`` method of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links ` returns a -list of matching :class:`scrapy.link.Link` objects from a +list of matching :class:`~scrapy.link.Link` objects from a :class:`~scrapy.http.Response` object. Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders -through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link -extractors in regular spiders. +through a set of :class:`~scrapy.spiders.Rule` objects. + +You can also use link extractors in regular spiders. For example, you can instantiate +:class:`LinkExtractor ` into a class +variable in your spider, and use it from your spider callbacks:: + + def parse(self, response): + for link in self.link_extractor.extract_links(response): + yield Request(link.url, callback=self.parse) .. _topics-link-extractors-ref: @@ -145,4 +152,12 @@ LxmlLinkExtractor .. automethod:: extract_links +Link +---- + +.. module:: scrapy.link + :synopsis: Link from link extractors + +.. autoclass:: Link + .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/scrapy/link.py b/scrapy/link.py index 684735f6e..e70667361 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst class Link: - """Link objects represent an extracted link by the LinkExtractor.""" + """Link objects represent an extracted link by the LinkExtractor. + + Using the anchor tag sample below to illustrate the parameters:: + + Dont follow this one + + :param url: the absolute url being linked to in the anchor tag. + From the sample, this is ``https://example.com/nofollow.html``. + + :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``. + + :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``. + + :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute + of the anchor tag. + """ __slots__ = ['url', 'text', 'fragment', 'nofollow'] From 159e2b2e2fbbcbb2b083a79e83e37cf4e60ee5ee Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Fri, 2 Oct 2020 00:23:08 +0530 Subject: [PATCH 093/127] allowing to run .pyw files (#4646) * allow .pyw in scrapy/commands/runspider.py * aesthetics * added tests for '.pyw' * created class for testing .pyw files * name=None parameter in get_log * small fix * .pyw tests for non-windows * used @skipIf for tests * two more tests skipped --- scrapy/commands/runspider.py | 2 +- tests/test_commands.py | 91 ++++++++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index aedd8c2ce..b957c29fb 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -11,7 +11,7 @@ def _import_file(filepath): abspath = os.path.abspath(filepath) dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) - if fext != '.py': + if fext not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path diff --git a/tests/test_commands.py b/tests/test_commands.py index 2899e5f24..3e54a0948 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -496,6 +496,8 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): + spider_filename = 'myspider.py' + debug_log_spider = """ import scrapy @@ -507,11 +509,23 @@ class MySpider(scrapy.Spider): return [] """ + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + def start_requests(self): + raise Exception("oops!") + """ + @contextmanager - def _create_file(self, content, name): + def _create_file(self, content, name=None): tmpdir = self.mktemp() os.mkdir(tmpdir) - fname = abspath(join(tmpdir, name)) + if name: + fname = abspath(join(tmpdir, name)) + else: + fname = abspath(join(tmpdir, self.spider_filename)) with open(fname, 'w') as f: f.write(content) try: @@ -519,12 +533,12 @@ class MySpider(scrapy.Spider): finally: rmtree(tmpdir) - def runspider(self, code, name='myspider.py', args=()): + def runspider(self, code, name=None, args=()): with self._create_file(code, name) as fname: return self.proc('runspider', fname, *args) - def get_log(self, code, name='myspider.py', args=()): - p, stdout, stderr = self.runspider(code, name=name, args=args) + def get_log(self, code, name=None, args=()): + p, stdout, stderr = self.runspider(code, name, args=args) return stderr def test_runspider(self): @@ -556,7 +570,7 @@ class MySpider(scrapy.Spider): # which is intended, # but this should not be because of DNS lookup error # assumption: localhost will resolve in all cases (true?) - log = self.get_log(""" + dnscache_spider = """ import scrapy class MySpider(scrapy.Spider): @@ -565,23 +579,20 @@ class MySpider(scrapy.Spider): def parse(self, response): return {'test': 'value'} -""", - args=('-s', 'DNSCACHE_ENABLED=False')) - print(log) +""" + log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False')) self.assertNotIn("DNSLookupError", log) self.assertIn("INFO: Spider opened", log) def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=1')) - print(log1) self.assertIn("[myspider] DEBUG: It Works!", log1) self.assertIn("[scrapy]", log1) self.assertNotIn("[scrapy.core.engine]", log1) log2 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=0')) - print(log2) self.assertIn("[myspider] DEBUG: It Works!", log2) self.assertNotIn("[scrapy]", log2) self.assertIn("[scrapy.core.engine]", log2) @@ -599,15 +610,7 @@ class MySpider(scrapy.Spider): self.assertIn('Unable to load', log) def test_start_requests_errors(self): - log = self.get_log(""" -import scrapy - -class BadSpider(scrapy.Spider): - name = "bad" - def start_requests(self): - raise Exception("oops!") - """, name="badspider.py") - print(log) + log = self.get_log(self.badspider, name='badspider.py') self.assertIn("start_requests", log) self.assertIn("badspider.py", log) @@ -696,6 +699,54 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +class WindowsRunSpiderCommandTest(RunSpiderCommandTest): + + spider_filename = 'myspider.pyw' + + def setUp(self): + super(WindowsRunSpiderCommandTest, self).setUp() + + def test_start_requests_errors(self): + log = self.get_log(self.badspider, name='badspider.pyw') + self.assertIn("start_requests", log) + self.assertIn("badspider.pyw", log) + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_run_good_spider(self): + super().test_run_good_spider() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider(self): + super().test_runspider() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_dnscache_disabled(self): + super().test_runspider_dnscache_disabled() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_log_level(self): + super().test_runspider_log_level() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_log_short_names(self): + super().test_runspider_log_short_names() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_runspider_no_spider_found(self): + super().test_runspider_no_spider_found() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_output(self): + super().test_output() + + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") + def test_overwrite_output(self): + super().test_overwrite_output() + + def test_runspider_unable_to_load(self): + raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") + + class BenchCommandTest(CommandTest): def test_run(self): From 797a6690c07ae90f7a2d703832e2fe44466d656f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 1 Oct 2020 23:11:11 -0300 Subject: [PATCH 094/127] Tests: use classes instead of paths in settings (#4817) --- tests/test_dupefilters.py | 10 +-- tests/test_logformatter.py | 4 +- tests/test_pipelines.py | 2 +- tests/test_request_attribute_binding.py | 14 ++-- tests/test_request_cb_kwargs.py | 4 +- tests/test_spidermiddleware_output_chain.py | 92 ++++++++++----------- 6 files changed, 63 insertions(+), 63 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 95a4fca0d..680bb6dc8 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} + settings = {'DUPEFILTER_CLASS': DirectDupeFilter} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) @@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index dc5be398f..6381f895b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): self.base_settings = { 'LOG_LEVEL': 'DEBUG', 'ITEM_PIPELINES': { - __name__ + '.DropSomeItemsPipeline': 300, + DropSomeItemsPipeline: 300, }, } @@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): @defer.inlineCallbacks def test_skip_messages(self): settings = self.base_settings.copy() - settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter' + settings['LOG_FORMATTER'] = SkipMessagesLogFormatter crawler = CrawlerRunner(settings).create_crawler(ItemSpider) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index c72f1a338..ff3af9a74 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase): def _create_crawler(self, pipeline_class): settings = { - 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1}, + 'ITEM_PIPELINES': {pipeline_class: 1}, } crawler = get_crawler(ItemSpider, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 907117468..00c532c41 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, + RaiseExceptionRequestMiddleware: 590, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -119,7 +119,7 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".ProcessResponseMiddleware": 595, + ProcessResponseMiddleware: 595, } }) crawler = runner.create_crawler(SingleRequestSpider) @@ -149,8 +149,8 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, - __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + RaiseExceptionRequestMiddleware: 590, + CatchExceptionOverrideRequestMiddleware: 595, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -170,8 +170,8 @@ class CrawlTestCase(TestCase): url = self.mockserver.url("/status?n=200") runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".RaiseExceptionRequestMiddleware": 590, - __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + RaiseExceptionRequestMiddleware: 590, + CatchExceptionDoNotOverrideRequestMiddleware: 595, }, }) crawler = runner.create_crawler(SingleRequestSpider) @@ -188,7 +188,7 @@ class CrawlTestCase(TestCase): """ runner = CrawlerRunner(settings={ "DOWNLOADER_MIDDLEWARES": { - __name__ + ".AlternativeCallbacksMiddleware": 595, + AlternativeCallbacksMiddleware: 595, } }) crawler = runner.create_crawler(AlternativeCallbacksSpider) diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index bd49179aa..145a4e9b2 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider): name = 'kwargs' custom_settings = { 'DOWNLOADER_MIDDLEWARES': { - __name__ + '.InjectArgumentsDownloaderMiddleware': 750, + InjectArgumentsDownloaderMiddleware: 750, }, 'SPIDER_MIDDLEWARES': { - __name__ + '.InjectArgumentsSpiderMiddleware': 750, + InjectArgumentsSpiderMiddleware: 750, }, } diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 2f454addc..029bf8bd6 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -16,11 +16,20 @@ class LogExceptionMiddleware: # ================================================================================ # (0) recover from an exception on a spider callback +class RecoveryMiddleware: + def process_spider_exception(self, response, exception, spider): + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + class RecoverySpider(Spider): name = 'RecoverySpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.RecoveryMiddleware': 10, + RecoveryMiddleware: 10, }, } @@ -34,15 +43,6 @@ class RecoverySpider(Spider): raise TabError() -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: @@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider): custom_settings = { 'SPIDER_MIDDLEWARES': { # spider - __name__ + '.LogExceptionMiddleware': 10, - __name__ + '.FailProcessSpiderInputMiddleware': 8, - __name__ + '.LogExceptionMiddleware': 6, + FailProcessSpiderInputMiddleware: 8, + LogExceptionMiddleware: 6, # engine } } @@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, + LogExceptionMiddleware: 10, }, } @@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider) name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 100000, + LogExceptionMiddleware: 100000, }, } @@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider): name = 'NotGeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, + LogExceptionMiddleware: 10, }, } @@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 100000, + LogExceptionMiddleware: 100000, }, } # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) -class GeneratorOutputChainSpider(Spider): - name = 'GeneratorOutputChainSpider' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.GeneratorFailMiddleware': 10, - __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8, - __name__ + '.GeneratorRecoverMiddleware': 5, - __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3, - }, - } - - def start_requests(self): - yield Request(self.mockserver.url('/status?n=200')) - - def parse(self, response): - yield {'processed': ['parse-first-item']} - yield {'processed': ['parse-second-item']} - - class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: @@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass -# ================================================================================ -# (5) exceptions from a middleware process_spider_output method (not generator) -class NotGeneratorOutputChainSpider(Spider): - name = 'NotGeneratorOutputChainSpider' +class GeneratorOutputChainSpider(Spider): + name = 'GeneratorOutputChainSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.NotGeneratorFailMiddleware': 10, - __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8, - __name__ + '.NotGeneratorRecoverMiddleware': 5, - __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3, + GeneratorFailMiddleware: 10, + GeneratorDoNothingAfterFailureMiddleware: 8, + GeneratorRecoverMiddleware: 5, + GeneratorDoNothingAfterRecoveryMiddleware: 3, }, } def start_requests(self): - return [Request(self.mockserver.url('/status?n=200'))] + yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + yield {'processed': ['parse-first-item']} + yield {'processed': ['parse-second-item']} +# ================================================================================ +# (5) exceptions from a middleware process_spider_output method (not generator) + class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] @@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew pass +class NotGeneratorOutputChainSpider(Spider): + name = 'NotGeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + NotGeneratorFailMiddleware: 10, + NotGeneratorDoNothingAfterFailureMiddleware: 8, + NotGeneratorRecoverMiddleware: 5, + NotGeneratorDoNothingAfterRecoveryMiddleware: 3, + }, + } + + def start_requests(self): + return [Request(self.mockserver.url('/status?n=200'))] + + def parse(self, response): + return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod From 004b40a7193a7c0c5138abe764cad6d1d9a4fc57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 3 Oct 2020 00:53:55 +0200 Subject: [PATCH 095/127] =?UTF-8?q?as=20soon=20as=20=E2=86=92=20as=20long?= =?UTF-8?q?=20as=20(#4825)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 675f55c38..4d2580a6c 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -140,7 +140,7 @@ original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as the original author is acknowledged by keeping +not considered rude as long as the original author is acknowledged by keeping his/her commits. You can pull an existing pull request to a local branch From 892dd9da57c5cf005670743c8abfdfffc7027acc Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:00:58 +0100 Subject: [PATCH 096/127] Adding support for zstd in HttpCompressionMiddleware --- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b1abb8b5c 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -14,6 +14,12 @@ try: except ImportError: pass +try: + import zstd + ACCEPTED_ENCODINGS.append(b'zstd') +except ImportError: + pass + class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be @@ -67,4 +73,6 @@ class HttpCompressionMiddleware: body = zlib.decompress(body, -15) if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) + if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: + body = zstd.decompress(body) return body From c6c3f2ce661ef9296302b9d489d5a2cb2e49f481 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:10:40 +0100 Subject: [PATCH 097/127] Updating the doc entry for the HTTP compress downloader middleware on zstd --- docs/topics/downloader-middleware.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..9645ed5fd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -684,11 +684,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstd: https://pypi.org/project/zstd/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From da3171d4f71d50a15b08b76b2b1e06bddfa56694 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:18:58 +0100 Subject: [PATCH 098/127] Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info See also: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53 --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9645ed5fd..7c63a623d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -685,13 +685,13 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstd: https://pypi.org/project/zstd/ +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b1abb8b5c..56421a6ba 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,4 +1,5 @@ import zlib +import io from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse @@ -15,7 +16,7 @@ except ImportError: pass try: - import zstd + import zstandard ACCEPTED_ENCODINGS.append(b'zstd') except ImportError: pass @@ -74,5 +75,8 @@ class HttpCompressionMiddleware: if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: - body = zstd.decompress(body) + # Using its streaming API since its simple API could handle only cases + # where there is content size data embedded in the frame + reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) + body = reader.read() return body From 50e1f35d1fcaeb2e0a0c4fb29894cb7c686a33cd Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:43:12 +0100 Subject: [PATCH 099/127] Adding test cases for the zstd content encoding --- tests/requirements-py3.txt | 3 +- .../html-zstd-static-content-size.bin | Bin 0 -> 8066 bytes .../html-zstd-static-no-content-size.bin | Bin 0 -> 8063 bytes .../html-zstd-streaming-no-content-size.bin | Bin 0 -> 8047 bytes ...st_downloadermiddleware_httpcompression.py | 26 ++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/sample_data/compressed/html-zstd-static-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-static-no-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..2eed2f5da 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -16,6 +16,7 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython -brotlipy +brotlipy # optional for HTTP compress downloader middleware tests +zstandard # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..b5c2038e893481e2835ac74bc790e3718986e426 GIT binary patch literal 8066 zcmV-|AAR5`wJ-goRU`obmHPmitfU$&;EqO3lFJAV16>@^C8(hY1ZsEGeDXVhj51gD z{wbwTMW0fDVIjsC8zUwJLIPd^Tmj940%8FN4lqZfU`(TE6bp>O3tV-7e>YaR^FV=)2Knk9h* zPD2Bxu`I2bEJ}oPC|O!VVVHKB!!*Tl8rtL0DmWw>I4vT=w8gSua2St>L(&)r1jK1r zB93OkEFwfW4Gr3a353%c%i_@@6d(g3YGzZS2 zoTW`Hi$=w=8$L-PHPy)01wDyvS31MEJ`~NhjNz41g9-ZJB>s+5DPeP;2ej=V*+6! z2AsyCEXzc~31Ax}lf`M!Fs)Hq1*RRwGC2*?lnG^m!iYdlH$+BRYv$Pg*Bn!uBmIj$fL>Nb+ zI3N}nq$wB$#lixsfDwsMEIcR{9u$&CQHqNO3cLt#DP1=;z9HvN0E*SB0B*i|V@? zo!0dgG3lazM?J5RBBIe@s@IB`{`D4B*O(MBjS}6Wt}`j>9cd;_G)lU*(nS1E>Agxb zjnrOTx<i{{ zuB!54rdz~ydTv6L_*KKIR4?Lcu5WP{s>|c5E+#Mj_g#pa>{ZT1y12Y}n{th}_Fg9A z1<pTU+@2?hd|hVob(Mx!|A9L6+?2gL$o znujl=Vp?bP`Io;tI|2zX28DP;Bu<1Tv0&gpczD1Z49GyhvY;TT5Mhqv;Q{e@YIOW5 zqWax`XR5Q;t%z4Y--^b(_2;8+t;tDU>hpQ0m`+OFBE|bpS6Zlk`l0deZdK^3BEqA> zRR2pCDs;Ng>RwcLUCNkuG%jD}5BD@tF&)x8I_jk&p7V)G(`gZ{6xZdtwNm_#i_Tl2 zqB?K&qnD;qh`TiXT=d>C?`4E=@0jSv^u2Ns{X||@`%Yx~m|nN3bgg-Nx_tMh zb2;^SUoY!f553Y~t6QVm_vRBZ^WMz=)_7=tdGooqRzH&-Za(+c=|tQ_UuU{hqXP@< z0cji$4~m6_Csd@4D6L2Ibahm`g!G*LBSHufrT@5&`E=gt9~CLC;}Jc7L|=%~6FWqh z)KQ{@tS;?4N|cF5C{yuIl!%f;qC}+9QL>bGMwkhqyiClabV{lyoz%Zjs#H$uh%&8( zGE+h*FQsBA^U|pZX{r@dh?4q3dQWF3rYIBo$(bpW`jX0&^eGck=d==2g+9fQD#cPN zrs*jaN-9yMJ4&e7Vo6!qdOH>KdRm#P*!oJDnd^{Z>vwA1tum=Ey;6m#)M8huSXwL< zyYE+pnzz`xLRIF)^nSQ}mAF#nVoIqle<}8*>gtp&UCI#3`$D>wuEmgpUg~u*WtaXF zV@FG=YEsvw?77xb#?qIJe2AxK>MKiD`*%9?=$NK5X1y|B^~%4xGPYPE{XHF$5%uVv zh$RPg#J-Nu4u_8E88LN0hKV6`NVMXQA;#D(2W5Cv4reS8VvI4C3^^pk7z44y7-P{K z3Za>!Q7F?WBrt{yiq@z^(6q(i%pd>(0YCr&0Mioy02mAmh>Sx5DrwgN6o3I56b29! zEDRI~0t7&C;P4u`C=>t&LSZl%C>~Ta1x^BJ0QDms#I{8t+xDvw^)H%OmM^UtD6u=U zJi2g;L;Z>qIeYC%x77AvCUM0~2dZsvqsQWjcw9QI^3oQ^-Ny=pBn5wXbuZyGOE-C_ zU+(88f@qO{Z!@13Ld9B{bEg_PV51mS_}EE@qw|Xtd&KjUfOP~dg4w^u)KKYBv)CAw zot38EujidM6)6_Q0emIE5v2Dd^J_S#jAd|478Ri)!r?XqW)yq}yGjGyFZpbke@%9A z_^u{F`#3YR{645{L-~!AtK$yGb6vk!62X2+S8;$e6dz#?0lkbchOLSGA;3MZNDi7_ zDZ`<`7%Pq~Ov@c1`J>Q+#XEwZ=En{ju0tq)SgwM`F;(vui(^FjAxj@&3Tt+_cON8; zz{D;>Q?B)}jJ^@EIH)ri__uIYjQ$Y@q47ge6H$0uQ(bqJX;zw;fYKs74qQlWnM$zn zt42$TaFlT17RDKQ`q8l82lUt`FP*I7d0CAn3p5-cre}g%ko2+aBjS01-TT!%Qrs`2 zL~I4ZXS$tk!Q*FY&%tSqr}IP3GN6x2d0K&-ZUNiTB0Ivy7Z|@j7m;$}>sJ?+$a1=&4s!@7FS6X}iQx6KvgDT?Bz{7oilbL9z-4ct_rAsPzI!Y^_3E}j__m*@@3lVL2ujan z-u(=F9dJ)WDyvlfO1+zK;~Mmg=xq%#`vU6msw@-M!y$M-#>{el>f_pp|83onyRevw z@EiId#6snzKtlyy;h30OWXlx=z~ z8}Cu#3gc># zIsI;^khN5tfp}$v>?3qU-7kD0p8a@sT=af|0#_@H>GQR&&9K?&p^xx*zgWVa9Vz4P z4;Zh}^^L+z-surWhn#H`tOz}3p7HvZ9Kw{D2R?l=7)&}Gyqjjo#btC!7Syv>l%_4S zLwFzbs!lxJBv+9Xhv{~*+wa~&-|>*CZ`3;@nD5bjTf>=$P3rs}kA#>Wz7~BbuZJv8 zndrY)4{tW7znl}>9`ep3Kh#XT?jiGkI#$4%dBoD`P5%KpD0O2=r;|Mb<&a&A2?8>sh%T{-p_2KWE-H%9O1BV(V`{m17l>qV~z=?BrOh;_ok7@rd2|~ z`IqZ~j#yB*?T5UQcoG{c>rM7YkW2>8;YPtzq;4M3q*%5%@@zBzElOkE`!xUm@V@_h zF-3O)mbFwQ7h|5XRm0Ma9d?>`gzZKP8)duDY^I3gof79a|Lels+-KtuK4VlhC3Ih0 zS@X39#s>(w_aSJpT9w=&J}k{$ads~zVPbbbhdHgGLH>|G%lp-{0F#l;doyZWe3rMJ*W&^`p7joRO!v022)e(S+p z#yehuj3s;f^${>C5j_r4M@v^RvCZ_?fQoR~@4}WtcGlRGBi|pxdi$syZ}exD$01+G z!M>kq1o{kDtgmOKHpPtot=pS5O4zuF=()lO#Q8LVERrYO{T8<%5qO!rCdcTd9Diy& zC!{vY%$8?enEN)b92Nl>-FXKbAROmvSiFp3vPTh4Zp%8Z=gHJ?K>4LnS(xX zb1J2eqR#`H`qoU?o?WSpX&zX!Zl6h?>1f9m)UeGc;iMW6RBN` zkl5>1k(4{f2WZS$^!@ zVpweBLB^Ep_DMvN#Cg|2ESpjZP&ZnkE5Kw{5h08s;?v_4zyDJ|M$~D7c0cAczQA*4 z7-2EajlyM1j2A#K9ukMp75cR6ap^JDiQXLOfGnP;iKT?t%&rN zW6;7Lzin2>bV9bWHd_5g$q_v)YP1k_D3ry?S7nHbVHw+=Vu%rQXzPTumK-5KS(A;& zI_u;DQK;-3XAyWu3o+}Y=g>xm!&@972xxn$eg8vt_ardqgyqTVVZNKWmp~shY`27o zq69i#6~P3nHkYnM3*&Hgg-N^nKa0yH`ZIH~mG|}O!Cn<_vP~wR2{kxG{GrA*RvY8K zNliJ&ZTjR9^!=>yVcRHcZv-zA@5v#vp zYI;Fe6~Qj#_X>8S-WlaX9v?%^mbl-kD|H4URprK77}T9wsfA$5#tsiB@w$k-t2hKB z;ckC-UZ+?VQq@0oR}SiGfG>2g(5`^|$T}f|n$P~Ac@h%RoBOO)QL0U-2!P%wips{d zsLED_O>Q@N_{5ZgmBp$Ed)<`1hO#8uDj?_B5&nnuv)yVNfeq``N6E(e$>H=yi3F5z zzy_?9I!OZLvNVpiytoK=mlJ$ulD1hq-oO!53_>++*WX4Vi1}7lS2-&}5bnJ|t~%?P zMTEb0rtS#zI2RWRHDwG}h3{y=&8>Qr>j)!3u*(hn3>26YL%s@o1my&q359knG`dqz zyk30)IO4i98PPUCphScEb9w~$piA#FaT3dUWv)o-pKBElLM2yeoAZMUpHI8uN>O^v zQO}LjuKd*pH2f1}na}!3u<@8A)Sw#kF4XQ89j;t)w(`+N4DWmn?d z2+i9@u~Dq`ZcPfB=IjZG?h=BI+}}+Lkz+7!EQf@RHL1(nSRA_-oF9*2U@q;UN@u$VVQ8KO9Bvr|F6nB#ZEtf8?B^8gnu^v z`WG&m zKEQJZfNxH;;~fY175Z-j&m#Hdqmg+R-GHQ&gD9o0HrWqwurD#R-ySmGv-=HI^c1gD zwYgO}Sti&zRVeh`-zlbaJxPTod6hmnHB5!=sR#GjjqI&I4d)OoY{jCiG%^9l`AFErDd9JHjyE;%@t zoSwtaPOn#wVgJ$`LE{=$fGi zEM+B+x6g!hLK!By3B{`z+qL$?*CYWGQ)=tpGL@Ll5^qV$>QEAfhw)z*nPM$S6o7Ih zPB}WUZ@V^N5TKb_H){!DoiWxV@iZfO7$#tj3H9|I10G4Lhw9o@v5pZbX^G0yBDE>E z|M1D4{8|9ZO#=&)kYbd&S{&&EXvJ17;={#0B+H$nB)%W7rnMddqR5PS43SNe;E;F9 zE+QMlb_G*0toS}$mRuj?y+v7epym)z7Z*RiJ!kDUyGx!0q#goF z1lU@}erQBm5)WJ5R6+dPE02?nDoLSL6FPT->dG~d1yVNUjUUQs69|>6qQVy4P6z8egty%z#uULZ&s2jW?Fp4iQKq^Wz~HyVK$F)P4V7DLN8Y9!+F`+s znMjga|7GOl_+X&4Yx4(;8>S(-{XLklu7{62fB4o9#sA7TODSq|iYeuGFw-i$ALJwN z?YCcejmWGw_gsJF8AQ>ZjFHOfvhE@DzBc6g;L7?@tUm03-g@0*e(pV7BHJZZakVL5 z3tSnbG5*j7S#h&Gcvoe0R%iolUJz&v@cCRHj!~e_TnU>%gmH^RUWYh7Vn#D;m~?tb zuU91Uf>ZwMkbrR1IxN3DxJybgk`Q>`iCW05&3=^63x1eOF14G3enY>+)Go5O#604MLGWJRHEUjnhpxgKg>cKb(@=Fvn|J%$(wPu(&hPAF+r8GnOx ztum~If0)stw2OxFuNI4bk5~l2m}^0?mriM`sW;69uY9h=Vb(Cf0*U2sy4{>uAw%B zmEQ2}+14&Cu-OiA%TFJYs#r*A z)qynN(DPUDsJJ9+gOoC+Aw|wc5S6{q50eQ*p^r`s1kvFnwbU|(kkUi$93rM%zaD9d zKM;gFhj z^O4=x(ZU?v!;*H1)LPX`2>=bGQEkTmJFYLS`d8S(-k}egdju$|jTk*FF*uV6U_@|@ zF67`#tjJ{MOP91@OW2{?20f)Y;;NuYLPh#Mog7UaG>(aKx2S@$}hMGhrsiA+qxmf%?3EFI|SAoj}a zXG$P0BuNp$3bkPn{;4;MdRR7kMEh=P=&L&u9D0*M-p~(25WEBby}_n^<(CAU57?P`b1-VsB6%%MEM$O4iO*V8pw;o%2mG0`0C^h>T2oiL&; z4R&bCN1-EepFzb5&1BdoF~aR!R0X6}B043_OI3o-I+&2q(r7uR4r0D>!=QmuFB_dc z#Cd;!C_GL`*pmI&4>q!5tu`kwjfe|zTvjZtTG1)=g;$!8TSusve#Lmrp?xxO6q0;2 ztc4FD_D7#paqN}5hyVE8I#)3PpWNT+=kcq#uruz8Pg&StbtUaE1Dv%t&A)%2qlZS?rpybhE#F9kknmP+7Fn%e(p)Bikzjs zcR?+z>z0)1fs*;ryQ|0mNP;L{`(hO2G*75pX;+M>D%f45d<|;iFVkU)W7BDdCoHDZMKHGZ}fDahIn9VA30Ww&x=UM=SKb$wa>@yLXOClaN6gNTHP z9DQv@Q1xLW;JQ&`4&{bmhJ4BgRg(**gUYT=D?zte@jkjMMa3q3vpvVv^M8SRBpUl2 zLpemb6)#?PfMRYM6i*kHj+U?%`$pV0?7{x{_$IY_oK@u zkRPiRQarhDiz4%x$WRVtGT*2`$(^|mp!j-<>F~%-hSAHL!e7lv79~ymqYD$oGd}~T z=R=M&7xyD#K)|>|OF%;A~C=c3?=8w2fzC4ql1WI<>&7| zi6DkQ?l1Z<9A<3{Ihn03Q1PPFrcD%B04SP1R7AY5;9mXqSsB}?!;SRcQB;W1l1+49 zCkOOzJ=c@$Gv`Yyw%v9W&-~bM6XTwaveGIflSPj82XY4+q$E4hO4k>PuzU$I5*T=* z0xBms9%iY1P9R$;^u2JzEj2V#T(%A=Tq*u^&kdP^-hMCmgHpsS%<+F6QqbD_d<^+2 zq&&gVmRy3$wWQod1t5diDF+o(>iFdl+UU@Vu{EE6gwipxWqEJS-4zp>)t+s z+CmgEpIf%5cVC5QEM7S;&F zpNMBaX`fV?!T{488eyRFVNiT%K0z2oY|(O+v~@Z_7nW}3Nx zmCsSsxh)?|OR~R|-qkyEAi>n?Lp$x2wsrgVwa~IJ2yx zg)qj0`!t6(&At=G4uAmQ0mNC-E+nFiNfqr0baw3p?4DvAIAwv&#u3OmDi(ND^by8T zEVKXBZi%IJWhNr%lMy*J+w zbV-M7DA?|fpWc_gx{Y5?yTCy_hgBu0ex(2j`f+xKbP1SEm7n*Pa1r{74DruGNsbVL zTw49`*DnkEFstbn-5>JUWa{LCm0L@QeZ1cOrfa?3T5F%eR~bs5QGO*Wckg~uFtwW| z`Cm6sdO>HzPn}}t7n9^DRpfn%JJn|rX!jD&(yLv~rmxBWEZ6n(3m>aN+TM0m-*t+9}@VeBC4QW82|tP literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d494192e2c72294810309eb14e8d55e0431fd1c GIT binary patch literal 8063 zcmV-_AAsN}wJ-eyIF~>I5=%-jHWmwButy&FfHONt+7lRiP9buC_okxG);TpGzz4dXqx1-M#8ky6bI)x zG!o@Bi=#bSV_BRAaV(Yzlf{Gykw(I_$)bVN9t(&9Iv#{dt=WU^pFYb;7T5QlP>$ONY?N;{22IS>mtaNrz=#bW|tA_kns zqAbfq!U z0Z>pT8WEhvqA(&7OFPZdnk6D}9F0=pFj~_n5XrJ&nzOVPaU=`JX_f|=NJJP%qBtNH z7^Eo}1;xSwtAG)SP%Jzs79JFmM^l!CgCl{&LV@FuSS)90k4KAGK(O{`ltyvR0%I(i zL$Safkj9}{U=SieAaEWq00Be+3pfBk5lC?}eXVd+jp*mfb+R!RpI3#OcZ=$~8lBel z6*1|ee@8v9ks_keVXD`PnEv$^Ro9plF^v-4qOLP3>K$n&O*Bfnw$eoWPwBl%GmX?< zT)IZaU7G1Ty}P|Ds&mn)dJ(@)Uf&P>t9*~i20*$Q%YQ4C$9CUiHr&rS8s1Z2=Nw=>mM!q3-u5#y01^w{Vn2J7d_#fl&-4s zVy0Wfb$V_>l=xNET+FY!#f5A2gvd#ca?g9Y{^6=zlrGb)N_)|ZU*|e^U00zla?+po zbn%`BmUF!3Brp)MJ`>)Lxrpm$UOgrLN2{wm zO=#WYU42p4$HQmpdVOX7w|RFf)0e0B+f>}kREczbr|ZAXg_vHqsdTM*d%Ar0rgJ&< zd0#K;Y*U z4-blkg(p;`jwr21^mKJpyoB_e{v$#N5vBjQj`?)n=^qs-uHz9se?(u1(i1yGnAB0C zgsd*@J4%#^M<`SAPn3v~L!v~a(owRMcSe{Ap}b7YqjXBDD4o>5P^wf;>WDI}g)&n@ zC@-aADD%>(2x+PnQ;3rKLV8bUC#EP9`pKCollqd%l=LYRQs=Z1Q-waokSfJeDyHcv z6-p{mr8`Qf*kVap*?Kz_^LkpDs@VEUnVIX5V(WKm-K{dIFTGNQsnlXusaRSp6}#_O zg_^h6x5n_xnmJB&0#25py#291I915YC zqfsc+C?qh342ssMM9{Rw;LIQZ0Rcb&0RYnz000;a42X20~#l7$_c8GzCrqXaMyi9mKXpA=~z=5%n*cSe7rX87Q$kvpl+R zi$nd26FGbBNw?JYU?y?JOb4oMZ==WJiFjN(t@6?q$KA&YgCqricy%w~G)p&ms9)~q zCxU2^e{VCN7DB~ZnRBNaIbfq0RruIRhoke06nn(;lz?>vErQv<#?(;hQM1?>m7SHQ z-mmAKHWeur#Q}UJz!9YPB=c)Hr;KHAOcoWPBEsP|1ZEU`2fIoG-7ooUn14-narmw# zLHjr}v;01&ZA1Bul&j+o$8%l3SQ5d0Nmp@zG!!3U4FSE3Fovy({2{*kTT@+km1$O*n1IqEJPuq)ZJA22@vBBl zig1*0;1?7iNf!+JnJW|{*qeN^4 z!e_djZo%VcYR|!GqdrGaSYM@6!tPi5=8+jQ0?X=S!<6n8>IG%IfoSVP)+iIyE}0OV zL#OjY&N85nN_kp=oNfWz(IPv-#upgBJ{OU4;_F=dwc&mburUsT9QqbC6?cSFjTzyv z%CB#Y|A}=Rs%mBg{+PlsvEy#xZkh2XM$evI462Lh$eyse1hw{~e0&T4wBd3kckv%C zS1zz9`v1S-@u}ZK{&Cu8R}Knq^WMH6f#C}R8wG)g%2(+@%U}(KxTyUy+*Z=FS4lk= zb?**ykLamaRPWa^VQIU>Q4?(4T3rNzZWo~)OT8Y7EjOc9*G5Z4Q@v1Bn*25uqKgP^*e(K}eiT`cgkh`#$itroy zAjCrDr9eXkUg4Oqx;okXoBE|%iMbMM{Xlr=qn1jw@3eagE5kwb^M54o3%Ll*<#YtN zQMdjR!bA{fw^~iqGWMq={2^f4#lrPLVzY^`^JKv9L2ZT+WsEZ%B7kGCc)wV}o*gOU?hhER z(e;hOOy21cMu(hj6s!n6W}fl-mmI>BnFl_7G8jxc9K4%m$i-!JNfy+zSCpnLvqN|v z^r}ug-6U6$6o=_{vfJ<8Lf`R_sc+OfBbe{eeOtqshfV7I9*=~W9=;ZRD6fYsPnqby zR}XJCr@x#N+aB`HBR|wkyzU|Me>zscnt8<1=}rFuIw*BxNvD%N0_BihiwOxw>&*HM z(|6#X19f8)w5gsWklxR1qhuSP%N*gbaM7YA>jPtCzGIFFrX(#6llP{P$EH<6!11Pl(WF?mIPz>W|1C;m-TO5E|M0&5doe|K z0hYB?Bo||zvQ@*nx%VMxv09bfAU-V0h+DrSXw@`38f+0-wfmhLvJu{9Rtu&PwHMsiQX(k{YV279 zGpilv6LfwEfi`e0J?vc}8KF?I7RAL9KfC&T*fjHh~*Yi&y_`gmbPb|{tW6rmU=6Bgs$z#H;T6iR2At_ zOt1@hhWMG?uW6nNn>@z$NBw&Ai0HY(2*mj`fh>|I-2E1}AQ5<(ye7x!r5t~1JSU_! z%FLE$U6}heuN)Qu7~OdX93UL$YFNCCVX{XNPHxLOuII_-Ij!pcDDe??PpG?5vGr}a z%6Q#2oPJ2($TE!))I$4I?lq@zMSP}6m1APoLPoUThI>>n6L|`=G3`4QESZBoaC0i9 zkEDBnbw<10xh*3#HD!-bL%TL((x-A7w=~G~UHL{yAqT%ZDMfkK;kXs1qB!tvQn_B4 zr2?47!_e{p$TPm30^E)#6F1mK9qv?GexK_G)^N&SX%D_MT*-C|g5 z<3Yxh>-I@RlEit}K`fh62~am$p)0^-RuLhLBI48I6uv8EZ)rsC5=zuJ)V)vC%bTO(_5x!GNZymYP+*p{5sW>zK zVlCJxp~bz*xO4QZmCBM}3W3}ik~XGxW=O```o=|_z#wTcC8_#{8?#HF?xr&SvEk80 zTVo0$=TBE>qP$>PFp7f|V|OvNA4*?w3ZwpKv|QG$2#le z0#T^!9A^=DM+-6Qr038^hQnJNAqZ%DseS)LcK0MO=Y-|S>S4Z{xtBm6G;FtoiJ}BL zUKPOvt2UReL<{3^b%ja0`#+1zCHgaSvX%Gs>A_wVZ?a7$p9wWMMEs$~HC7wrzDZ3v z$8GxL5%m46@nPF2Yi|T!uSBzCIb}}pczx1KX;rC?S?=o!U}p*cv+Mfs$`Px-VQP9o zR~5l7|sVj8`B30$aS{T%wTB(I#%Ek^4C-J(7ysJ0_BjIj; zcV4Gh7E;webyp7RYJe|vu+Xl6{Kz^XgPPC&pm`D!(wqCNRZ*%y@s+R+A1LD*b)AR^|Re-8-We$)kn$3`pMz+Mu`NJaKHwv zl{!fRX6SB39r!Og9DlEHt`PP`qA! z0XX8iGa1n~K%hi}`g3{&_@GPgGjS5jd1bCh>Yr;B4?-nZX`Azd3!hKB;!06^%~8*d z)2{s02Q>T>WSP(UNwD#lB-Ef9^DflxS7f`H$92v5wUj76}mTWU}44aeCJg;)(?w-VvzbVf_cHc;TP0= z-DQuwZO-H0s9IT>2WvXLvBL-?wfjGwyZQRg#v2|vD=l6I*Mec(=YjEjD^G(WXWJXsY%N?mRv+Rz@1)8wM*y$+$(~W3^WqRmT>E?Wo@H0!+z8Fv zMzK+>^=?fHn&#{Yi0%@Cj@;i(3z1_mZY+m{jWwyu+gKdC7n~oDVPG!p%&FlqN0zlX zTE0O7qCQzpgKsz&nxK!~>gsl)iO5GDgg+cb?d3k`DF3g@JH<{vfE%r(MTCDg{`wa# zn&gn)C{ho9aaLfgiir1%4sl})YeO<~m)>Ah3D!ajhY7EUD0Wg}+7cYS4>a4n-T1}TKErjhGuA99;^K<_PeTRz@H2cmY`irh{ik-&%4x)V(1 z55C>CaRsXJkdvIK#fx%bnjOguFEg5?=THCmgHWsW&Y!+|3*Y1 zKs<1y2K_jFm@Kd(XDS&!mvbSSWgfwICDeJd`HXm_mh%b?R4+8snjEyG#x6NHmzx(UUr7~8e>!`CDM6H{vI-ZGV#&Ju4)%IZ)OhKKQA7nx!$NECo_Bu+Ux zv2VLJU=X00S~qJ6VVyD7B=IyOc^D>OjtTYk9RnUos)y>@Rk4l{DQSty(;~GgxBu|T zp8Q$>%S{6dlaOMRx>_9R18BuoE#kw)ha?~U`@y`F0G(MneLdzXhm`;)|8pkT0BgaTFy+AO$r zc|>z;ZOXA6HfQ{FKzXfD?}K%i8aao}#w5#~qa?l`ucoyg0;0%_c?^+FlHibc$}S=s z!*&HzF|7DLT$l;$S7m6}l!Nl;-H8uNeo~!HMX@7Xhczr;^(6^xc2{RCG?s^_9n6yJ zrx$FhmFiv|kG(}%cA(}EP!|_JzCCB{Hsu&U?6}s1J$$f1IRGJo{JTq@1*9GVN(9(i z#(ro-S`rUi-BdyR+bfTgjVei@RTDaQg6hgOkp)sV<&7W8X%h&Qs-nUc-Q`CQQ8HD` z{^C$fc1{QDJcPI1BgPcLG0#+kBkc*5N>Qe|7{K7S#XytS7!8$MY)9Uv9NJ;QjhRT2 zTK{F_ay-3^u9La`ryj?QLH}ffZlrDV}9;ETq4^gRdKZ`Ukh9r zq%r=`23c{lJa|`SbyjEtZC(&)4eyUtO)H*D`Jh)3rF_I8?-;0;u&*kv(G3bz~Vx01q&(qS)Y5(p>+xsJ08CjMw zj96&m3T@9*Isi5zobu@pzkOFpkTK}tChPK9PY`x-`%jdq)M69fH}ByLdR$Yu9@Q=h z+u?sUMS+N?L;xr6qhv*)ZC?Vh%DEnDUUvIOljhMxRXv6n2~XWJAxd3TLBshLF-j?i?bfT)!S^ia!v9 zb$N?J+4zS_k3~knp*_PTrzkI@xN2Fgdk6yrUP{tZv@y)$V<9LDydiD|pc|yB!t;^c z*U`cp-NTZ0iPT!vO9=oCq)~0g|2wWPt@>Bk!rq||ntKE&s*M;uEHOBf31CEUjV|Qi zORUIb=1Z5fU`ya_?aZ}1#Abkf&@814#@Mn1!Tsf&W**r(9XhI9JvgO5bf1?|kKISBp67iH8wnXi zGBziPrB~%RSJ}N_K zE+k12!3wou5dNt*i+Wf#dPMtfYUryw6C8SzLEg|0LlDA~+&gK6$U@fuv1B(i6WP#! zbu0RQS@)pN+9kI=?ColY%ia+~eaxXez{moU5ZBW+S>fRaWiinl?DR{n6P+-kEe&>P z%15Ciai2lO3C(2KCo#h9TvP?5RU$ei%}Z5+&N`To(9&o*rVe7hal@d2QZE~wKE!!{ zfG9jpNZ6A7*bg?cVy!kOFO7%`aa>j`tyF4pQxv(?ticeYCV09(!Faw@v$ngXdK$~E}R~O#- z=sZ(Z}-Z^5+F4B2@(Yve207!x;Ui)Gca%cnTUB*QwczeKQ+r#6y7Ld`OIPXfsg3;yw2fcK-zCXgSi z6;eF8Z;K-HnaEHMWisEWK*^oC51{yZis|skPKMFTo5EksNfsqd{G$sK#WOzxr{_bC zGZ*(GVnD#SLrXwIW@(*}E4JNs70>+Ga1-O6jA#V(Ms1Bim-eMF%lSfq5>)> zI38xHeNG@-DfGQ?#4R;6Q(U$VDO@T3bk7Z$g5G{F_=8f!EX?tL9a7NR`+N-fDx^HY z(Ux3-%C)Kp&Y8un&v|F3^NC_#JkvU8u$0fLK4OW;1Qa_W!MMaSxLLSdZ|mMZg4#k9 zGM`(vsCeouYh`#%KB$zT&0vbp;x2;M4MJA(;U*~3np@aJ1-s-{fPUy6K}3pzLe|67 z;}D^W4%}L;ENmB3|F2R`y2p@GU`VDWvV>}7WpHM~ink?_Ga`9u8YPGIa~9SJ#Gi;~ zKWU#-nZf|m92#Mu@?lVXXg)z0T&wb-A0J=?DH(CQ?3n!D#j|%!#}r^Bs)|iw4GZ?q zC4JPB6q*!Of!zHjxUfYtC8C`N(1x2%HdhVy)@;#om9%v_Kp1s)8xK){`evHBfR)ct z)a3pOMMWetiqN0)^p(rHABV!>05>p)B#T)LJSWuVsl5pJ21UKd-$O4-Os3!@(=Z$a zdtysai~}M!jCy7L_ts(qY1wrm((ihE8CMER*1I!vlbb*G8Mmv+vV+#N?>MupqJ=QV zgZngxHqE{h#SVY~-~q&0(k>*Tj7b&k33PVt1?-+;95`ix&BhVPIw}@;RP+(XP%N|! zH(RO|3em0IZ)KNlmMD}mJPF1r+sPXP#1qSi1*lgj*6NX7rpoAmp-G3$BE2`?5Ohh0 zY$(|7j-TF_zPgQHPrJZDJcm^!sD7mY3Hot%hI9#-O_iVbmT(dJiVX43LP?Ggf?Qht z@YgR3`!K8N7Tq86*ktPDf|XlKh<&`?{-$fa-CAp(!dDqepHY4#D|hdHQZTifCi!1C zP&}Os3O%?TsHs! literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..97bdbcae01d8838c3ee046f227b91b99893daabe GIT binary patch literal 8047 zcmV-#ACTZEwJ-eySQYyK`m2ZriV$moT92z_% zjb-wPIGDz0$^#6}5>XnZX%w0S38$gK!b~Pkn^+oiz+qbRU~nFb37FO_2_$eD8ZeDz zY0YF&BAi3X(i#fGw9_1>DUQ?79*>mIZ^uctjkM#yB7#PQwy$Gz(@C zA;M{B&?ZbEoYq(tj~1Z-X%GxfV~Id`s6euKaF_$(;lXJft&tq1MJyte#?l(5J#d!E z#93O?D3YZ$iy~<(%Azb7h%=cut?_V3NSKHN6O5%T7F0M5jb*}ElnACda2Dk(ZDLt8 z62-x3OJg*}At7Pf1czx6XK9UP(nyr{m_Pxtkf3SW1E*0S%|z2Ar!^9$ou)WA$DxrZ zr&%2B(HhI*G>BudOqeVtOo%iRrcD+Noc35i6bcrQWpSFqIH#cv&`=%|k_7`iAd|=f zrm-}KM3Zn@!#D_RW%p$t8Q`OT0J3h(xcq-Uao(*Di@{8bgR-{^y1gK&Ry44sEeHR=RIBgruQ~; zeYfz7e5(-Ei@2KWTik`}^0=yt$&3Ge7vd&+m2;6UE-&7uT;r|1m&tem^s6*^jf=)r zxpec^*1h~DU#>#9ZyprqaLE{$)+|kefqgQY0n796X^sO~Hoj#v;is_`(EmFMybftyrrym;c?pB4qDk3~8RR2pCDs;Ng>RwcL zUCNkuG%jD}5BD@tF&)x8I_f3jIiHv`ofgqbab2!kE5-k~=)4sws`FMqdTAX|w3+w@D91jnQg@q$h zM@5v@BYL_zDqcc*PX7@hgox6AT*rJm@AQv~6xZ>Ho+Mv`>uF`GV(TkqX0Ahut>39rt4!)kuT)_wwb)fEmR4pSyYE+pnzz`xLgvNv zez<&Xa>A$`H!?Lb{f&#gKzu>UA+?m;MuDM@y+{QrD&Ixzd`$BOAhLYeI2144jt1o zV(Nel6GP~bXvH5xjImn|%J8Th&R8PE7-K9Ma!80V24aaZ#-ceCLNiCBP^M8xU<{c6 zW6+A!s8sN@wcyMk0099&0098g6951h3=D`&Ljo#o*Z~xP0U8tr5ELv76bJ$YKycs) z6R{{100u%~Fc2snR5S%n0%!pBBOPGdQ;==LD-nJdO=c4|$1C}PGP^V5qHDG|4zGZh zGuoa6mpUIw5*KB$2&)LTzs6=^=~3KbQ!DjV8nIu?cXB|a zEQ>?>3gr>1_ax;Pa}Z4|B4ke5@^yrrZ7847!w!y0bF*K2HhgkThvLXtO-T1~R3KwN zbg&IoG}3s-AjgxqUydGO{3V6r7}qdy1gMGYM(7M15&82CbX-9nG({<+pM%j~9B-KB zA7SrN;DeP%1k~n7H*D=gSo*N+gJzzpi!at8g7+ae9)X{_cQ_w^IFG>6FS=Gv|FD4H z2%82T2t z8JQJ_s^P=o*v64g>*6U`ja}~PhuZ|aQkJ+aFY76dfyNx#&Wm+|0$f9}2I-MkP)~oys1m zZxWugw@wlJtRZz@ARO<|WdeFQF7GFjSvE?2eLK0|rX6zg7Jd;6hdxcQ0Db9dsGvI> znO4`eHtVHcs8)$QOD!Lcg+A_5*%xT}p90Hp4FCL1l6P=64=nrA)U%VV z=35zSR1(z?&~mYeUqH09BMhF5`1d@UVW*675)Kx<>wE-4H=PRQ%8Zo55)>|r=qPO5 zZ!q$P)-SW0(EgqO9o&WE6+eK1aw4vN!!X`69@m1ag<|@DLj{_pwuSjCAjl)=ME$l= zghyPx5P4q&`~-2rDnW`;sTL9bf{HxswJU~3eZ3QVJITL<#^G^0yZ76#&}T8^ z@s09-1l9L&Z|gSmL=;87hag+x`<~f9H0dGRQ{KhzyTTjh>EGwXnjQ{&WY`a9YU0Tc z8UE>IJc8OIj7}f*N5MhC8;d+0{s_oJKJj8*I1YG=)0uu%_Z%8D*3_odj*xIaM>ncP zU%mE-(ZVGjEmh6oJF69W3bD2Z&uw!$SjI`v-uAI8q3xbxBmyV{f`q%s5QQ#nm;HbzfAg$!m>{kE}TSu(#Njl|&IAz+i}5 z#3Q)Z6gV2@BLKB~)D2S+UO?8((^$<5Ue!|Al-xG95Ch|{9XSxPeKDUOIbH%8TlR+XBT!c&`VInXOG;y6 z^7LA95V5o0ST4Wstbvpx-5>4u=36@y(Vs1k<15DjydRgRmTTC3e-pRleQMPx;@>dEj8t^N06a? zn6c1PIgT497foJ?Hc6#`-%lxpoV7b{QB$`b@W)}yI0{Tr3t%Z8hL%u3o^kC2OZz_g zxXC=~aHnGW{dB2z7A|2bL15Dzzs#Jc0vxyTWpeFW!Xwb$DB95?X7~4q1?|M)H01u( zAGh>|Yz2zBey`i8T$$UgBvCW2=W{zx^44%_0pNzVqg&IW2oF-xwhrNL-gB^wi6k?uVqM*+)n-^FpCWm#n+ixUor|2$&oZXb z%&?Aiv5kxGf!Vr%O;T(Qw**L@>!t$yvES(0t1%govsYKwKzWhaaGwUv2c22ayt#WB^7zJ+Wu*(e`!v*W)9WQKId zO@&B)?Y ziMqTP~yYlOs^Uvj}Me0h;2%5hSPqtgOM3LCjsLEdgs5W=3#Jh12jbfxd^PeROauGXo z_Li6M>0wfpaa=@9i!3!lQ2v448cPG?9#T`Bf3yDN(S1MY@zHFQuN$F!R~BP#Ws#ir zcv|TdpsLhYQSOBbV2qXJ*@gR{+Y#K~U@<*%Rh8;4AbUl(QKLTOgE>Bq(3V8ssaSPJ z@T$stEmhXt-cy9ZjN%UGCsDh2Vpq8YBi3#Ys$b`^tZS-&S1U(=PpCGNR%ik>3}>1$ zL$?0@a6bvw(<9s4IgpuKK%U&Fj4mK8we z*v9Y&@Yy1?&9@DU>_;PF{rPabjS3wP8o~xS;+jODf*^LAoNY^l`^pK=Y>#cak5};# zc7jmOZ5IGSNdA#;rTQvGBLsWze{+Re&)|px%1KJV$&eTcU$Uq6I>QCzt9D=Sw&$)V# z$(5FWUg+s)@l2JR(A$6=95B9OoZvCNM%Oc}3|AKE!;+B+GM=ycgrRqwKhz*N=Ed9Y zR+QV!IJ&7BVBYem@Q3BU!m4fF8JCoYL%bpu?rOj&t%wOk%lbA*wD%~7Xq3oui12;$ z%H50GU~xrw+_$OH5G#=O1r8$_{uW}Teq>y^d&R-P6`&bBv>{aQA3oOt&2 z;z^m+jzE6%>z-jbXAU8fu=dw9Fi)?9fe}J&8#ANi*1I_=*q-z8Uv!sN9o6}6K_Wi` z;;~#UEytv;{YJsqH8|fW!@zIa_gjPi91&@uxBL$PLPR9VG04GO?u?m4y{=WMCc+5h z0WZVRZLcJQ4*&eBsZ*r$1KbFmXhb+?+*+VH4-|mObq4|op;axBTMqWcr)*@Q1?w_A&GblrN-tsp{*>izh`P0zL#Smnq?lr zc_rj|v*C?6rE0hQbneK3z8N<*@#oV8#Xned#K@uxTEbkwWsULZ%1T>7qwG1 zQc0DOA6j>sVs+PwHIxS#%+U;%1xpxSIPrenvvWOwWB>Nbp0xb@A zi!i~G#eTT%km92cJ{VLbpct0aOyG(Q3fKQqaSfQq$M#6TjO3#Z_R`kpg#=pK+O8yu z*u;8`_Arg{NgO_X-VbI(ls8tTUs~F*qDX|Jb|AzLtE4<|Hs$FEmw*f+oBKopr_9>cMdI6Taqa+Z+|v0Z^QhA+Plnq~q&ptiL( z<*mu_?uSE`pR{^Y#o5utA)V!OwEgxrv8|+XRZGDypy@cDd8TxJ(7Vzkn5!8aLfq4f@WHU`iFe%y#4eGy%U)==bi|zJVO-mkumT| zT=qX~|7)aN9~{LW#czjr=&f=+W~T<1Q6l>#Rk5`xXH`xa%$e&j16i?H9+<0M$RlXO z&_!=87O3Nq9$oS>qhW7F+(-Q2m`%lJHdS7MsG}^=TcwAFmgV$ZM5U05SJ8V61i2C{$@>xd0sE4jHllVZs z;34hodJDxMFH!Y5wpluqaXb5~Fvu~>`%RF}_714cS(RiNe#b_k_#$tTg~H;u8|_2C ztpibDP&eH@(IE)yYD|lRY1wU;&TMvSxaBAhy9q4Bw(2vze0cn^K4>nv_8`R=Y1E#x zA`qoe`9t>uA@ZX;0znfvNr?OzLlb-OZ4S|r>*Gc`><=(&UC#d@y#68XvE+z8+}LoY zDN@KNaZ*d8Jxr&8mr7d7Z;ZP7SeD5G{)!tK6oYhCc!sh2Kv|gEa`;>^a^U<1hhAU?T#DB7>uC0x%1%zCwok5|#0rd`V49lM+~F&3aLCxIzm8t@VN_rx|A z)M9PkAMe_lHgafkZ=O*-Gh}O(RlzecPnB&}+%?8}DB@%K^o+v7G=va%5?6>OAxbd8 z;&e78YyTw!WOLrwp$yCn36X1*`iL-tFk7HZ>;q(buf0sxAIf>D1e+)Y=C*~xu!bBe z3`IZ|1czaTvo-a*EiG=)kH|dV4kAu_EFP_7p2}+wE@YU>*tH~JykQ@P0F1}6cfg3P3*7-C$!_RmvZDd&R92Sv`hDcK15#j#=QFzmQVN(&MADm#th&Oj# z8sR9!VJFzwyJFM#MXogXx9*~thKe>cC)H#sDI^GJSTY|M#g9&?;xjJk9{BIQb+2MN zKDjm1|M5ZP(AH8FowTss>q^_o3~+g0>X9n1cCDjIXxe{yMS z(Bno0WHAuVMI>PJsCb*-y6(5zhH2-MGv))>KI(KXA2E^r#o2+-kB*LF&pBJp%8)Ma zgVqEKES{3(s(Y$8;U=%B!16tl{b?AcO`dybs$%(4PhC)1qPR=?q8YyV(Sa)l{BjWO z>ApBBDa{ivSJ@R4s|tG-310)6hn8s}0oruht1IH5JAB((6a z)JP#89c#tL4yUl(-gmCv#PR{OA+uQsZUoo16|GjH^Wh;3_E(^k+>PCzIFeMEN=9#7bCZR)inDx1xjv-;l|1r};lj9tjTp238If+)5cQH^DPvOxzmP%HI;R zO++NsOj;2q`Hh0B&3>^be@j0)AVqC;1R5=fAfhI@gbFhbl4mr66ye$hoD|TIaIUZl z@a`rf9?03+uLo-6Ig4bFsaezClMv-62LFy00Q~65Nan|`FcjbJ+oDL^Oly`S8O3f? zu*`Pu0w{VdiZOX4t;5LLO{Z4pC3BS~?xUNDBAFk+sevK4lZ!h!F#u@XaZEravw({x zfr`nl1}~J!*<_p$EwLD2D25XA<$+%P|ItH0u>$kgpG1)2AFo$^_!?$R4E@Yj5~>(e zs?#P4Jb)LPu9PCOYr#4F?pYZx)bU1o-cdw}(#a-@yVFGUXFbTOYR-rVBA$a&7B z2ufK$VZ8K4e8C0*V|)f^mT`c=@>OM{AxRA#EWFnXW5YRWwJIu`s+$Xi%Nd zMqrA|<6eqa3_@1;FjEvIlUvxM0#qOLAA{xpb1@!oQX(g6rnR`X->_& zAIFWu;cj3INrtr8e@-YTsYems4T^D*e~VtM8&JVX7Q>(k@Wdsds73c~R(fU6Zy$^O zwq>_SWccvFGR_J1y?1BmCO3ZU8E&_nWmkNo9p#L!ilV|u)_122ttVV36k-6_hlgQ~ z#$A3fGCGZPH{RJV1uWf=@sr6;m>BU-)@Cf&R54^2X*p@*(rjNVC@ifV-IDFnY&DSM zPW6mg-^uF^l&6m22C%n~o7JPi<=NCR!1Khpss=hTD#`$)reM9NkF+URcs3JH;VtoJr literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..63a69f7af 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -20,6 +20,12 @@ FORMAT = { 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), 'br': ('html-br.bin', 'br'), + # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin + 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), + # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin + 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin + 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), } @@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b" Date: Mon, 5 Oct 2020 23:55:48 +0100 Subject: [PATCH 100/127] Minor adjustment to the test case in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 63a69f7af..7e9f9cc5d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -101,7 +101,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) if raw_content is None: raw_content = newresponse.body - assert raw_content == newresponse.body + else: + assert raw_content == newresponse.body assert newresponse is not response assert newresponse.body.startswith(b" Date: Tue, 6 Oct 2020 10:50:17 -0300 Subject: [PATCH 101/127] Docs: mention limitation about Cookie header --- docs/topics/downloader-middleware.rst | 5 +++++ docs/topics/request-response.rst | 12 ++++++++++++ docs/topics/settings.rst | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..ae84b54fb 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -207,6 +207,11 @@ CookiesMiddleware a warning. Refer to :ref:`topics-logging-advanced-customization` to customize the logging behaviour. + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 30b1945d0..f3aaa2c8f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -61,6 +61,12 @@ Request objects :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If ``None`` is passed as value, the HTTP header will not be sent at all. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type headers: dict :param cookies: the request cookies. These can be sent in two forms. @@ -102,6 +108,12 @@ Request objects ) For more info see :ref:`cookies-mw`. + + .. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + :type cookies: dict or list :param encoding: the encoding of this request (defaults to ``'utf-8'``). diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 06234c5d9..71331c841 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -352,6 +352,11 @@ Default:: The default headers used for Scrapy HTTP Requests. They're populated in the :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`. +.. caution:: Cookies set via the ``Cookie`` header are not considered by the + :ref:`cookies-mw`. If you need to set cookies for a request, use the + :class:`Request.cookies ` parameter. This is a known + current limitation that is being worked on. + .. setting:: DEPTH_LIMIT DEPTH_LIMIT From 9b1f86b613d2039b0a66ba2b527a7e9bffadaf3a Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 6 Oct 2020 18:50:55 +0500 Subject: [PATCH 102/127] Use f-strings --- scrapy/utils/iterators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index e140e3f6f..3b504e56a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -35,7 +35,7 @@ def xmliter(obj, nodename): namespaces = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): - tag = re.search(r'<\s*%s.*?xmlns[:=][^>]*>' % tagname, text[:header_end_idx[1]], re.S) + tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S) if tag: namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group())) @@ -45,7 +45,7 @@ def xmliter(obj, nodename): document_header + match.group().replace( nodename, - '%s %s' % (nodename, ' '.join(namespaces.values())), + f'{nodename} {" ".join(namespaces.values())}', 1 ) + header_end @@ -56,7 +56,7 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = f'{{{namespace}}}{nodename}'if namespace else nodename + tag = f'{{{namespace}}}{nodename}' if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: From e40788153ca7dee9e0d4f8ac16a9e17278f79058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 6 Oct 2020 19:13:29 +0200 Subject: [PATCH 103/127] Do not consider about: URLs invalid --- scrapy/http/request/__init__.py | 6 +++++- tests/test_http_request.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ef58deacc..498f1b052 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -65,7 +65,11 @@ class Request(object_ref): s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) - if ('://' not in self._url) and (not self._url.startswith('data:')): + if ( + '://' not in self._url + and not self._url.startswith('about:') + and not self._url.startswith('data:') + ): raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 0a303dbe2..74579dfc4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -43,6 +43,15 @@ class RequestTest(unittest.TestCase): assert r.headers is not headers self.assertEqual(r.headers[b"caca"], b"coco") + def test_url_scheme(self): + # This test passes by not raising any (ValueError) exception + self.request_class('http://example.org') + self.request_class('https://example.org') + self.request_class('s3://example.org') + self.request_class('ftp://example.org') + self.request_class('about:config') + self.request_class('data:,Hello%2C%20World!') + def test_url_no_scheme(self): self.assertRaises(ValueError, self.request_class, 'foo') self.assertRaises(ValueError, self.request_class, '/foo/') From 2e734e6b35f686f62dea11e9484c7646c7c49ca8 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:51:05 +0100 Subject: [PATCH 104/127] Minor update on the import order in scrapy/downloadermiddlewares/httpcompression.py --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56421a6ba..f504302e2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,5 @@ -import zlib import io +import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse From 156bb0a1d413c7c6acafc30003ef98030a06e47f Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:53:40 +0100 Subject: [PATCH 105/127] Fixing the minor typo on test file path in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7e9f9cc5d..4c5bfc577 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -25,7 +25,7 @@ FORMAT = { # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin - 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + 'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'), } From 1a597d5e3dda7a467d69cff51df7e731f2f2d6b5 Mon Sep 17 00:00:00 2001 From: OfirD1 Date: Tue, 6 Oct 2020 21:54:42 +0300 Subject: [PATCH 106/127] moved the sentence about processing pending requests when a spider is closed onto a generic note. --- docs/topics/extensions.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 14096ada4..519f18b63 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -257,6 +257,12 @@ settings: * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` +.. note:: + + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + requests) are still processed. + .. setting:: CLOSESPIDER_TIMEOUT CLOSESPIDER_TIMEOUT @@ -279,8 +285,6 @@ Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. -Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT From 13ae17aecc632dc03af1bd8b2b63e48301147cc2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Oct 2020 14:04:52 -0300 Subject: [PATCH 107/127] Add xfail_strict=true to pytest.ini --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index ca8191f42..1c95f715a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +xfail_strict = true usefixtures = chdir python_files=test_*.py __init__.py python_classes= @@ -40,4 +41,3 @@ flake8-ignore = scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 - From b55c911ddc21a41a6e4204aaf239a16e892de7b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 21 Sep 2020 10:59:55 -0300 Subject: [PATCH 108/127] Fix CachingHostnameResolver --- scrapy/resolver.py | 57 +++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f191deac6..0350c82b9 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,4 +1,5 @@ from twisted.internet import defer +from twisted.internet._resolver import HostResolution from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider @@ -50,6 +51,27 @@ class CachingThreadedResolver(ThreadedResolver): return result +@provider(IResolutionReceiver) +class _CachingResolutionReceiver: + def __init__(self, resolutionReceiver, hostName): + self.resolutionReceiver = resolutionReceiver + self.hostName = hostName + self.addresses = [] + + def resolutionBegan(self, resolution): + self.resolutionReceiver.resolutionBegan(resolution) + self.resolution = resolution + + def addressResolved(self, address): + self.resolutionReceiver.addressResolved(address) + self.addresses.append(address) + + def resolutionComplete(self): + self.resolutionReceiver.resolutionComplete() + if self.addresses: + dnscache[self.hostName] = self.addresses + + @implementer(IHostnameResolver) class CachingHostnameResolver: """ @@ -73,33 +95,22 @@ class CachingHostnameResolver: def install_on_reactor(self): self.reactor.installNameResolver(self) - def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, - addressTypes=None, transportSemantics='TCP'): - - @provider(IResolutionReceiver) - class CachingResolutionReceiver(resolutionReceiver): - - def resolutionBegan(self, resolution): - super().resolutionBegan(resolution) - self.resolution = resolution - self.resolved = False - - def addressResolved(self, address): - super().addressResolved(address) - self.resolved = True - - def resolutionComplete(self): - super().resolutionComplete() - if self.resolved: - dnscache[hostName] = self.resolution - + def resolveHostName( + self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP" + ): try: - return dnscache[hostName] + addresses = dnscache[hostName] except KeyError: return self.original_resolver.resolveHostName( - CachingResolutionReceiver(), + _CachingResolutionReceiver(resolutionReceiver, hostName), hostName, portNumber, addressTypes, - transportSemantics + transportSemantics, ) + else: + resolutionReceiver.resolutionBegan(HostResolution(hostName)) + for addr in addresses: + resolutionReceiver.addressResolved(addr) + resolutionReceiver.resolutionComplete() + return resolutionReceiver From 8fe5876597825a4df03bd15e6b1cd1682b806de8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 30 Sep 2020 14:56:17 -0300 Subject: [PATCH 109/127] HostResolution implementation --- scrapy/resolver.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0350c82b9..0bef555a6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,7 +1,6 @@ from twisted.internet import defer -from twisted.internet._resolver import HostResolution from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple +from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache @@ -51,6 +50,15 @@ class CachingThreadedResolver(ThreadedResolver): return result +@implementer(IHostResolution) +class HostResolution: + def __init__(self, name): + self.name = name + + def cancel(self): + raise NotImplementedError() + + @provider(IResolutionReceiver) class _CachingResolutionReceiver: def __init__(self, resolutionReceiver, hostName): From 868826b346714ceff821886536a3b259072f8396 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 2 Oct 2020 15:16:58 -0300 Subject: [PATCH 110/127] CachingHostnameResolver tests --- .../alternative_name_resolver.py | 15 ---------- .../caching_hostname_resolver.py | 30 +++++++++++++++++++ .../caching_hostname_resolver_ipv6.py | 19 ++++++++++++ tests/CrawlerProcess/default_name_resolver.py | 11 +++++-- tests/test_crawler.py | 20 +++++++++---- 5 files changed, 72 insertions(+), 23 deletions(-) delete mode 100644 tests/CrawlerProcess/alternative_name_resolver.py create mode 100644 tests/CrawlerProcess/caching_hostname_resolver.py create mode 100644 tests/CrawlerProcess/caching_hostname_resolver_ipv6.py diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py deleted file mode 100644 index 2c466da04..000000000 --- a/tests/CrawlerProcess/alternative_name_resolver.py +++ /dev/null @@ -1,15 +0,0 @@ -import scrapy -from scrapy.crawler import CrawlerProcess - - -class IPv6Spider(scrapy.Spider): - name = "ipv6_spider" - start_urls = ["http://[::1]"] - - -process = CrawlerProcess(settings={ - "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", -}) -process.crawl(IPv6Spider) -process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..f9eab3543 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,30 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + name = "caching_hostname_resolver_spider" + + def start_requests(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..3340d2f84 --- /dev/null +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,19 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = CrawlerProcess(settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + }) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index 60d91b68b..05a98fbec 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ name = "ipv6_spider" start_urls = ["http://[::1]"] -process = CrawlerProcess(settings={"RETRY_ENABLED": False}) -process.crawl(IPv6Spider) -process.start() +if __name__ == "__main__": + process = CrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 85035a220..246e54860 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from tests.mockserver import MockServer + class BaseCrawlerTest(unittest.TestCase): @@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: - def run_script(self, script_name): + def run_script(self, script_name, *script_args): script_path = os.path.join(self.script_dir, script_name) - args = (sys.executable, script_path) + args = [sys.executable, script_path] + list(script_args) p = subprocess.Popen(args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() @@ -321,11 +323,19 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log) - def test_ipv6_alternative_name_resolver(self): - log = self.run_script('alternative_name_resolver.py') - self.assertIn('Spider closed (finished)', log) + def test_caching_hostname_resolver_ipv6(self): + log = self.run_script("caching_hostname_resolver_ipv6.py") + self.assertIn("Spider closed (finished)", log) self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_caching_hostname_resolver_finite_execution(self): + with MockServer() as mock_server: + log = self.run_script("caching_hostname_resolver.py", mock_server.http_address) + self.assertIn("Spider closed (finished)", log) + self.assertNotIn("ERROR: Error downloading", log) + self.assertNotIn("TimeoutError", log) + self.assertNotIn("twisted.internet.error.DNSLookupError", log) + def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) From 015c82b974841897a637dcd3ddbd40ab48a70ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:09:45 +0200 Subject: [PATCH 111/127] Scrapy 2.4 release notes (#4808) --- docs/news.rst | 306 ++++++++++++++++++++++++++++++++- docs/topics/asyncio.rst | 2 + docs/topics/feed-exports.rst | 18 +- docs/topics/media-pipeline.rst | 31 ++-- docs/topics/settings.rst | 2 +- 5 files changed, 344 insertions(+), 15 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 850b323ef..d5c342c86 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,310 @@ Release notes ============= +.. _release-2.4.0: + +Scrapy 2.4.0 (2020-10-??) +------------------------- + +Highlights: + +* Python 3.5 support has been dropped. + +* The ``file_path`` method of :ref:`media pipelines ` + can now access the source :ref:`item `. + + This allows you to set a download file path based on item data. + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` + +* You can now choose whether :ref:`feed exports ` + overwrite or append to the output file. + + For example, when using the :command:`crawl` or :command:`runspider` + commands, you can use the ``-O`` option instead of ``-o`` to overwrite the + output file. + +* Zstd-compressed responses are now supported if zstandard_ is installed. + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +* Python 3.6 or greater is now required; support for Python 3.5 has been + dropped + + As a result: + + - When using PyPy, PyPy 7.2.0 or greater :ref:`is now required + ` + + - For Amazon S3 storage support in :ref:`feed exports + ` or :ref:`media pipelines + `, botocore_ 1.4.87 or greater is now required + + - To use the :ref:`images pipeline `, Pillow_ 4.0.0 or + greater is now required + + (:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`, + :issue:`4764`) + + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again + discards cookies defined in :attr:`Request.headers + `. + + We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it + was reported that the current implementation could break existing code. + + If you need to set cookies for a request, use the :class:`Request.cookies + ` parameter. + + A future version of Scrapy will include a new, better implementation of the + reverted bug fix. + + (:issue:`4717`, :issue:`4823`) + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the + values of ``access_key`` and ``secret_key`` from the running project + settings when they are not passed to its ``__init__`` method; you must + either pass those parameters to its ``__init__`` method or use + :class:`S3FeedStorage.from_crawler + ` + (:issue:`4356`, :issue:`4411`, :issue:`4688`) + +* :attr:`Rule.process_request ` + no longer admits callables which expect a single ``request`` parameter, + rather than both ``request`` and ``response`` (:issue:`4818`) + + +Deprecations +~~~~~~~~~~~~ + +* In custom :ref:`media pipelines `, signatures that + do not accept a keyword-only ``item`` parameter in any of the methods that + :ref:`now support this parameter ` are now + deprecated (:issue:`4628`, :issue:`4686`) + +* In custom :ref:`feed storage backend classes `, + ``__init__`` method signatures that do not accept a keyword-only + ``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`, + :issue:`4512`) + +* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated + (:issue:`4684`, :issue:`4701`) + +* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use + :func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`, + :issue:`4776`) + + +New features +~~~~~~~~~~~~ + +.. _media-pipeline-item-parameter: + +* The following methods of :ref:`media pipelines ` now + accept an ``item`` keyword-only parameter containing the source + :ref:`item `: + + - In :class:`scrapy.pipelines.files.FilesPipeline`: + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.file_path` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download` + + - In :class:`scrapy.pipelines.images.ImagesPipeline`: + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded` + + - :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download` + + (:issue:`4628`, :issue:`4686`) + +* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows + to define keyword parameters to pass to :ref:`item exporter classes + ` (:issue:`4606`, :issue:`4768`) + +* :ref:`Feed exports ` gained overwrite support: + + * When using the :command:`crawl` or :command:`runspider` commands, you + can use the ``-O`` option instead of ``-o`` to overwrite the output + file + + * You can use the ``overwrite`` key in the :setting:`FEEDS` setting to + configure whether to overwrite the output file (``True``) or append to + its content (``False``) + + * The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage + backend classes ` now receive a new keyword-only + parameter, ``feed_options``, which is a dictionary of :ref:`feed + options ` + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* Zstd-compressed responses are now supported if zstandard_ is installed + (:issue:`4831`) + +* In settings, where the import path of a class is required, it is now + possible to pass a class object instead (:issue:`3870`, :issue:`3873`). + + This includes also settings where only part of its value is made of an + import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or + :setting:`DOWNLOAD_HANDLERS`. + +* :ref:`Downloader middlewares ` can now + override :class:`response.request `. + + If a :ref:`downloader middleware ` returns + a :class:`~scrapy.http.Response` object from + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response` + or + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + with a custom :class:`~scrapy.http.Request` object assigned to + :class:`response.request `: + + - The response is handled by the callback of that custom + :class:`~scrapy.http.Request` object, instead of being handled by the + callback of the original :class:`~scrapy.http.Request` object + + - That custom :class:`~scrapy.http.Request` object is now sent as the + ``request`` argument to the :signal:`response_received` signal, instead + of the original :class:`~scrapy.http.Request` object + + (:issue:`4529`, :issue:`4632`) + +* When using the :ref:`FTP feed storage backend `: + + - It is now possible to set the new ``overwrite`` :ref:`feed option + ` to ``False`` to append to an existing file instead of + overwriting it + + - The FTP password can now be omitted if it is not necessary + + (:issue:`547`, :issue:`716`, :issue:`4512`) + +* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now + supports an ``errors`` parameter to indicate how to handle encoding errors + (:issue:`4755`) + +* When :ref:`using asyncio `, it is now possible to + :ref:`set a custom asyncio loop ` (:issue:`4306`, + :issue:`4414`) + +* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are + spider methods that delegate on other callable (:issue:`4756`) + +* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged + message is now a warning, instead of an error (:issue:`3874`, + :issue:`3886`, :issue:`4752`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`genspider` command no longer overwrites existing files + unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`, + :issue:`4623`) + +* Cookies with an empty value are no longer considered invalid cookies + (:issue:`4772`) + +* The :command:`runspider` command now supports files with the ``.pyw`` file + extension (:issue:`4643`, :issue:`4646`) + +* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + middleware now simply ignores unsupported proxy values (:issue:`3331`, + :issue:`4778`) + +* Checks for generator callbacks with a ``return`` statement no longer warn + about ``return`` statements in nested functions (:issue:`4720`, + :issue:`4721`) + +* The system file mode creation mask no longer affects the permissions of + files generated using the :command:`startproject` command (:issue:`4722`) + +* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names + (:issue:`861`, :issue:`4746`) + +* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can + work when using a headless browser (:issue:`4835`) + + +Documentation +~~~~~~~~~~~~~ + +* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`, + :issue:`4724`) + +* Improved the documentation of + :ref:`link extractors ` with an usage example from + a spider callback and reference documentation for the + :class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`) + +* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the + :class:`~scrapy.extensions.closespider.CloseSpider` extension + (:issue:`4836`) + +* Removed references to Python 2’s ``unicode`` type (:issue:`4547`, + :issue:`4703`) + +* We now have an :ref:`official deprecation policy ` + (:issue:`4705`) + +* Our :ref:`documentation policies ` now cover usage + of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged` + directives, and we have removed usages referencing Scrapy 1.4.0 and earlier + versions (:issue:`3971`, :issue:`4310`) + +* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`, + :issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Extended typing hints (:issue:`4243`, :issue:`4691`) + +* Added tests for the :command:`check` command (:issue:`4663`) + +* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`) + +* Improved Windows test coverage (:issue:`4723`) + +* Switched to :ref:`formatted string literals ` where possible + (:issue:`4307`, :issue:`4324`, :issue:`4672`) + +* Modernized :func:`super` usage (:issue:`4707`) + +* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`, + :issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`, + :issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`, + :issue:`4820`, :issue:`4822`, :issue:`4839`) + + .. _release-2.3.0: Scrapy 2.3.0 (2020-08-04) @@ -4008,9 +4312,9 @@ First release of Scrapy. .. _six: https://six.readthedocs.io/ .. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib .. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py .. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 .. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/ .. _Zsh: https://www.zsh.org/ +.. _zstandard: https://pypi.org/project/zstandard/ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index bfb430d52..91e1cca0d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -1,3 +1,5 @@ +.. _using-asyncio: + ======= asyncio ======= diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 1744cfd74..843ed25f9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_. * ``s3://mybucket/path/to/export.csv`` * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` - * Required external libraries: `botocore`_ + * Required external libraries: `botocore`_ >= 1.4.87 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -319,6 +319,8 @@ For instance:: }, } +.. _feed-options: + The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition: @@ -329,6 +331,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. @@ -337,6 +341,8 @@ as a fallback value if that key is not provided for a specific feed definition: - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. + .. versionadded:: 2.4.0 + - ``overwrite``: whether to overwrite the file if it already exists (``True``) or append to its content (``False``). @@ -355,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + .. versionadded:: 2.4.0 + - ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. - ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. @@ -517,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. setting:: FEED_EXPORT_BATCH_ITEM_COUNT FEED_EXPORT_BATCH_ITEM_COUNT ------------------------------ +---------------------------- + +.. versionadded:: 2.3.0 Default: ``0`` @@ -586,11 +596,15 @@ The function signature should be as follows: If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` is always ``1``. + .. versionadded:: 2.3.0 + - ``batch_time``: UTC date and time, in ISO format with ``:`` replaced with ``-``. See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + .. versionadded:: 2.3.0 + - ``time``: ``batch_time``, with microseconds set to ``0``. :type params: dict diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 06809c24b..156897274 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -56,6 +56,8 @@ this: error will be logged and the file won't be present in the ``files`` field. +.. _images-pipeline: + Using the Images Pipeline ========================= @@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to -JPEG/RGB format, so you need to install this library in order to use it. -`Python Imaging Library`_ (PIL) should also work in most cases, but it is known -to cause troubles in some setups, so we recommend to use `Pillow`_ instead of -PIL. +The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for +thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow -.. _Python Imaging Library: http://www.pythonware.com/products/pil/ .. _topics-media-pipeline-enabling: @@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses the passive connection mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +.. _media-pipelines-s3: + Amazon S3 storage ----------------- .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 -bucket. Scrapy will automatically upload the files to the bucket. +If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and +:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will +automatically upload the files to the bucket. For example, this is a valid :setting:`IMAGES_STORE` value:: @@ -187,8 +188,9 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like -self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: +You can also use other S3-like storages. Storages like self-hosted `Minio`_ or +`s3.scality`_. All you need to do is set endpoint option in you Scrapy +settings:: AWS_ENDPOINT_URL = 'http://minio.example.com:9000' @@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) +.. _botocore: https://github.com/boto/botocore +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _s3.scality: https://s3.scality.com/ -.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _media-pipeline-gcs: @@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: FilesPipeline.get_media_requests(item, info) As seen on the workflow, the pipeline will get the URLs of the images to @@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline: By default the :meth:`file_path` method returns ``full/.``. + .. versionadded:: 2.4 + The *item* parameter. + .. method:: ImagesPipeline.get_media_requests(item, info) Works the same way as :meth:`FilesPipeline.get_media_requests` method, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 71331c841..912757850 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section. Import paths and classes ======================== -.. versionadded:: VERSION +.. versionadded:: 2.4.0 When a setting references a callable object to be imported by Scrapy, such as a class or a function, there are two different ways you can specify that object: From 47eac8374040c0dc389eb8152a60938dab358bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:11:14 +0200 Subject: [PATCH 112/127] Set a release date for Scrapy 2.4.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index d5c342c86..a3889705d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.4.0: -Scrapy 2.4.0 (2020-10-??) +Scrapy 2.4.0 (2020-10-11) ------------------------- Highlights: From c340e72988fc6ec615b7b9851c3d28c16c26a839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 11 Oct 2020 22:12:45 +0200 Subject: [PATCH 113/127] =?UTF-8?q?Bump=20version:=202.3.0=20=E2=86=92=202?= =?UTF-8?q?.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3c1c8f891..0f142472e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.0 +current_version = 2.4.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 276cbf9e2..197c4d5c2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.3.0 +2.4.0 From 585e4a8aee649f2b439c42d91d857ed1fbdf4fa5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 9 Oct 2020 10:41:19 -0300 Subject: [PATCH 114/127] Replace local server address --- tests/test_crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 246e54860..b6de33189 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -330,7 +330,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_caching_hostname_resolver_finite_execution(self): with MockServer() as mock_server: - log = self.run_script("caching_hostname_resolver.py", mock_server.http_address) + http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1") + log = self.run_script("caching_hostname_resolver.py", http_address) self.assertIn("Spider closed (finished)", log) self.assertNotIn("ERROR: Error downloading", log) self.assertNotIn("TimeoutError", log) From a5872a0fad090c3d3f91f6e03a772265aa50f901 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 30 Oct 2020 20:36:39 +0200 Subject: [PATCH 115/127] Fix output file overwrite with -O (FeedExporter updated) (#4859) --- scrapy/extensions/feedexport.py | 2 +- tests/test_commands.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7dcb2f52e..3fb4d0e2c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -452,7 +452,7 @@ class FeedExporter: crawler = getattr(self, 'crawler', None) def build_instance(builder, *preargs): - return build_storage(builder, uri, preargs=preargs) + return build_storage(builder, uri, feed_options=feed_options, preargs=preargs) if crawler and hasattr(feedcls, 'from_crawler'): instance = build_instance(feedcls.from_crawler, crawler) diff --git a/tests/test_commands.py b/tests/test_commands.py index 3e54a0948..85aee55a5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -680,9 +680,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ @@ -813,9 +818,14 @@ class MySpider(scrapy.Spider): ) return [] """ + with open(os.path.join(self.cwd, "example.json"), "w") as f1: + f1.write("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + with open(os.path.join(self.cwd, "example.json")) as f2: + first_line = f2.readline() + self.assertNotEqual(first_line, "not empty") def test_output_and_overwrite_output(self): spider_code = """ From c292957cb19085137146bde72fe82639591c5e1c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Thu, 5 Nov 2020 11:15:58 -0300 Subject: [PATCH 116/127] Run Windows tests on GitHub actions (#4869) --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++++++ azure-pipelines.yml | 22 ---------------------- 2 files changed, 31 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 azure-pipelines.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..28771216c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,31 @@ +name: Run test suite +on: [push, pull_request] + +jobs: + test-windows: + name: "Windows Tests" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest] + python-version: [3.7, 3.8] + env: [TOXENV: py] + include: + - os: windows-latest + python-version: 3.6 + env: + TOXENV: windows-pinned + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Run test suite + env: ${{ matrix.env }} + run: | + pip install -U tox twine wheel codecov + tox diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index c03e258c7..000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - TOXENV: py -pool: - vmImage: 'windows-latest' -strategy: - matrix: - Python36: - python.version: '3.6' - TOXENV: windows-pinned - Python37: - python.version: '3.7' - Python38: - python.version: '3.8' -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' -- script: | - pip install -U tox twine wheel codecov - tox - displayName: 'Run test suite' From 5b5478ae9d6f8d5e0028fe47a63252679c457364 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 5 Nov 2020 14:01:34 -0300 Subject: [PATCH 117/127] Call asyncio.get_event_loop when installing the asyncio reactor --- scrapy/utils/reactor.py | 3 +- .../CrawlerProcess/asyncio_deferred_signal.py | 45 +++++++++++++++++++ tests/test_crawler.py | 16 +++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/CrawlerProcess/asyncio_deferred_signal.py diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 831d29462..6723d9b37 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None): if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.new_event_loop() + event_loop = asyncio.get_event_loop() asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..bce300afe --- /dev/null +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,45 @@ +import asyncio +import sys + +import scrapy + +from scrapy.crawler import CrawlerProcess +from twisted.internet.defer import Deferred + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + loop = asyncio.get_event_loop() + return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(scrapy.Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + }) + process.crawl(UrlSpider) + process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b6de33189..0faaa79a3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -364,6 +364,22 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertIn("Using asyncio event loop: uvloop.Loop", log) + @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly") + @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows") + def test_custom_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + + def test_default_loop_asyncio_deferred_signal(self): + log = self.run_script("asyncio_deferred_signal.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("Using asyncio event loop: uvloop.Loop", log) + self.assertIn("async pipeline opened!", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From 3095d39740c4818d2c1c98d392255981ebed2a10 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 12:16:10 -0300 Subject: [PATCH 118/127] Test: disable asyncio reactor on Windows for Py>=3.8 --- tests/test_crawler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0faaa79a3..ab113710d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -373,6 +373,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From 114229eb4a5e3e0289000500cf063518be908d40 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 6 Nov 2020 13:29:14 -0300 Subject: [PATCH 119/127] Docs: add a note about asyncio.set_event_loop --- docs/topics/settings.rst | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 912757850..0086a6c74 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP Default: ``None`` -Import path of a given asyncio event loop class. +Import path of a given ``asyncio`` event loop class. -If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the -asyncio event loop to be used with it. Set the setting to the import path of the +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the desired asyncio event loop class. If the setting is set to ``None`` the default asyncio event loop will be used. If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` -function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop -class to be used. +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. +.. caution:: Please be aware that, when using a non-default event loop + (either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with + :func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call + :func:`asyncio.set_event_loop`, which will set the specified event loop + as the current loop for the current OS thread. + .. setting:: BOT_NAME BOT_NAME From a2c4a7f9200a06b227a297b9dd2919fe3ec37dbe Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Sun, 8 Nov 2020 19:12:18 -0300 Subject: [PATCH 120/127] Add missing f-string prefix to genspider output --- scrapy/commands/genspider.py | 2 +- tests/test_commands.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 72248bded..5f44daa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -98,7 +98,7 @@ class Command(ScrapyCommand): print(f"Created spider {name!r} using template {template_name!r} ", end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n {spiders_module.__name__}.{module}") + print(f"in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): template_file = join(self.templates_dir, f'{template}.tmpl') diff --git a/tests/test_commands.py b/tests/test_commands.py index 85aee55a5..d3ac05eac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest): def test_template(self, tplname='crawl'): args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' + spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) From 7e98a76ac455a8c69950104766719cde313bbb74 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 12:17:15 -0300 Subject: [PATCH 121/127] Use deferred_from_coro in asyncio test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bce300afe..dd82aa2ff 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -1,9 +1,9 @@ import asyncio import sys -import scrapy - +from scrapy import Spider from scrapy.crawler import CrawlerProcess +from scrapy.utils.defer import deferred_from_coro from twisted.internet.defer import Deferred @@ -14,13 +14,13 @@ class UppercasePipeline: def open_spider(self, spider): loop = asyncio.get_event_loop() - return Deferred.fromFuture(loop.create_task(self._open_spider(spider))) + return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): return {"url": item["url"].upper()} -class UrlSpider(scrapy.Spider): +class UrlSpider(Spider): name = "url_spider" start_urls = ["data:,"] custom_settings = { From b20cfef1e54d9f22769f6d0ec6ae06031bf86ec3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Nov 2020 13:58:52 -0300 Subject: [PATCH 122/127] Remove unnecessary line from test --- tests/CrawlerProcess/asyncio_deferred_signal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index dd82aa2ff..46c2a12a4 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -13,7 +13,6 @@ class UppercasePipeline: await asyncio.sleep(0.1) def open_spider(self, spider): - loop = asyncio.get_event_loop() return deferred_from_coro(self._open_spider(spider)) def process_item(self, item, spider): From c20b34269f488dae4de9433d9c7c783bc481bc6f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 10 Nov 2020 15:35:09 -0300 Subject: [PATCH 123/127] Remove unnecessary pytest-azurepipelines package (#4876) --- tests/requirements-py3.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2eed2f5da..7f8a5c52e 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,7 +6,6 @@ mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 -pytest-azurepipelines pytest-cov pytest-twisted >= 1.11 pytest-xdist From 85604e1078b927c2a875040e29c58b4594c8d386 Mon Sep 17 00:00:00 2001 From: joaquin garmendia Date: Wed, 11 Nov 2020 15:16:01 -0500 Subject: [PATCH 124/127] Add failed and success count stats to feedstorage backends (#4850) --- scrapy/extensions/feedexport.py | 22 ++++--- tests/test_feedexport.py | 106 ++++++++++++++++++++++++++++---- 2 files changed, 110 insertions(+), 18 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3fb4d0e2c..bec114707 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -319,18 +319,26 @@ class FeedExporter: # Use `largs=log_args` to copy log_args into function's scope # instead of using `log_args` from the outer scope d.addCallback( - lambda _, largs=log_args: logger.info( - logfmt % "Stored", largs, extra={'spider': spider} - ) + self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__ ) d.addErrback( - lambda f, largs=log_args: logger.error( - logfmt % "Error storing", largs, - exc_info=failure_to_exc_info(f), extra={'spider': spider} - ) + self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__ ) return d + def _handle_store_error(self, f, largs, logfmt, spider, slot_type): + logger.error( + logfmt % "Error storing", largs, + exc_info=failure_to_exc_info(f), extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") + + def _handle_store_success(self, f, largs, logfmt, spider, slot_type): + logger.info( + logfmt % "Stored", largs, extra={'spider': spider} + ) + self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}") + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ea4e8e12..d248824fc 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -8,6 +8,7 @@ import tempfile import warnings from abc import ABC, abstractmethod from collections import defaultdict +from contextlib import ExitStack from io import BytesIO from logging import getLogger from pathlib import Path @@ -47,6 +48,21 @@ from scrapy.utils.test import ( ) from tests.mockserver import MockFTPServer, MockServer +from tests.spiders import ItemSpider + + +def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + +def printf_escape(string): + return string.replace('%', '%%') + + +def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) class FileFeedStorageTest(unittest.TestCase): @@ -620,12 +636,6 @@ class FeedExportTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - def path_to_url(path): - return urljoin('file:', pathname2url(str(path))) - - def printf_escape(string): - return string.replace('%', '%%') - FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { printf_escape(path_to_url(file_path)): feed_options @@ -748,6 +758,69 @@ class FeedExportTest(FeedExportTestBase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) + @defer.inlineCallbacks + def test_stats_file_success(self): + settings = { + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver: + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + + @defer.inlineCallbacks + def test_stats_file_failed(self): + settings = { + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with ExitStack() as stack: + mockserver = stack.enter_context(MockServer()) + stack.enter_context( + mock.patch( + "scrapy.extensions.feedexport.FileFeedStorage.store", + side_effect=KeyError("foo")) + ) + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1) + + @defer.inlineCallbacks + def test_stats_multiple_file(self): + settings = { + 'AWS_ACCESS_KEY_ID': 'access_key', + 'AWS_SECRET_ACCESS_KEY': 'secret_key', + "FEEDS": { + printf_escape(path_to_url(self._random_temp_filename())): { + "format": "json", + }, + "s3://bucket/key/foo.csv": { + "format": "csv", + }, + "stdout:": { + "format": "xml", + } + }, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"): + yield crawler.crawl(mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()) + self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1) + @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -1256,11 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - def build_url(path): - if path[0] != '/': - path = '/' + path - return urljoin('file:', path) - FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { build_url(file_path): feed @@ -1550,6 +1618,22 @@ class BatchDeliveriesTest(FeedExportTestBase): data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) + @defer.inlineCallbacks + def test_stats_batch_file_success(self): + settings = { + "FEEDS": { + build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): { + "format": "json", + } + }, + "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + } + crawler = get_crawler(ItemSpider, settings) + with MockServer() as mockserver: + yield crawler.crawl(total=2, mockserver=mockserver) + self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()) + self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12) + @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() From 15d301e968aa3e26a28f771cf1b45635e84ef094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 09:16:08 +0100 Subject: [PATCH 125/127] Cover Scrapy 2.4.1 in the release notes (#4884) Co-authored-by: Mikhail Korobov --- docs/news.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index a3889705d..e92493252 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,29 @@ Release notes ============= +.. _release-2.4.1: + +Scrapy 2.4.1 (2020-11-17) +------------------------- + +- Fixed :ref:`feed exports ` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`) + +- Fixed the AsyncIO event loop handling, which could make code hang + (:issue:`4855`, :issue:`4872`) + +- Fixed the IPv6-capable DNS resolver + :class:`~scrapy.resolver.CachingHostnameResolver` for download handlers + that call + :meth:`reactor.resolve ` + (:issue:`4802`, :issue:`4803`) + +- Fixed the output of the :command:`genspider` command showing placeholders + instead of the import part of the generated spider module (:issue:`4874`) + +- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, + :issue:`4876`) + + .. _release-2.4.0: Scrapy 2.4.0 (2020-10-11) From 26836c4e1ae9588ee173c5977fc6611364ca7cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 09:17:39 +0100 Subject: [PATCH 126/127] =?UTF-8?q?Bump=20version:=202.4.0=20=E2=86=92=202?= =?UTF-8?q?.4.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0f142472e..956c512cb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0 +current_version = 2.4.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 197c4d5c2..005119baa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.4.0 +2.4.1 From 63becd1bc89395750b39139e2114193607f3ca61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Nov 2020 21:58:08 +0100 Subject: [PATCH 127/127] Update news.rst --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index e92493252..0391506c4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -20,7 +20,7 @@ Scrapy 2.4.1 (2020-11-17) (:issue:`4802`, :issue:`4803`) - Fixed the output of the :command:`genspider` command showing placeholders - instead of the import part of the generated spider module (:issue:`4874`) + instead of the import path of the generated spider module (:issue:`4874`) - Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`, :issue:`4876`)