From ad6075440c0285d903ec7238354d093ea300b8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Oct 2019 19:00:03 +0200 Subject: [PATCH 001/119] Fix references to Python types in parameter type fields --- docs/topics/contracts.rst | 2 +- docs/topics/email.rst | 10 ++++----- docs/topics/exporters.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/link-extractors.rst | 14 ++++++------- docs/topics/loaders.rst | 6 +++--- docs/topics/request-response.rst | 36 ++++++++++++++++---------------- scrapy/crawler.py | 2 +- scrapy/robotstxt.py | 4 ++-- scrapy/settings/__init__.py | 33 ++++++++++++----------------- scrapy/signalmanager.py | 2 +- 11 files changed, 53 insertions(+), 60 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 62f9a743b..15443f4cc 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -85,7 +85,7 @@ override three methods: .. class:: Contract(method, \*args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 949cdc638..73b1bdc3b 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -63,10 +63,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :type smtpport: int :param smtptls: enforce using SMTP STARTTLS - :type smtptls: boolean + :type smtptls: bool :param smtpssl: enforce using a secure SSL connection - :type smtpssl: boolean + :type smtpssl: bool .. classmethod:: from_settings(settings) @@ -81,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: str or list of str + :type to: str or list :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: str or list of str + :type cc: str or list :param body: the e-mail body :type body: str @@ -97,7 +97,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. appear on the e-mail's attachment, ``mimetype`` is the mimetype of the attachment and ``file_object`` is a readable file object with the contents of the attachment - :type attachs: iterable + :type attachs: collections.abc.Iterable :param mimetype: the MIME type of the e-mail :type mimetype: str diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index a698a6a4e..da304922d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -300,7 +300,7 @@ CsvItemExporter :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. - :type include_headers_line: boolean + :type include_headers_line: bool :param join_multivalued: The char (or chars) that will be used for joining multi-valued fields, if found. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 8278e9849..657f1cc61 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. :param ignore: if given, all objects from the specified class (or tuple of classes) will be ignored. - :type ignore: class or classes tuple + :type ignore: type or tuple .. function:: get_oldest(class_name) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 713a94e10..13b9ad7a5 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -59,13 +59,13 @@ LxmlLinkExtractor :param allow: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be extracted. If not given (or empty), it will match all links. - :type allow: a regular expression (or list of) + :type allow: str or list :param deny: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be excluded (ie. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. - :type deny: a regular expression (or list of) + :type deny: str or list :param allow_domains: a single value or a list of string containing domains which will be considered for extracting the links @@ -97,7 +97,7 @@ LxmlLinkExtractor that the link's text must match in order to be extracted. If not given (or empty), it will match all links. If a list of regular expressions is given, the link will be extracted if it matches at least one. - :type restrict_text: a regular expression (or list of) + :type restrict_text: str or list :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. @@ -115,11 +115,11 @@ LxmlLinkExtractor different for requests with canonicalized and raw URLs. If you're using LinkExtractor to follow links it is more robust to keep the default ``canonicalize=False``. - :type canonicalize: boolean + :type canonicalize: bool :param unique: whether duplicate filtering should be applied to extracted links. - :type unique: boolean + :type unique: bool :param process_value: a function which receives each value extracted from the tag and attributes scanned and can modify the value and return a @@ -141,7 +141,7 @@ LxmlLinkExtractor if m: return m.group(1) - :type process_value: callable + :type process_value: collections.abc.Callable :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces @@ -150,6 +150,6 @@ LxmlLinkExtractor elements, etc., so LinkExtractor strips space chars by default. Set ``strip=False`` to turn it off (e.g. if you're extracting urls from elements or attributes which allow leading/trailing whitespaces). - :type strip: boolean + :type strip: bool .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 1c2f1da4d..4137fdd24 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -320,7 +320,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the given value using :meth:`~scrapy.utils.misc.extract_regex` method, applied before processors - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -365,7 +365,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected XPath region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -408,7 +408,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected CSS region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 727c67482..2f99a72f5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,7 +31,7 @@ Request objects a :class:`Response`. :param url: the URL of this request - :type url: string + :type url: str :param callback: the function that will be called with the response of this request (once its downloaded) as its first parameter. For more information @@ -40,10 +40,10 @@ Request objects :meth:`~scrapy.spiders.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. - :type callback: callable + :type callback: collections.abc.Callable :param method: the HTTP method of this request. Defaults to ``'GET'``. - :type method: string + :type method: str :param meta: the initial values for the :attr:`Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. @@ -54,7 +54,7 @@ Request objects ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). - :type body: str or unicode + :type body: str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -105,7 +105,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to ``str`` (if given as ``unicode``). - :type encoding: string + :type encoding: str :param priority: the priority of this request (defaults to ``0``). The priority is used by the scheduler to define the order used to process @@ -117,7 +117,7 @@ Request objects the scheduler. This is used when you want to perform an identical request multiple times, to ignore the duplicates filter. Use it with care, or you will get into crawling loops. Default to ``False``. - :type dont_filter: boolean + :type dont_filter: bool :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed @@ -125,7 +125,7 @@ Request objects as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. - :type errback: callable + :type errback: collections.abc.Callable :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -407,7 +407,7 @@ fields with form data from :class:`Response` objects. :param formdata: is a dictionary (or iterable of (key, value) tuples) containing HTML Form data which will be url-encoded and assigned to the body of the request. - :type formdata: dict or iterable of tuples + :type formdata: dict or collections.abc.Iterable The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: @@ -439,20 +439,20 @@ fields with form data from :class:`Response` objects. :type response: :class:`Response` object :param formname: if given, the form with name attribute set to this value will be used. - :type formname: string + :type formname: str :param formid: if given, the form with id attribute set to this value will be used. - :type formid: string + :type formid: str :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: string + :type formxpath: str :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: string + :type formcss: str :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. - :type formnumber: integer + :type formnumber: int :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is @@ -470,7 +470,7 @@ fields with form data from :class:`Response` objects. :param dont_click: If True, the form data will be submitted without clicking in any element. - :type dont_click: boolean + :type dont_click: bool The other parameters of this class method are passed directly to the :class:`FormRequest` constructor. @@ -558,7 +558,7 @@ dealing with JSON requests. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be set to ``'POST'`` automatically. - :type data: JSON serializable object + :type data: object :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data into JSON format. @@ -587,10 +587,10 @@ Response objects downloaded (by the Downloader) and fed to the Spiders for processing. :param url: the URL of this response - :type url: string + :type url: str :param status: the HTTP status of the response. Defaults to ``200``. - :type status: integer + :type status: int :param headers: the headers of this response. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). @@ -730,7 +730,7 @@ TextResponse objects body, it will be encoded using this encoding (remember the body attribute is always a string). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. - :type encoding: string + :type encoding: str :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ded3c082b..84acf543f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -292,7 +292,7 @@ class CrawlerProcess(CrawlerRunner): If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. - :param boolean stop_after_crawl: stop or not the reactor when all + :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..7faad308a 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -43,10 +43,10 @@ class RobotParser(with_metaclass(ABCMeta)): """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: string + :type url: str :param user_agent: User agent - :type user_agent: string + :type user_agent: str """ pass diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28c7940d..95c02021e 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -105,10 +105,9 @@ class BaseSettings(MutableMapping): Get a setting value without affecting its original type. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return self[name] if self[name] is not None else default @@ -123,10 +122,9 @@ class BaseSettings(MutableMapping): ``'0'`` will return ``False`` when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ got = self.get(name, default) try: @@ -145,10 +143,9 @@ class BaseSettings(MutableMapping): Get a setting value as an int. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return int(self.get(name, default)) @@ -157,10 +154,9 @@ class BaseSettings(MutableMapping): Get a setting value as a float. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return float(self.get(name, default)) @@ -173,10 +169,9 @@ class BaseSettings(MutableMapping): ``'one,two'`` will return a list ['one', 'two'] when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -194,10 +189,9 @@ class BaseSettings(MutableMapping): and losing all information about priority and mutability. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -209,7 +203,7 @@ class BaseSettings(MutableMapping): counterpart. :param name: name of the dictionary-like setting - :type name: string + :type name: str """ compbs = BaseSettings() compbs.update(self[name + '_BASE']) @@ -222,7 +216,7 @@ class BaseSettings(MutableMapping): the given ``name`` does not exist. :param name: the setting name - :type name: string + :type name: str """ if name not in self: return None @@ -252,14 +246,13 @@ class BaseSettings(MutableMapping): otherwise they won't have any effect. :param name: the setting name - :type name: string + :type name: str :param value: the value to associate with the setting - :type value: any :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() priority = get_settings_priority(priority) @@ -283,11 +276,11 @@ class BaseSettings(MutableMapping): uppercase variable of ``module`` with the provided ``priority``. :param module: the module or the path of the module - :type module: module object or string + :type module: types.ModuleType or str :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(module, six.string_types): @@ -316,7 +309,7 @@ class BaseSettings(MutableMapping): :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(values, six.string_types): diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..c24b16fcb 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -17,7 +17,7 @@ class SignalManager(object): section. :param receiver: the function to be connected - :type receiver: callable + :type receiver: collections.abc.Callable :param signal: the signal to connect to :type signal: object From 5479e7ecc7d30424dd2f3d9bbfb18abca765be92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:44 +0200 Subject: [PATCH 002/119] Indicate that lists of emails may be provided as a single string or as a list of strings --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 73b1bdc3b..284849c56 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -80,13 +80,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. - :param to: the e-mail recipients + :param to: the e-mail recipients as a string or as a list of strings :type to: str or list :param subject: the subject of the e-mail :type subject: str - :param cc: the e-mails to CC + :param cc: the e-mails to CC as a string or as a list of strings :type cc: str or list :param body: the e-mail body From d96b9f860b01256a84bda641b190b423eb4910b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:59 +0200 Subject: [PATCH 003/119] Use object as type for parameters that allow any value --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 95c02021e..d1a5093a6 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -108,6 +108,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return self[name] if self[name] is not None else default @@ -125,6 +126,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ got = self.get(name, default) try: @@ -146,6 +148,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return int(self.get(name, default)) @@ -157,6 +160,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return float(self.get(name, default)) @@ -172,6 +176,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -192,6 +197,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -249,6 +255,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting + :type default: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From 0946eb335a285e1f210ba1185a564699f53b17d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 17:56:21 +0100 Subject: [PATCH 004/119] =?UTF-8?q?Port=20code=20from=20Twisted=E2=80=99s?= =?UTF-8?q?=20deprecated=20HTTPClientFactory=20into=20ScrapyHTTPClientFact?= =?UTF-8?q?ory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/webclient.py | 90 ++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3fe13414a..16fd214a3 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,9 +1,9 @@ from time import time from six.moves.urllib.parse import urlparse, urlunparse, urldefrag -from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient -from twisted.internet import defer +from twisted.internet import defer, reactor +from twisted.internet.protocol import ClientFactory from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached @@ -93,18 +93,30 @@ class ScrapyHTTPPageGetter(HTTPClient): (self.factory.url, self.factory.timeout))) -class ScrapyHTTPClientFactory(HTTPClientFactory): - """Scrapy implementation of the HTTPClientFactory overwriting the - setUrl method to make use of our Url object that cache the parse - result. - """ +class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter + waiting = 1 noisy = False followRedirect = False afterFoundGet = False + def _build_response(self, body, request): + request.meta['download_latency'] = self.headers_time-self.start_time + status = int(self.status) + headers = Headers(self.response_headers) + respcls = responsetypes.from_args(headers=headers, url=self._url) + return respcls(url=self._url, status=status, headers=headers, body=body) + + def _set_connection_attributes(self, request): + parsed = urlparse_cached(request) + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) + proxy = request.meta.get('proxy') + if proxy: + self.scheme, _, self.host, self.port, _ = _parse(proxy) + self.path = self.url + def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface @@ -139,21 +151,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): elif self.method == b'POST': self.headers['Content-Length'] = 0 - def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time-self.start_time - status = int(self.status) - headers = Headers(self.response_headers) - respcls = responsetypes.from_args(headers=headers, url=self._url) - return respcls(url=self._url, status=status, headers=headers, body=body) + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, self.url) - def _set_connection_attributes(self, request): - parsed = urlparse_cached(request) - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) - proxy = request.meta.get('proxy') - if proxy: - self.scheme, _, self.host, self.port, _ = _parse(proxy) - self.path = self.url + def _cancelTimeout(self, result, timeoutCall): + if timeoutCall.active(): + timeoutCall.cancel() + return result + + def buildProtocol(self, addr): + p = ClientFactory.buildProtocol(self, addr) + p.followRedirect = self.followRedirect + p.afterFoundGet = self.afterFoundGet + if self.timeout: + timeoutCall = reactor.callLater(self.timeout, p.timeout) + self.deferred.addBoth(self._cancelTimeout, timeoutCall) + return p def gotHeaders(self, headers): self.headers_time = time() self.response_headers = headers + + def gotStatus(self, version, status, message): + """ + Set the status of the request on us. + @param version: The HTTP version. + @type version: L{bytes} + @param status: The HTTP status code, an integer represented as a + bytestring. + @type status: L{bytes} + @param message: The HTTP status message. + @type message: L{bytes} + """ + self.version, self.status, self.message = version, status, message + + def page(self, page): + if self.waiting: + self.waiting = 0 + self.deferred.callback(page) + + def noPage(self, reason): + if self.waiting: + self.waiting = 0 + self.deferred.errback(reason) + + def clientConnectionFailed(self, _, reason): + """ + When a connection attempt fails, the request cannot be issued. If no + result has yet been provided to the result Deferred, provide the + connection failure reason as an error result. + """ + if self.waiting: + self.waiting = 0 + # If the connection attempt failed, there is nothing more to + # disconnect, so just fire that Deferred now. + self._disconnectedDeferred.callback(None) + self.deferred.errback(reason) From 42954d0df9a78e6641996fbf2eb76afdef9c1856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 20 Nov 2019 08:16:33 +0100 Subject: [PATCH 005/119] Mention that ScrapyHTTPClientFactory has Twisted code --- scrapy/core/downloader/webclient.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 16fd214a3..26726afd4 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -93,6 +93,10 @@ class ScrapyHTTPPageGetter(HTTPClient): (self.factory.url, self.factory.timeout))) +# This class used to inherit from Twisted’s +# twisted.web.client.HTTPClientFactory. When that class was deprecated in +# Twisted (https://github.com/twisted/twisted/pull/643), we merged its +# non-overriden code into this class. class ScrapyHTTPClientFactory(ClientFactory): protocol = ScrapyHTTPPageGetter From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001 From: faizan2700 Date: Sun, 2 Feb 2020 16:54:22 +0530 Subject: [PATCH 006/119] SCRAPY_CHECK will be set while running contact --- scrapy/commands/check.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 9d4437a47..09a76ca7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -78,19 +78,19 @@ class Command(ScrapyCommand): elif tested_methods: self.crawler_process.crawl(spidercls) - # start checks - if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(' * %s' % method) - else: - start = time.time() - self.crawler_process.start() - stop = time.time() + # start checks + if opts.list: + for spider, methods in sorted(contract_reqs.items()): + if not methods and not opts.verbose: + continue + print(spider) + for method in sorted(methods): + print(' * %s' % method) + else: + start = time.time() + self.crawler_process.start() + stop = time.time() - result.printErrors() - result.printSummary(start, stop) - self.exitcode = int(not result.wasSuccessful()) + result.printErrors() + result.printSummary(start, stop) + self.exitcode = int(not result.wasSuccessful()) From e5b23f4b00962df76d8302ebf869ef4a4319e142 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 18 Mar 2020 11:26:59 +0300 Subject: [PATCH 007/119] fix #4250: add batch deliveries --- scrapy/extensions/feedexport.py | 54 +++++++++++++++++++++-------- scrapy/settings/default_settings.py | 1 + 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 998d2a5d1..906f99fee 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -241,6 +241,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') + self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -250,19 +251,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): uri = uri % self._get_uri_params(spider, feed['uri_params']) - storage = self._get_storage(uri) - file = storage.open(spider) - exporter = self._get_exporter( - file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], - ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) - self.slots.append(slot) - if slot.store_empty: - slot.start_exporting() + self.slots.append(self._start_new_batch(None, uri, feed, spider)) def close_spider(self, spider): deferred_list = [] @@ -285,11 +274,48 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None + def _start_new_batch(self, previous_batch_slot, uri, feed, spider): + """ + Redirect the output data stream to a new file. + Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + """ + if previous_batch_slot is not None: + previous_batch_slot.exporter.finish_exporting() + previous_batch_slot.storage.store(previous_batch_slot.file) + storage = self._get_storage(uri) + file = storage.open(spider) + exporter = self._get_exporter( + file=file, + format=feed['format'], + fields_to_export=feed['fields'], + encoding=feed['encoding'], + indent=feed['indent'] + ) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + if slot.store_empty: + slot.start_exporting() + return slot + + def _get_uri_of_partial(self, slot, feed, spider): + """Get uri for each partial using datetime.now().isoformat()""" + uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.' + uri = uri + datetime.now().isoformat() + '.' + feed['format'] + return uri + def item_scraped(self, item, spider): - for slot in self.slots: + slots = [] + for idx, slot in enumerate(self.slots): slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 + if self.storage_batch and slot.itemcount % self.storage_batch == 0: + uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider) + slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) + self.feeds[uri] = self.feeds[slot.uri] + self.feeds.pop(slot.uri) + self.slots[idx] = None + self.slots = [slot for slot in self.slots if slot is not None] + self.slots.extend(slots) def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 077317c81..690e044c5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,6 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } +FEED_STORAGE_BATCH = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From 8b4566ff93843cdf17ada069dc09261a99971d26 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 18 Mar 2020 14:21:21 +0300 Subject: [PATCH 008/119] fix wrong name of first file in partial deliveries --- scrapy/extensions/feedexport.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 906f99fee..4f7c6bf07 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -249,6 +249,8 @@ class FeedExporter: raise NotConfigured def open_spider(self, spider): + if self.storage_batch: + self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()} for uri, feed in self.feeds.items(): uri = uri % self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch(None, uri, feed, spider)) @@ -296,11 +298,11 @@ class FeedExporter: slot.start_exporting() return slot - def _get_uri_of_partial(self, slot, feed, spider): + def _get_uri_of_partial(self, template_uri, feed, spider): """Get uri for each partial using datetime.now().isoformat()""" - uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.' - uri = uri + datetime.now().isoformat() + '.' + feed['format'] - return uri + template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params'])) + uri_name = template_uri.split('.')[0] + return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"]) def item_scraped(self, item, spider): slots = [] @@ -309,11 +311,12 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 if self.storage_batch and slot.itemcount % self.storage_batch == 0: - uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider) + uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider) slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) self.feeds[uri] = self.feeds[slot.uri] self.feeds.pop(slot.uri) self.slots[idx] = None + self.slots = [slot for slot in self.slots if slot is not None] self.slots.extend(slots) From 0723e3f4f9777a87d0df3b2e2fddfeac9099dd3b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 19 Mar 2020 21:17:02 +0300 Subject: [PATCH 009/119] add batch_id, add error if uri is specified incorrectly --- scrapy/extensions/feedexport.py | 73 ++++++++++++++++++++--------- scrapy/settings/default_settings.py | 2 +- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4f7c6bf07..38b25bf4a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -180,14 +180,16 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri): self.file = file self.exporter = exporter self.storage = storage # feed params - self.uri = uri + self.batch_id = batch_id self.format = format self.store_empty = store_empty + self.template_uri = template_uri + self.uri = uri # flags self.itemcount = 0 self._exporting = False @@ -241,19 +243,28 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH') + self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured + if not self._batch_deliveries_supported(uri): + raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured def open_spider(self, spider): - if self.storage_batch: - self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()} for uri, feed in self.feeds.items(): - uri = uri % self._get_uri_params(spider, feed['uri_params']) - self.slots.append(self._start_new_batch(None, uri, feed, spider)) + batch_id = 1 + uri_params = self._get_uri_params(spider, feed['uri_params']) + uri_params['batch_id'] = batch_id + self.slots.append(self._start_new_batch( + previous_batch_slot=None, + uri=uri % uri_params, + feed=feed, + spider=spider, + batch_id=batch_id, + template_uri=uri + )) def close_spider(self, spider): deferred_list = [] @@ -276,10 +287,17 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider): + def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store + to get the file properly closed. + :param uri: uri of the new batch to start + :param feed: dict with parameters of feed + :param spider: user spider + :param batch_id: sequential batch id starting at 1 + :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: previous_batch_slot.exporter.finish_exporting() @@ -293,30 +311,30 @@ class FeedExporter: encoding=feed['encoding'], indent=feed['indent'] ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri) if slot.store_empty: slot.start_exporting() return slot - def _get_uri_of_partial(self, template_uri, feed, spider): - """Get uri for each partial using datetime.now().isoformat()""" - template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params'])) - uri_name = template_uri.split('.')[0] - return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"]) - def item_scraped(self, item, spider): slots = [] for idx, slot in enumerate(self.slots): slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - if self.storage_batch and slot.itemcount % self.storage_batch == 0: - uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider) - slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) - self.feeds[uri] = self.feeds[slot.uri] - self.feeds.pop(slot.uri) + if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: + batch_id = slot.batch_id + 1 + uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) + uri_params['batch_id'] = batch_id + self.slots.append(self._start_new_batch( + previous_batch_slot=slot, + uri=slot.template_uri % uri_params, + feed=self.feeds[slot.template_uri], + spider=spider, + batch_id=batch_id, + template_uri=slot.template_uri + )) self.slots[idx] = None - self.slots = [slot for slot in self.slots if slot is not None] self.slots.extend(slots) @@ -335,6 +353,17 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) + def _batch_deliveries_supported(self, uri): + """ + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s + to distinguish different files of partial output + """ + if not self.storage_batch_size: + return True + if '%(time)s' in uri or '%(batch_id)s' in uri: + return True + logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + def _storage_supported(self, uri): scheme = urlparse(uri).scheme if scheme in self.storages: @@ -364,7 +393,7 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') + ts = datetime.utcnow().isoformat().replace(':', '-') params['time'] = ts uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 690e044c5..7f90a2280 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH = 0 +FEED_STORAGE_BATCH_SIZE = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From d11411b402ae68874c6ccc2883836be0b9cf8326 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sat, 21 Mar 2020 10:48:13 +0300 Subject: [PATCH 010/119] fix comments --- scrapy/extensions/feedexport.py | 31 ++++++++++++++++++----------- scrapy/settings/default_settings.py | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 38b25bf4a..ab0a0de37 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) @@ -243,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE') + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -263,7 +262,7 @@ class FeedExporter: feed=feed, spider=spider, batch_id=batch_id, - template_uri=uri + template_uri=uri, )) def close_spider(self, spider): @@ -290,7 +289,7 @@ class FeedExporter: def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store to get the file properly closed. :param uri: uri of the new batch to start @@ -309,9 +308,18 @@ class FeedExporter: format=feed['format'], fields_to_export=feed['fields'], encoding=feed['encoding'], - indent=feed['indent'] + indent=feed['indent'], + ) + slot = _FeedSlot( + file=file, + exporter=exporter, + storage=storage, + uri=uri, + format=feed['format'], + store_empty=feed['store_empty'], + batch_id=batch_id, + template_uri=template_uri, ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri) if slot.store_empty: slot.start_exporting() return slot @@ -326,13 +334,13 @@ class FeedExporter: batch_id = slot.batch_id + 1 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) uri_params['batch_id'] = batch_id - self.slots.append(self._start_new_batch( + slots.append(self._start_new_batch( previous_batch_slot=slot, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, batch_id=batch_id, - template_uri=slot.template_uri + template_uri=slot.template_uri, )) self.slots[idx] = None self.slots = [slot for slot in self.slots if slot is not None] @@ -358,11 +366,10 @@ class FeedExporter: If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_size: + if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri: return True - if '%(time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + return False def _storage_supported(self, uri): scheme = urlparse(uri).scheme diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7f90a2280..c3463a505 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = 0 +FEED_STORAGE_BATCH_SIZE = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From 39d0d13d3f7bd671d5b29646b209c62e23373fab Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 26 Mar 2020 14:18:35 +0300 Subject: [PATCH 011/119] Add partial deliveries tests --- tests/test_feedexport.py | 195 +++++++++++++++++++++++++++++++-------- 1 file changed, 159 insertions(+), 36 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c5589e52f..1ebe44e12 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -6,6 +6,7 @@ import shutil import string import tempfile import warnings +from abc import ABC, abstractmethod from io import BytesIO from pathlib import Path from string import ascii_letters, digits @@ -21,8 +22,9 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter -from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage, +from scrapy.extensions.feedexport import (BlockingFeedStorage, FeedExporter, FileFeedStorage, FTPFeedStorage, IFeedStorage, S3FeedStorage, StdoutFeedStorage) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -76,6 +78,7 @@ class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -129,6 +132,7 @@ class BlockingFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -390,23 +394,63 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): pass -class FeedExportTest(unittest.TestCase): +class FeedExportTestBase(ABC, unittest.TestCase): + __test__ = False class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() baz = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + return os.path.join(self.temp_dir, inter_dir, filename) + def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - def _random_temp_filename(self): - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - return os.path.join(self.temp_dir, filename) + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @abstractmethod + def run_and_export(self, spider_cls, settings): + pass + + +class FeedExportTest(FeedExportTestBase): + __test__ = True @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -417,7 +461,6 @@ class FeedExportTest(unittest.TestCase): urljoin('file:', pathname2url(str(file_path))): feed for file_path, feed in FEEDS.items() } - content = {} try: with MockServer() as s: @@ -435,35 +478,6 @@ class FeedExportTest(unittest.TestCase): defer.returnValue(content) - @defer.inlineCallbacks - def exported_data(self, items, settings): - """ - Return exported data which a spider yielding ``items`` would return. - """ - class TestSpider(scrapy.Spider): - name = 'testspider' - - def parse(self, response): - for item in items: - yield item - - data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) - - @defer.inlineCallbacks - def exported_no_data(self, settings): - """ - Return exported data which a spider yielding no ``items`` would return. - """ - class TestSpider(scrapy.Spider): - name = 'testspider' - - def parse(self, response): - pass - - data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) - @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -970,3 +984,112 @@ class FeedExportTest(unittest.TestCase): } data = yield self.exported_no_data(settings) self.assertEqual(data['csv'], b'') + + +class PartialDeliveriesTest(FeedExportTestBase): + __test__ = True + _file_mark = '_%(time)s_#%(batch_id)s' + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + urljoin('file:', file_path): feed + for file_path, feed in FEEDS.items() + } + from collections import defaultdict + content = defaultdict(list) + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for path, feed in FEEDS.items(): + dir_name = os.path.dirname(path) + for file in sorted(os.listdir(dir_name)): + with open(os.path.join(dir_name, file), 'rb') as f: + data = f.read() + content[feed['format']].append(data) + finally: + pass + defer.returnValue(content) + + @defer.inlineCallbacks + def assertPartialExported(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, + }, + }) + data = yield self.exported_data(items, settings) + data['jl'] = b''.join(data['jl']) + parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] + + rows = [{k: v for k, v in row.items() if v} for row in rows] + self.assertEqual(rows, parsed) + + @defer.inlineCallbacks + def test_partial_deliveries(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + rows = [ + {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, + {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, + {'foo': 'bar3', 'baz': 'quux3'} + ] + settings = { + 'FEED_STORAGE_BATCH_SIZE': 1 + } + yield self.assertPartialExported(items, rows, settings=settings) + + def test_wrong_path(self): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'xml'}, + }, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + crawler = get_crawler(settings_dict=settings) + self.assertRaises(NotConfigured, FeedExporter, crawler) + + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + for fmt in ('json', 'jsonlines', 'xml', 'csv'): + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + data = yield self.exported_no_data(settings) + data[fmt] = b''.join(data[fmt]) + self.assertEqual(data[fmt], b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[]'), + ('jsonlines', b''), + ('xml', b'\n'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORE_EMPTY': True, + 'FEED_EXPORT_INDENT': None, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + data = yield self.exported_no_data(settings) + data[fmt] = b''.join(data[fmt]) + self.assertEqual(data[fmt], expctd) From ffa8a533e74478a5c81fbf453f2c65601bb1d244 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sat, 28 Mar 2020 11:40:16 +0300 Subject: [PATCH 012/119] Set batch_id in _get_uri_params --- scrapy/extensions/feedexport.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ab0a0de37..06ea6c5b2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -253,15 +253,12 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - batch_id = 1 - uri_params = self._get_uri_params(spider, feed['uri_params']) - uri_params['batch_id'] = batch_id + uri_params = self._get_uri_params(spider, feed['uri_params'], None) self.slots.append(self._start_new_batch( previous_batch_slot=None, uri=uri % uri_params, feed=feed, spider=spider, - batch_id=batch_id, template_uri=uri, )) @@ -286,7 +283,7 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): + def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. @@ -295,12 +292,15 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param batch_id: sequential batch id starting at 1 :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: + previous_batch_id = previous_batch_slot.batch_id previous_batch_slot.exporter.finish_exporting() previous_batch_slot.storage.store(previous_batch_slot.file) + else: + previous_batch_id = 0 + storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter( @@ -317,7 +317,7 @@ class FeedExporter: uri=uri, format=feed['format'], store_empty=feed['store_empty'], - batch_id=batch_id, + batch_id=previous_batch_id + 1, template_uri=template_uri, ) if slot.store_empty: @@ -331,15 +331,12 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: - batch_id = slot.batch_id + 1 - uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) - uri_params['batch_id'] = batch_id + uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) slots.append(self._start_new_batch( previous_batch_slot=slot, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, - batch_id=batch_id, template_uri=slot.template_uri, )) self.slots[idx] = None @@ -396,12 +393,12 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params): + def _get_uri_params(self, spider, uri_params, slot): params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().isoformat().replace(':', '-') - params['time'] = ts + params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 + params['time'] = datetime.utcnow().isoformat().replace(':', '-') uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params From 963580463b96315eb58319e6d35b4cd52672371a Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 15 Apr 2020 20:14:33 +0300 Subject: [PATCH 013/119] Update tests --- tests/test_feedexport.py | 203 ++++++++++++++++++++++++++++++--------- 1 file changed, 159 insertions(+), 44 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ebe44e12..c6cd867b1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ import string import tempfile import warnings from abc import ABC, abstractmethod +from collections import defaultdict from io import BytesIO from pathlib import Path from string import ascii_letters, digits @@ -444,10 +445,31 @@ class FeedExportTestBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def assertExported(self, items, header, rows, settings=None, ordered=True): + yield self.assertExportedCsv(items, header, rows, settings, ordered) + yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) + yield self.assertExportedMarshal(items, rows, settings) + yield self.assertExportedMultiple(items, rows, settings) + @abstractmethod def run_and_export(self, spider_cls, settings): pass + def _load_until_eof(self, data, load_func): + result = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result + class FeedExportTest(FeedExportTestBase): __test__ = True @@ -478,6 +500,22 @@ class FeedExportTest(FeedExportTestBase): defer.returnValue(content) + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -543,18 +581,6 @@ class FeedExportTest(FeedExportTestBase): json_rows = json.loads(to_unicode(data['json'])) self.assertEqual(rows, json_rows) - def _load_until_eof(self, data, load_func): - result = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} @@ -583,15 +609,6 @@ class FeedExportTest(FeedExportTestBase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) - @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) - @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -615,7 +632,7 @@ class FeedExportTest(FeedExportTestBase): }, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], b'') + self.assertEqual(b'', data[fmt]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -635,7 +652,7 @@ class FeedExportTest(FeedExportTestBase): 'FEED_EXPORT_INDENT': None, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], expctd) + self.assertEqual(expctd, data[fmt]) @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -734,7 +751,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'\nTest\xd6'.encode('utf-8'), + 'xml': u'\nTest\xd6'.encode( + 'utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } @@ -751,7 +769,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'\nTest\xd6'.encode('latin-1'), + 'xml': u'\nTest\xd6'.encode( + 'latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } @@ -772,7 +791,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'\n\n \n FOO\n \n'.encode('latin-1'), + 'xml': u'\n\n \n FOO\n \n'.encode( + 'latin-1'), 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } @@ -988,7 +1008,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time)s_#%(batch_id)s' + _file_mark = '_%(time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -999,7 +1019,6 @@ class PartialDeliveriesTest(FeedExportTestBase): urljoin('file:', file_path): feed for file_path, feed in FEEDS.items() } - from collections import defaultdict content = defaultdict(list) try: with MockServer() as s: @@ -1014,26 +1033,120 @@ class PartialDeliveriesTest(FeedExportTestBase): data = f.read() content[feed['format']].append(data) finally: - pass + self.tearDown() defer.returnValue(content) @defer.inlineCallbacks - def assertPartialExported(self, items, rows, settings=None): + def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - data = yield self.exported_data(items, settings) - data['jl'] = b''.join(data['jl']) - parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] - + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] rows = [{k: v for k, v in row.items() if v} for row in rows] - self.assertEqual(rows, parsed) + data = yield self.exported_data(items, settings) + for batch in data['jl']: + got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def test_partial_deliveries(self): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + data = yield self.exported_data(items, settings) + for batch in data['csv']: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + self.assertEqual(list(header), got_batch.fieldnames) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, list(got_batch)) + + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + # JSON + json_rows = rows.copy() + for batch in data['json']: + got_batch = json.loads(batch) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import pickle + for batch in data['pickle']: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import marshal + for batch in data['marshal']: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_export_items(self): items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1042,14 +1155,16 @@ class PartialDeliveriesTest(FeedExportTestBase): rows = [ {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, - {'foo': 'bar3', 'baz': 'quux3'} + {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_SIZE': 2 } - yield self.assertPartialExported(items, rows, settings=settings) + header = self.MyItem.fields.keys() + yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): + """If path without %(time)s or %(batch_id)s an exception must be raised""" settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1069,8 +1184,8 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEED_STORAGE_BATCH_SIZE': 1 } data = yield self.exported_no_data(settings) - data[fmt] = b''.join(data[fmt]) - self.assertEqual(data[fmt], b'') + data = dict(data) + self.assertEqual(b'', data[fmt][0]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -1088,8 +1203,8 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_SIZE': 1, } data = yield self.exported_no_data(settings) - data[fmt] = b''.join(data[fmt]) - self.assertEqual(data[fmt], expctd) + data = dict(data) + self.assertEqual(expctd, data[fmt][0]) From cac1f3a6adedc32977e0fb1830917a5e7d758bef Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 16 Apr 2020 10:06:56 +0300 Subject: [PATCH 014/119] Update documentation --- docs/topics/feed-exports.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9e5968a29..0bba03a7c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,6 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` + * :setting:`FEED_EXPORT_BATCH_SIZE` .. currentmodule:: scrapy.extensions.feedexport @@ -429,3 +430,37 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + +.. setting:: FEED_EXPORT_BATCH_SIZE + +FEED_EXPORT_BATCH_SIZE +---------------------- +Default: ``None`` + +An integer number which represent number of scraped items stored in each output +file. Whenever the number of items exceeds this setting, a new file +creates and output redirects to it. The name of the new file will be selected +based on timestamp when the feed is being created and/or batch sequence number. +Therefore you must specify %(time)s or %(batch_id)s or both in the file path. + +* ``%(time)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_id)s`` - gets replaced by sequence number of batch + +For instance:: + + FEED_EXPORT_BATCH_SIZE=100 + +Your request can be like:: + + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json + +The result directory tree of above can be like:: + +->projectname +-->dirname +--->1-filename2020-03-28T14-45-08.237134.json +--->2-filename2020-03-28T14-45-09.148903.json +--->3-filename2020-03-28T14-45-10.046092.json + +Where first and second files contain exactly 100 items. The last one contains +<= 100 items. \ No newline at end of file From 5980ae72c6cb177f47fbb41d17837e8d98d50025 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 16 Apr 2020 10:13:39 +0300 Subject: [PATCH 015/119] Some minor fixes and refactoring --- tests/test_feedexport.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 60e19d1df..e97e50e8e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -428,7 +428,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): yield item data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) + return data @defer.inlineCallbacks def exported_no_data(self, settings): @@ -443,7 +443,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): pass data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) + return data @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): @@ -735,8 +735,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'\nTest\xd6'.encode( - 'utf-8'), + 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } @@ -753,8 +752,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'\nTest\xd6'.encode( - 'latin-1'), + 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } @@ -775,8 +773,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'\n\n \n FOO\n \n'.encode( - 'latin-1'), + 'xml': u'\n\n \n FOO\n \n'.encode('latin-1'), 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } @@ -1148,7 +1145,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """If path without %(time)s or %(batch_id)s an exception must be raised""" + """If path is without %(time)s or %(batch_id)s an exception must be raised""" settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, From ec76445dd6753074c1531571f66467eecf22b498 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sat, 18 Apr 2020 09:29:23 +0300 Subject: [PATCH 016/119] Update tests --- tests/test_feedexport.py | 66 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e97e50e8e..8e03a91c8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1128,6 +1128,7 @@ class PartialDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items(self): + """ Test partial deliveries in all supported formats """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1145,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """If path is without %(time)s or %(batch_id)s an exception must be raised""" + """ If path is without %(time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1189,3 +1190,66 @@ class PartialDeliveriesTest(FeedExportTestBase): data = yield self.exported_no_data(settings) data = dict(data) self.assertEqual(expctd, data[fmt][0]) + + @defer.inlineCallbacks + def test_export_multiple_configs(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + + formats = { + 'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), + u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [u'\n\n \n FOO\n \n'.encode('latin-1'), + u'\n\n \n FOO1\n \n'.encode('latin-1')], + 'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', + 'indent': 0, + 'fields': ['bar'], + 'encoding': 'utf-8', + }, + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { + 'format': 'xml', + 'indent': 2, + 'fields': ['foo'], + 'encoding': 'latin-1', + }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'csv', + 'indent': None, + 'fields': ['bar', 'foo'], + 'encoding': 'utf-8', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(time)s replaced with the current date. + """ + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), '%(time)s'): { + 'format': 'json', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + data = yield self.exported_data(items, settings) + self.assertEqual(len(items) + 1, len(data['json'])) From f0f1be76d1e6cef65ac9a01d13c5d5060a03f648 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Mon, 27 Apr 2020 09:56:57 +0300 Subject: [PATCH 017/119] Using time_id instead of time as a timestamp --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 11 ++++++----- tests/test_feedexport.py | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0bba03a7c..2017be78f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file creates and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time)s or %(batch_id)s or both in the file path. +Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path. -* ``%(time)s`` - gets replaced by a timestamp when the feed is being created +* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 06ea6c5b2..72baa6269 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -292,7 +292,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: previous_batch_id = previous_batch_slot.batch_id @@ -360,12 +360,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -397,8 +397,9 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) + params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') + params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - params['time'] = datetime.utcnow().isoformat().replace(':', '-') uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e03a91c8..da759917a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time)s_#%(batch_id)s_' + _file_mark = '_%(time_id)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time)s replaced with the current date. + So %(time_id)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time)s'): { + os.path.join(self._random_temp_filename(), '%(time_id)s'): { 'format': 'json', }, }, From 2eee6c81017e08bb492da560bc73c03f4f375fcc Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Mon, 27 Apr 2020 09:58:14 +0300 Subject: [PATCH 018/119] Documentation spelling fix --- docs/topics/feed-exports.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2017be78f..6c463fc27 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -438,10 +438,10 @@ FEED_EXPORT_BATCH_SIZE Default: ``None`` An integer number which represent number of scraped items stored in each output -file. Whenever the number of items exceeds this setting, a new file -creates and output redirects to it. The name of the new file will be selected +file. Whenever the number of items exceeds this setting, a new file is +created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path. +Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. * ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch From 204737042ac6672eee73c975d0bd6735893d684c Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Mon, 27 Apr 2020 12:52:18 +0300 Subject: [PATCH 019/119] Extract the slot closing functionality to the function; minor changes --- scrapy/extensions/feedexport.py | 56 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72baa6269..fe6061c33 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -255,7 +255,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): uri_params = self._get_uri_params(spider, feed['uri_params'], None) self.slots.append(self._start_new_batch( - previous_batch_slot=None, + batch_id=1, uri=uri % uri_params, feed=feed, spider=spider, @@ -265,42 +265,38 @@ class FeedExporter: def close_spider(self, spider): deferred_list = [] for slot in self.slots: - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) - slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} - d = defer.maybeDeferred(slot.storage.store, slot.file) - d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, - extra={'spider': spider})) - d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + d = self._close_slot(slot, spider) deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri): + def _close_slot(self, slot, spider): + if not slot.itemcount and not slot.store_empty: + # We need to call slot.storage.store nonetheless to get the file + # properly closed. + return defer.maybeDeferred(slot.storage.store, slot.file) + slot.finish_exporting() + logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" + log_args = {'format': slot.format, + 'itemcount': slot.itemcount, + 'uri': slot.uri} + d = defer.maybeDeferred(slot.storage.store, slot.file) + d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, + extra={'spider': spider})) + d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, + exc_info=failure_to_exc_info(f), + extra={'spider': spider})) + return d + + def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. - :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store - to get the file properly closed. + :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri """ - if previous_batch_slot is not None: - previous_batch_id = previous_batch_slot.batch_id - previous_batch_slot.exporter.finish_exporting() - previous_batch_slot.storage.store(previous_batch_slot.file) - else: - previous_batch_id = 0 - storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter( @@ -317,7 +313,7 @@ class FeedExporter: uri=uri, format=feed['format'], store_empty=feed['store_empty'], - batch_id=previous_batch_id + 1, + batch_id=batch_id, template_uri=template_uri, ) if slot.store_empty: @@ -330,10 +326,12 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) + self._close_slot(slot, spider) slots.append(self._start_new_batch( - previous_batch_slot=slot, + batch_id=slot.batch_id + 1, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, From 3f9874fac9f93c0956afa5975d7b2bbb21816894 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 1 May 2020 11:52:16 +0300 Subject: [PATCH 020/119] Add test s3 export --- tests/test_feedexport.py | 70 ++++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 71 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index da759917a..9fc39c3a6 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1253,3 +1253,73 @@ class PartialDeliveriesTest(FeedExportTestBase): } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) + + @defer.inlineCallbacks + def test_s3_export(self): + """ + Test export of items into s3 bucket. + S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini + to perform this test: + [testenv] + setenv = + AWS_SECRET_ACCESS_KEY = ABCD + AWS_ACCESS_KEY_ID = ABCD + S3_TEST_BUCKET_NAME = ABCD + """ + try: + import boto3 + except ImportError: + raise unittest.SkipTest("S3FeedStorage requires boto3") + + assert_aws_environ() + s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + if not s3_test_bucket_name: + raise unittest.SkipTest("No S3 BUCKET available for testing") + + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + prefix = 'tmp/{filename}'.format(filename=filename) + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + bucket_name=s3_test_bucket_name, prefix=prefix + ) + storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) + settings = { + 'FEEDS': { + s3_test_file_uri: { + 'format': 'json', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + s3 = boto3.resource('s3') + my_bucket = s3.Bucket(s3_test_bucket_name) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + TestSpider.start_urls = [s.url('/')] + yield runner.crawl(TestSpider) + + for file_uri in my_bucket.objects.filter(Prefix=prefix): + content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) + if not content and not items: + break + content = json.loads(content.decode('utf-8')) + expected_batch, items = items[:batch_size], items[batch_size:] + self.assertEqual(expected_batch, content) diff --git a/tox.ini b/tox.ini index cd118c921..c77fae1f0 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = # Extras botocore>=1.3.23 Pillow>=3.4.2 + boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sat, 2 May 2020 01:21:03 +0300 Subject: [PATCH 021/119] Change time_id to batch_time --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6c463fc27..2106b41f5 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. -* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fe6061c33..a262f5d18 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -295,7 +295,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -396,7 +396,7 @@ class FeedExporter: for k in dir(spider): params[k] = getattr(spider, k) params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') + params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9fc39c3a6..2217bb4ed 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time_id)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time_id)s replaced with the current date. + So %(batch_time)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time_id)s'): { + os.path.join(self._random_temp_filename(), '%(batch_time)s'): { 'format': 'json', }, }, @@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) From 286fca733f23fa41165edcdcc7ab7593cc6b074f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 6 May 2020 16:20:33 +0200 Subject: [PATCH 022/119] Fix parameter name, broken by copy-pasting --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 8f6fd3e6a..99ffa0dc9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -248,7 +248,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting - :type default: object + :type value: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From bbd9d05dbff7abb6ab3a5ea575d75cfe88cb2ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 May 2020 11:44:43 +0200 Subject: [PATCH 023/119] request-response.rst: review type references around body mentions --- docs/topics/request-response.rst | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1638c202d..4fec70e13 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,10 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str + :param body: the request body. If a string is passed, it is converted to + bytes using *encoding*, which defaults to ``utf-8``. If not passed or + ``None`` is passed, an empty bytes array is stored. + :type body: bytes :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +104,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes if given as a string. :type encoding: str :param priority: the priority of this request (defaults to ``0``). @@ -159,7 +157,7 @@ Request objects .. attribute:: Request.body - A str that contains the request body. + The request body as bytes. This attribute is read-only. To change the body of a Request use :meth:`replace`. @@ -598,7 +596,7 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. To access the decoded text as str you can use + :param body: the response body. To access the decoded text as a string, use ``response.text`` from an encoding-aware :ref:`Response subclass `, such as :class:`TextResponse`. @@ -646,10 +644,10 @@ Response objects .. attribute:: Response.body - The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use - :attr:`TextResponse.text` (only available in :class:`TextResponse` - and subclasses). + The response body as bytes. + + If you want the body as a string, use :attr:`TextResponse.text` (only + available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -768,10 +766,10 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode - body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the - encoding will be looked up in the response headers and body instead. + response. If you create a :class:`TextResponse` object with a string as + body, it will be converted to bytes encoded using this encoding. If + *encoding* is ``None`` (default), the encoding will be looked up in the + response headers and body instead. :type encoding: str :class:`TextResponse` objects support the following attributes in addition @@ -779,7 +777,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -787,8 +785,8 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding + ``str(response.body)`` is not a correct way to convert the response + body into a string: you would be using the system default encoding (typically ``ascii``) instead of the response encoding. From c6746f0e381a44f3d66efad86a4aca87805138bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 9 May 2020 15:51:11 +0200 Subject: [PATCH 024/119] =?UTF-8?q?bytes=20array=20=E2=86=92=20bytes=20obj?= =?UTF-8?q?ect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4fec70e13..ad6c10b6e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -53,7 +53,7 @@ Request objects :param body: the request body. If a string is passed, it is converted to bytes using *encoding*, which defaults to ``utf-8``. If not passed or - ``None`` is passed, an empty bytes array is stored. + ``None`` is passed, an empty :class:`bytes` object is stored. :type body: bytes :param headers: the headers of this request. The dict values can be strings From e07708e3744fb26fa72042720b33deab59331cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 9 May 2020 15:54:31 +0200 Subject: [PATCH 025/119] request-response: update the consequences of str(b'') --- docs/topics/request-response.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ad6c10b6e..397632932 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -786,8 +786,7 @@ TextResponse objects .. note:: ``str(response.body)`` is not a correct way to convert the response - body into a string: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + body into a string: ``str(b'')`` returns ``"b''"``. .. attribute:: TextResponse.encoding From b5684909d1cb01ad138a389caa750485b51f79cf Mon Sep 17 00:00:00 2001 From: Jacty Date: Mon, 11 May 2020 11:18:25 +0800 Subject: [PATCH 026/119] Unnecessary update when value is None When value is None, it is not necessary to invoke update and run other methods and conditions to make the code complicated there. --- scrapy/settings/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b9a13c018..f28fbfaf9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -83,7 +83,8 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - self.update(values, priority) + if values is not None: + self.update(values, priority) def __getitem__(self, opt_name): if opt_name not in self: From 33ab0a36635fbd45debbc44584002bd7a4ef7fed Mon Sep 17 00:00:00 2001 From: Jacty Date: Wed, 13 May 2020 06:11:07 +0800 Subject: [PATCH 027/119] Update __init__.py --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28fbfaf9..0425b48b3 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -83,7 +83,7 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - if values is not None: + if values: self.update(values, priority) def __getitem__(self, opt_name): From 2327ecead085a41d1a71a70a12eb988bbf982268 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 13 May 2020 22:50:04 +0300 Subject: [PATCH 028/119] Rename FEED_STORAGE_BATCH_SIZE to FEED_STORAGE_BATCH_ITEM_COUNT --- docs/topics/feed-exports.rst | 8 ++++---- scrapy/extensions/feedexport.py | 10 +++++----- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 28 ++++++++++++++-------------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2106b41f5..917240d4d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_SIZE` + * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_EXPORT_BATCH_SIZE +.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT -FEED_EXPORT_BATCH_SIZE +FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` @@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. For instance:: - FEED_EXPORT_BATCH_SIZE=100 + FEED_STORAGE_BATCH_ITEM_COUNT=100 Your request can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a262f5d18..5bc946634 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -290,7 +290,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. + Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -326,7 +326,7 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) self._close_slot(slot, spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s + If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c3463a505..5a7dc533e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = None +FEED_STORAGE_BATCH_ITEM_COUNT = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2217bb4ed..1a21eeba9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 2 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=settings) @@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] with MockServer() as s: runner = CrawlerRunner(Settings(settings)) From 8662d3587df74841d4ea640c0432446569e59262 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 13 May 2020 23:41:01 +0300 Subject: [PATCH 029/119] Documentation and code refactoring --- docs/topics/feed-exports.rst | 21 ++++++++++++--------- scrapy/extensions/feedexport.py | 7 ++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 917240d4d..0f15044b3 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -437,24 +437,27 @@ FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` -An integer number which represent number of scraped items stored in each output +An integer number that represents the number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is -created and output redirects to it. The name of the new file will be selected -based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. +created and the output is redirected to it. The name of the new file will be selected +based on the timestamp when the feed is being created and/or on the batch sequence number. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created -* ``%(batch_id)s`` - gets replaced by sequence number of batch +(e.g. `2020-03-28T14-45-08.237134`) -For instance:: +* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch +(e.g. `2` for the second file) + +For instance, if your settings include:: FEED_STORAGE_BATCH_ITEM_COUNT=100 -Your request can be like:: +And your :command:`crawl` command line is:: scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json -The result directory tree of above can be like:: +The resulting directory tree of above can be like:: ->projectname -->dirname @@ -462,5 +465,5 @@ The result directory tree of above can be like:: --->2-filename2020-03-28T14-45-09.148903.json --->3-filename2020-03-28T14-45-10.046092.json -Where first and second files contain exactly 100 items. The last one contains +Where the first and second files contain exactly 100 items. The last one contains <= 100 items. \ No newline at end of file diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5bc946634..4c9362f3a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,6 +25,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -337,9 +338,9 @@ class FeedExporter: spider=spider, template_uri=slot.template_uri, )) - self.slots[idx] = None - self.slots = [slot for slot in self.slots if slot is not None] - self.slots.extend(slots) + else: + slots.append(slot) + self.slots = slots def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) From 69c005f013eb0dc000611853e66371fad17dea9d Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 14 May 2020 10:35:56 +0300 Subject: [PATCH 030/119] Documentation indent fix --- docs/topics/feed-exports.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 42c4e2267..dfeea5b7f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -444,10 +444,10 @@ based on the timestamp when the feed is being created and/or on the batch sequen Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created -(e.g. `2020-03-28T14-45-08.237134`) + (e.g. `2020-03-28T14-45-08.237134`) * ``%(batch_id)s`` - gets replaced by the batch sequence number of batch -(e.g. `2` for the second file) + (e.g. `2` for the second file) For instance, if your settings include:: From 1cdcf8b08b8f1e68c5b107b6ae39b2da1aedd245 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 15 May 2020 19:46:36 +0300 Subject: [PATCH 031/119] Minor fixes --- docs/topics/feed-exports.rst | 19 ++++++++++--------- scrapy/extensions/feedexport.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dfeea5b7f..638733b6a 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT ----------------------------- Default: ``None`` -An integer number that represents the number of scraped items stored in each output -file. Whenever the number of items exceeds this setting, a new file is -created and the output is redirected to it. The name of the new file will be selected -based on the timestamp when the feed is being created and/or on the batch sequence number. -Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. +If assigned an integer number higher than ``0``, Scrapy generates multiple output files +storing up to the specified number of items in each output file. + +When generating multiple output files, you must use at least one of the following +placeholders in :setting:`FEED_URI` to indicate how the different output file names are +generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created - (e.g. `2020-03-28T14-45-08.237134`) + (e.g. ``2020-03-28T14-45-08.237134``) * ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. `2` for the second file) + (e.g. ``2`` for the second file) For instance, if your settings include:: @@ -457,7 +458,7 @@ And your :command:`crawl` command line is:: scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json -The resulting directory tree of above can be like:: +The command line above can generate a directory tree like:: ->projectname -->dirname @@ -466,4 +467,4 @@ The resulting directory tree of above can be like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -<= 100 items. \ No newline at end of file +100 items or fever. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4c9362f3a..3d691c580 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): self.file = file self.exporter = exporter self.storage = storage @@ -188,7 +188,7 @@ class _FeedSlot: self.batch_id = batch_id self.format = format self.store_empty = store_empty - self.template_uri = template_uri + self.uri_template = uri_template self.uri = uri # flags self.itemcount = 0 @@ -260,7 +260,7 @@ class FeedExporter: uri=uri % uri_params, feed=feed, spider=spider, - template_uri=uri, + uri_template=uri, )) def close_spider(self, spider): @@ -288,7 +288,7 @@ class FeedExporter: extra={'spider': spider})) return d - def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): + def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. @@ -296,7 +296,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -315,7 +315,7 @@ class FeedExporter: format=feed['format'], store_empty=feed['store_empty'], batch_id=batch_id, - template_uri=template_uri, + uri_template=uri_template, ) if slot.store_empty: slot.start_exporting() @@ -329,14 +329,14 @@ class FeedExporter: slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: - uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) + uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( batch_id=slot.batch_id + 1, - uri=slot.template_uri % uri_params, - feed=self.feeds[slot.template_uri], + uri=slot.uri_template % uri_params, + feed=self.feeds[slot.uri_template], spider=spider, - template_uri=slot.template_uri, + uri_template=slot.uri_template, )) else: slots.append(slot) From 10ae1a284f759b541d086e3d1a13cda96b6e2040 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 15 May 2020 22:50:54 +0300 Subject: [PATCH 032/119] Minor fixes --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 2 +- tox.ini | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 638733b6a..6f7db20c4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -467,4 +467,4 @@ The command line above can generate a directory tree like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -100 items or fever. +100 items or fewer. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3d691c580..cc26ae173 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,11 +243,11 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._batch_deliveries_supported(uri): + if not self._settings_are_valid(uri): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -328,7 +328,7 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_size and slot.itemcount == self.storage_batch_size: + if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -357,12 +357,12 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _batch_deliveries_supported(self, uri): + def _settings_are_valid(self, uri): """ If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: + if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d1374f291..88f9a5933 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(data['csv'], b'') -class PartialDeliveriesTest(FeedExportTestBase): +class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)s_' diff --git a/tox.ini b/tox.ini index 6dd944dff..7507a14a6 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 - boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From a7d070f3bb350cbe1f7b580350d5f491f59d47d8 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Mon, 18 May 2020 22:25:29 +0300 Subject: [PATCH 033/119] Change log level to error --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cc26ae173..ce7fc372d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -364,7 +364,7 @@ class FeedExporter: """ if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') + logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): From 677e619d3761e6669c247786bb95822ce38c8080 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 21 May 2020 14:57:03 +0300 Subject: [PATCH 034/119] Fix too long lines --- scrapy/extensions/feedexport.py | 4 +++- tests/test_feedexport.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ce7fc372d..1f745be98 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -364,7 +364,9 @@ class FeedExporter: """ if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' + ) return False def _storage_supported(self, uri): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08ee24768..fecb17e29 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1204,12 +1204,20 @@ class BatchDeliveriesTest(FeedExportTestBase): items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] formats = { - 'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')], - 'xml': [u'\n\n \n FOO\n \n'.encode('latin-1'), - u'\n\n \n FOO1\n \n'.encode('latin-1')], - 'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), - u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), + '[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [ + ( + '\n' + '\n \n FOO\n \n' + ).encode('latin-1'), + ( + '\n' + '\n \n FOO1\n \n' + ).encode('latin-1') + ], + 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], } settings = { From dd96f94e8cc1517b7021e35e46cbdc92580c6333 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 22 May 2020 23:30:33 +0300 Subject: [PATCH 035/119] Push datetime.utcnow() to its own variable --- scrapy/extensions/feedexport.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1f745be98..45c2971a6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -398,8 +398,9 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') + utc_now = datetime.utcnow() + params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') + params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) From c3cee74fd401e6a6307b5eb1786e532bb2cd5aa8 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 26 Jun 2020 18:45:21 +0300 Subject: [PATCH 036/119] Change default value of FEED_STORAGE_BATCH_ITEM_COUNT to 0 --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 20 ++++++++++---------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 866ce78eb..0b37e9a7d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_STORAGE_BATCH_ITEM_COUNT ----------------------------- -Default: ``None`` +Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files storing up to the specified number of items in each output file. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1331782e3..e06116acd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,7 +243,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5a7dc533e..810acd5a3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = None +FEED_STORAGE_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1a6a5624b..578cd396b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=settings) + yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ @@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase): bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) - settings = { + settings = Settings({ 'FEEDS': { s3_test_file_uri: { 'format': 'json', }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, - } + }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) From 88a52198b90faa0129c8e05072197cdffbb9653b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sat, 27 Jun 2020 11:50:26 +0300 Subject: [PATCH 037/119] Add batch_item_count support in FEEDS setting --- scrapy/extensions/feedexport.py | 5 +++-- tests/test_feedexport.py | 39 ++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e06116acd..2312c994e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) @@ -337,7 +336,9 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: + if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ + and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', + self.storage_batch_item_count): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 578cd396b..3bc0c083c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase): '\n \n FOO1\n \n' ).encode('latin-1') ], - 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), - 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), + 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], + 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1348,9 +1349,16 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { 'format': 'csv', 'indent': None, - 'fields': ['bar', 'foo'], + 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'jsonlines', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + 'batch_item_count': 0, + }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) + @defer.inlineCallbacks + def test_batch_item_count_feeds_setting(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + + formats = { + 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), + '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { + 'format': 'jsonlines', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + 'batch_item_count': 1, + }, + }, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + @defer.inlineCallbacks def test_batch_path_differ(self): """ From 05c2587c6a32b84a94463f2b1187e49f94957aa2 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sun, 28 Jun 2020 09:45:45 +0300 Subject: [PATCH 038/119] Docs update and tiny fixes --- docs/topics/feed-exports.rst | 1 + tests/test_feedexport.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b37e9a7d..3da56821e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,6 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` +* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3bc0c083c..542cce70f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { 'format': 'jsonlines', 'indent': None, 'fields': ['foo', 'bar'], @@ -1423,8 +1423,8 @@ class BatchDeliveriesTest(FeedExportTestBase): [testenv] setenv = AWS_SECRET_ACCESS_KEY = ABCD - AWS_ACCESS_KEY_ID = ABCD - S3_TEST_BUCKET_NAME = ABCD + AWS_ACCESS_KEY_ID = EFGH + S3_TEST_BUCKET_NAME = IJKL """ try: import boto3 From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Wed, 1 Jul 2020 11:54:39 +0300 Subject: [PATCH 039/119] Minor updates --- docs/topics/feed-exports.rst | 4 ++-- scrapy/extensions/feedexport.py | 34 +++++++++++++++++---------------- scrapy/utils/conf.py | 4 ++++ tests/test_feedexport.py | 23 ++++++---------------- tests/test_utils_conf.py | 4 ++++ 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 3da56821e..0b659f30e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu storing up to the specified number of items in each output file. When generating multiple output files, you must use at least one of the following -placeholders in :setting:`FEED_URI` to indicate how the different output file names are +placeholders in the feed URI to indicate how the different output file names are generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created @@ -457,7 +457,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2312c994e..5908987a3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,6 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -253,7 +252,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params'], None) + uri_params = self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, @@ -299,7 +298,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. + Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -331,14 +330,15 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] - for idx, slot in enumerate(self.slots): + for slot in self.slots: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ - and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', - self.storage_batch_item_count): + if ( + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -369,15 +369,17 @@ class FeedExporter: def _settings_are_valid(self, uri): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s - to distinguish different files of partial output + If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error( - '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' - ) - return False + for uri_template, values in self.feeds.items(): + if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + ) + return False + return True def _storage_supported(self, uri): scheme = urlparse(uri).scheme @@ -404,7 +406,7 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params, slot): + def _get_uri_params(self, spider, uri_params, slot=None): params = {} for k in dir(spider): params[k] = getattr(spider, k) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5921f82bf..0e02f0f28 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) + out.setdefault( + "batch_item_count", + out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) + ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 542cce70f..db14b20b9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): ], 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', - 'indent': None, - 'fields': ['foo', 'bar'], - 'encoding': 'utf-8', - 'batch_item_count': 0, - }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] - + items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] formats = { - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), - '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), + '[{"foo": "FOO1"}]'.encode('utf-8')], } - settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', 'indent': None, - 'fields': ['foo', 'bar'], 'encoding': 'utf-8', 'batch_item_count': 1, }, diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e5d3ef582..95ec2b64a 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": None, + "batch_item_count": 2, }) From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 2 Jul 2020 12:38:08 +0300 Subject: [PATCH 040/119] Change setting name. Add leading zeroes to batch_id. Minor fixes. --- docs/topics/feed-exports.rst | 19 +++++++++-------- scrapy/extensions/feedexport.py | 24 +++++++++++++--------- scrapy/settings/default_settings.py | 2 +- scrapy/utils/conf.py | 5 +---- tests/test_feedexport.py | 32 ++++++++++++++--------------- tests/test_utils_conf.py | 4 ++-- 6 files changed, 45 insertions(+), 41 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b659f30e..56efa80a7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` + * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING @@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT +.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT -FEED_STORAGE_BATCH_ITEM_COUNT +FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- Default: ``0`` @@ -448,16 +448,19 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. ``2`` for the second file) +* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. +By replacing ``x`` with an integer you set the number of leading zeroes to prevent +inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: + ``%(batch_id)01d`` for the second batch gets replaced by ``2`` + ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` For instance, if your settings include:: - FEED_STORAGE_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT=100 And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5908987a3..adb6ea2e4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst import logging import os +import re import sys import warnings from datetime import datetime @@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -245,7 +247,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._settings_are_valid(uri): + if not self._settings_are_valid(): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -298,7 +300,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified + Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -334,10 +336,10 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one + # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one if ( - self.feeds[slot.uri_template]['batch_item_count'] - and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) @@ -367,16 +369,18 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _settings_are_valid(self, uri): + def _settings_are_valid(self): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): logger.error( - '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' + ''.format(uri_template) ) return False return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 810acd5a3..0016bbe1b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0e02f0f28..64f9c824b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -113,12 +113,9 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() + out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) - out.setdefault( - "batch_item_count", - out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) - ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index db14b20b9..d20b40e2f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(batch_time)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)02d_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 95ec2b64a..f3ef36127 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { From 6454d456d2bcab0828aba6d81d98f7393ab7e04d Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 3 Jul 2020 08:29:54 +0300 Subject: [PATCH 041/119] Make check of placeholder less strict --- docs/topics/feed-exports.rst | 11 ++++++----- scrapy/extensions/feedexport.py | 4 ++-- tests/test_feedexport.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 56efa80a7..0bb5f1733 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -448,11 +448,12 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. -By replacing ``x`` with an integer you set the number of leading zeroes to prevent -inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: - ``%(batch_id)01d`` for the second batch gets replaced by ``2`` - ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` +* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. + + Use :ref:`printf-style string formatting ` to + alter the number format. For example, to make the batch ID a 5-digit + number by introducing leading zeroes as needed, use ``%(batch_id)05d`` + (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``). For instance, if your settings include:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index adb6ea2e4..e15c1a09c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -375,9 +375,9 @@ class FeedExporter: %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): + if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' ''.format(uri_template) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d20b40e2f..4e0b867a4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, From f1020e0e6af064ab31b812c25bda6b0f08827222 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Mon, 6 Jul 2020 15:40:53 +0300 Subject: [PATCH 042/119] Tiny changes --- scrapy/extensions/feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e15c1a09c..21177b1b0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -305,7 +305,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -372,13 +372,13 @@ class FeedExporter: def _settings_are_valid(self): """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain - %(batch_time)s or %(batch_id)s to distinguish different files of partial output + %(batch_time)s or %(batch_id)d to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' ''.format(uri_template) ) From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Tue, 7 Jul 2020 15:23:29 +0530 Subject: [PATCH 043/119] Added basic `scrapy check` tests --- tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_command_check.py diff --git a/tests/test_command_check.py b/tests/test_command_check.py new file mode 100644 index 000000000..52005a4c5 --- /dev/null +++ b/tests/test_command_check.py @@ -0,0 +1,96 @@ +from os.path import join, abspath + +from tests.test_commands import CommandTest + + +class CheckCommandTest(CommandTest): + + command = 'check' + + def setUp(self): + super(CheckCommandTest, self).setUp() + self.spider_name = 'check_spider' + self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + + def _write_contract(self, contracts, parse_def): + with open(self.spider, 'w') as file: + file.write(f""" +import scrapy + +class CheckSpider(scrapy.Spider): + name = '{self.spider_name}' + start_urls = ['http://example.com'] + + def parse(self, response, **cb_kwargs): + \"\"\" + @url http://www.amazon.com/s?field-keywords=selfish+gene + {contracts} + \"\"\" + {parse_def} + """) + + def _test_contract(self, contracts='', parse_def='pass'): + self._write_contract(contracts, parse_def) + p, out, err = self.proc('check') + self.assertIn('OK', err) + self.assertEqual(p.returncode, 0) + + def test_check_returns_requests_contract(self): + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(contracts, parse_def) + + def test_check_returns_items_contract(self): + contracts = """ + @returns items 1 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_cb_kwargs_contract(self): + contracts = """ + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_check_scrapes_contract(self): + contracts = """ + @scrapes key1 key2 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_all_default_contracts(self): + contracts = """ + @returns items 1 + @returns requests 1 + @scrapes key1 key2 + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + yield scrapy.Request(url='http://next-url.com') + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_SCRAPY_CHECK_set(self): + parse_def = """ + import os + if not os.environ.get('SCRAPY_CHECK'): + raise Exception('SCRAPY_CHECK not set') + """ + self._test_contract(parse_def=parse_def) From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Tue, 7 Jul 2020 15:24:33 +0530 Subject: [PATCH 044/119] Ignore flake8 E501 for `scrapy check` tests` --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index bae68cd3a..97320a008 100644 --- a/pytest.ini +++ b/pytest.ini @@ -173,6 +173,7 @@ flake8-ignore = tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 + tests/test_command_check.py E501 tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 8 Jul 2020 17:13:57 +0530 Subject: [PATCH 045/119] Convert f-string to .format() --- tests/test_command_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 52005a4c5..72acd817c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(f""" + file.write(""" import scrapy class CheckSpider(scrapy.Spider): - name = '{self.spider_name}' + name = '{0}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://www.amazon.com/s?field-keywords=selfish+gene - {contracts} + {1} \"\"\" - {parse_def} - """) + {2} + """.format(self.spider_name, contracts, parse_def)) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 8 Jul 2020 19:48:42 +0530 Subject: [PATCH 046/119] Update url contract value --- tests/test_command_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 72acd817c..f27f526a3 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider): def parse(self, response, **cb_kwargs): \"\"\" - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://example.com {1} \"\"\" {2} @@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider): def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) p, out, err = self.proc('check') + self.assertNotIn('F', out) self.assertIn('OK', err) self.assertEqual(p.returncode, 0) From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Fri, 10 Jul 2020 18:22:43 +0530 Subject: [PATCH 047/119] Update pytest.ini --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 92c5bcb75..ca8191f42 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,4 +39,5 @@ flake8-ignore = scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 \ No newline at end of file + tests/test_loader.py E741 + From 8bdcdb0a76e3780681dcfbbd4a0eee62e2bb05b1 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 16 Jul 2020 09:13:54 +0300 Subject: [PATCH 048/119] Add quotes to example in docs --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0bb5f1733..7e91b365d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -461,7 +461,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json + scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json' The command line above can generate a directory tree like:: From 41263f61c6de8048023ba4c80e062f56b21e5a19 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Thu, 16 Jul 2020 18:41:45 +0300 Subject: [PATCH 049/119] Change single quotes to double in example in docs --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7e91b365d..fdc6e7cba 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -461,7 +461,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json' + scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json" The command line above can generate a directory tree like:: From 86f7ac2f2b5d58e0b2588fa2aa4c777a8decf299 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 17 Jul 2020 17:48:25 +0300 Subject: [PATCH 050/119] Try to fix error at Windows --- tests/test_feedexport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 129b7fc0b..cc124624d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1204,6 +1204,8 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) + if not os.path.exists(str(dir_name)): + continue for file in sorted(os.listdir(dir_name)): with open(os.path.join(dir_name, file), 'rb') as f: data = f.read() From 3e0492741d93b05c464457b3b128a2b0d24c994b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sun, 19 Jul 2020 00:10:29 +0300 Subject: [PATCH 051/119] Another try to fix test errors on Windows --- tests/test_feedexport.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cc124624d..c49b2e92f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1190,9 +1190,14 @@ class BatchDeliveriesTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', file_path): feed + build_url(file_path): feed for file_path, feed in FEEDS.items() } content = defaultdict(list) @@ -1204,8 +1209,6 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) - if not os.path.exists(str(dir_name)): - continue for file in sorted(os.listdir(dir_name)): with open(os.path.join(dir_name, file), 'rb') as f: data = f.read() From ece4fa6c7cf7fdce9e5e77f56b209941ad401f53 Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:21:45 +0000 Subject: [PATCH 052/119] Fix ignored testcase: boto is never installed --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 37384081a..21da9fdcd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -182,9 +182,9 @@ class S3FeedStorageTest(unittest.TestCase): create=True) def test_parse_credentials(self): try: - import boto # noqa: F401 + import botocore # noqa: F401 except ImportError: - raise unittest.SkipTest("S3FeedStorage requires boto") + raise unittest.SkipTest("S3FeedStorage requires botocore") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) From 234c8b8c50a2db0d6d71455c382957099f4c0dcc Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:31:56 +0000 Subject: [PATCH 053/119] Removing deprecated S3FeedStorage without AWS keys instancing. --- scrapy/extensions/feedexport.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72a34ae0d..e793c12dc 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,23 +94,10 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None): - # BEGIN Backward compatibility for initialising without keys (and - # without using from_crawler) - no_defaults = access_key is None and secret_key is None - if no_defaults: - from scrapy.utils.project import get_project_settings - settings = get_project_settings() - if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: - warnings.warn( - "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " - "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] - # END Backward compatibility + no_keys = access_key is None and secret_key is None + if no_keys: + raise NotConfigured('%s is missing AWS credentials' % + self.__class__.__name__) u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key From 98e8086d1b3da8906d1395f453b1299b4e3b499e Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 19:21:09 +0000 Subject: [PATCH 054/119] Adapt S3FeedStorage testcase --- tests/test_feedexport.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 21da9fdcd..34a67fff3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,6 +24,7 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -176,10 +177,6 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - @mock.patch('scrapy.utils.project.get_project_settings', - new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key', - 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}), - create=True) def test_parse_credentials(self): try: import botocore # noqa: F401 @@ -205,12 +202,9 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Backward compatibility for initialising without settings - with warnings.catch_warnings(record=True) as w: - storage = S3FeedStorage('s3://mybucket/export.csv') - self.assertEqual(storage.access_key, 'conf_key') - self.assertEqual(storage.secret_key, 'conf_secret') - self.assertTrue('without AWS keys' in str(w[-1].message)) + # Instantiate without credentials + with self.assertRaises(NotConfigured): + S3FeedStorage('s3://mybucket/export.csv') @defer.inlineCallbacks def test_store(self): From 2829cd4268d22a328acd64a6816c479b1fe21f39 Mon Sep 17 00:00:00 2001 From: nyov Date: Tue, 17 Mar 2020 10:19:13 +0000 Subject: [PATCH 055/119] Allow use without credentials --- scrapy/extensions/feedexport.py | 4 ---- tests/test_feedexport.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e793c12dc..68d6533d3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -94,10 +94,6 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None): - no_keys = access_key is None and secret_key is None - if no_keys: - raise NotConfigured('%s is missing AWS credentials' % - self.__class__.__name__) u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 34a67fff3..656bb515f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,7 +24,6 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -202,9 +201,6 @@ class S3FeedStorageTest(unittest.TestCase): aws_credentials['AWS_SECRET_ACCESS_KEY']) self.assertEqual(storage.access_key, 'uri_key') self.assertEqual(storage.secret_key, 'uri_secret') - # Instantiate without credentials - with self.assertRaises(NotConfigured): - S3FeedStorage('s3://mybucket/export.csv') @defer.inlineCallbacks def test_store(self): From 430d22e46ebb67aba39005365e2e68d834d8fb86 Mon Sep 17 00:00:00 2001 From: Artur Shellunts Date: Tue, 21 Jul 2020 23:39:04 +0200 Subject: [PATCH 056/119] Remove not used import warnings --- tests/test_feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 656bb515f..e80e07554 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,7 +5,6 @@ import random import shutil import string import tempfile -import warnings from io import BytesIO from logging import getLogger from pathlib import Path From a6c1d79b7cc3bc2c408eab356bbbf99a0536f110 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Tue, 28 Jul 2020 11:53:05 +0300 Subject: [PATCH 057/119] pep8 tiny changes --- docs/topics/feed-exports.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2c9774b55..dd4eb3c61 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -453,6 +453,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- + Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files @@ -474,7 +475,7 @@ generated: For instance, if your settings include:: - FEED_EXPORT_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT = 100 And your :command:`crawl` command line is:: From 52658539370c442e63102a3208781335953cdf53 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 28 Jul 2020 06:15:14 -0300 Subject: [PATCH 058/119] Use ItemAdapter.field_names when writing header in CsvItemExporter (#4668) --- scrapy/exporters.py | 8 +- tests/test_exporters.py | 203 ++++++++++++++++++++++++++++------------ 2 files changed, 146 insertions(+), 65 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 712572673..0aba1c904 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter): def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: - if isinstance(item, dict): - # for dicts try using fields of the first item - self.fields_to_export = list(item.keys()) - else: - # use fields declared in Item - self.fields_to_export = list(item.fields.keys()) + # use declared field names, or keys if the item is a dict + self.fields_to_export = ItemAdapter(item).field_names() row = list(self._build_row(self.fields_to_export)) self.csv_writer.writerow(row) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b27380309..25da54a65 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -8,6 +8,7 @@ from io import BytesIO from datetime import datetime import lxml.etree +from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -23,10 +24,37 @@ class TestItem(Item): age = Field() +def custom_serializer(value): + return str(int(value) + 2) + + +class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + +try: + from dataclasses import make_dataclass, field +except ImportError: + TestDataClass = None + CustomFieldDataclass = None +else: + TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) + CustomFieldDataclass = make_dataclass( + "CustomFieldDataclass", + [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] + ) + + class BaseItemExporterTest(unittest.TestCase): + item_class = TestItem + custom_field_item_class = CustomFieldItem + def setUp(self): - self.i = TestItem(name=u'John\xa3', age=u'22') + if self.item_class is None: + raise unittest.SkipTest("item class is None") + self.i = self.item_class(name=u'John\xa3', age=u'22') self.output = BytesIO() self.ie = self._get_exporter() @@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, exported_dict) + self.assertEqual(self.i, self.item_class(**exported_dict)) def _get_nonstring_types_item(self): return { @@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase): self.assertItemExportWorks(self.i) def test_export_dict_item(self): - self.assertItemExportWorks(dict(self.i)) + self.assertItemExportWorks(ItemAdapter(self.i).asdict()) def test_serialize_field(self): - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) + a = ItemAdapter(self.i) + res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) self.assertEqual(res, u'John\xa3') - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) + res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) self.assertEqual(res, u'22') def test_fields_to_export(self): @@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age=u'22') - + i = self.custom_field_item_class(name=u'John\xa3', age=u'22') + a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') + + +class BaseItemExporterDataclassTest(BaseItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass class PythonItemExporterTest(BaseItemExporterTest): @@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i3 = self.item_class(name=u'Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) @@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = TestItem(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i1 = self.item_class(name=u'Joseph', age='22') + i2 = self.item_class(name=u'Maria', age=[i1]) + i3 = self.item_class(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( @@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i3 = self.item_class(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( @@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = TestItem(name=u'John\xa3', age=u'22') + value = self.item_class(name=u'John\xa3', age=u'22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class PythonItemExporterDataclassTest(PythonItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PprintItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest): self._assert_expected_item(eval(self.output.getvalue())) +class PprintItemExporterDataclassTest(PprintItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PickleItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self._assert_expected_item(pickle.loads(self.output.getvalue())) def test_export_multiple_items(self): - i1 = TestItem(name='hello', age='world') - i2 = TestItem(name='bye', age='world') + i1 = self.item_class(name='hello', age='world') + i2 = self.item_class(name='bye', age='world') f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() @@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i2) ie.finish_exporting() f.seek(0) - self.assertEqual(pickle.load(f), i1) - self.assertEqual(pickle.load(f), i2) + self.assertEqual(self.item_class(**pickle.load(f)), i1) + self.assertEqual(self.item_class(**pickle.load(f)), i2) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.loads(fp.getvalue()), item) +class PickleItemExporterDataclassTest(PickleItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class MarshalItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest): self.assertEqual(marshal.load(fp), item) +class MarshalItemExporterDataclassTest(MarshalItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_header_export_all(self): self.assertExportResult( item=self.i, - fields_to_export=self.i.fields.keys(), + fields_to_export=ItemAdapter(self.i).field_names(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( - item=dict(self.i), + item=ItemAdapter(self.i).asdict(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, fields_to_export=['age'], @@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_header_export_two_items(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: output = BytesIO() ie = CsvItemExporter(output) ie.start_exporting() @@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest): b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, include_headers_line=False, @@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest): ) +class CsvItemExporterDataclassTest(CsvItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class XmlItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def xmltuple(elem): children = list(elem.iterchildren()) if children: - return [(child.tag, sorted(xmltuple(child))) - for child in children] + return [(child.tag, sorted(xmltuple(child))) for child in children] else: return [(elem.tag, [(elem.text, ())])] @@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - TestItem(name=[u'John\xa3', u'Doe']), - ( - b'\n' - b'John\xc2\xa3Doe' - ) + self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]), + b"""\n + + + John\xc2\xa3Doe + 123 + + + """ ) def test_nested_item(self): - i1 = TestItem(name=u'foo\xa3hoo', age='22') + i1 = dict(name=u'foo\xa3hoo', age='22') i2 = dict(name=u'bar', age=i1) - i3 = TestItem(name=u'buz', age=i2) + i3 = self.item_class(name=u'buz', age=i2) self.assertExportResult( i3, @@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = TestItem(name=u'foo') + i1 = dict(name=u'foo') i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = TestItem(name=u'buz', age=[i1, i2]) + i3 = self.item_class(name=u'buz', age=[i1, i2]) self.assertExportResult( i3, @@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest): ) +class XmlItemExporterDataclassTest(XmlItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonLinesItemExporterTest(BaseItemExporterTest): _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} @@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, dict(self.i)) + self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i3 = self.item_class(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonItemExporterTest(JsonLinesItemExporterTest): _expected_nested = [JsonLinesItemExporterTest._expected_nested] @@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [dict(self.i)]) + self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(item), dict(item)]) + self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) def test_two_items(self): self.assertTwoItemsExported(self.i) def test_two_dict_items(self): - self.assertTwoItemsExported(dict(self.i)) + self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i1 = self.item_class(name=u'Joseph\xa3', age='22') + i2 = self.item_class(name=u'Maria', age=i1) + i3 = self.item_class(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} + expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): i1 = dict(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) + i2 = self.item_class(name=u'Maria', age=i1) i3 = dict(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [item]) -class CustomItemExporterTest(unittest.TestCase): +class JsonItemExporterDataclassTest(JsonItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + +class CustomExporterItemTest(unittest.TestCase): + + item_class = TestItem + + def setUp(self): + if self.item_class is None: + raise unittest.SkipTest("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): @@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase): else: return super(CustomItemExporter, self).serialize_field(field, name, value) - i = TestItem(name=u'John', age='22') + i = self.item_class(name=u'John', age='22') + a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') i2 = {'name': u'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') +class CustomExporterDataclassTest(CustomExporterItemTest): + + item_class = TestDataClass + + if __name__ == '__main__': unittest.main() From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 10:16:18 +0530 Subject: [PATCH 059/119] Code cleanup scrapy.utils.python.WeakKeyCache #4684 --- scrapy/utils/python.py | 12 ------------ tests/test_utils_python.py | 18 +----------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9204977cf..7a393925e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..b23ae2e52 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,7 +9,7 @@ from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) @@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - wk = WeakKeyCache(lambda k: next(_values)) - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) def test_get_func_args(self): def f1(a, b, c): From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 18:05:33 +0530 Subject: [PATCH 060/119] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- tests/test_utils_python.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b23ae2e52..5a53d89e4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,9 +1,7 @@ import functools -import gc import operator import platform import unittest -from itertools import count from sys import version_info from warnings import catch_warnings @@ -12,7 +10,6 @@ from scrapy.utils.python import ( get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) - __doctests__ = ['scrapy.utils.python'] @@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_get_func_args(self): def f1(a, b, c): pass From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Thu, 30 Jul 2020 12:25:21 +0530 Subject: [PATCH 062/119] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- scrapy/utils/python.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7a393925e..c8f921ff3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024): In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing the start position of the match, and the ending (regarding the entire text). """ + def _chunk_iter(): offset = len(text) while True: @@ -158,6 +159,7 @@ def memoizemethod_noargs(method): if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] + return new_method @@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes): return True +@deprecated +class WeakKeyCache: + + def __init__(self, default_factory): + self.default_factory = default_factory + self._weakdict = weakref.WeakKeyDictionary() + + def __getitem__(self, key): + if key not in self._weakdict: + self._weakdict[key] = self.default_factory(key) + return self._weakdict[key] + + @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" From 890b2138a605af2bfbf340a0d48d9d83c4cda53b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 13:39:30 +0200 Subject: [PATCH 063/119] Remove the u prefix from strings --- docs/_ext/scrapydocs.py | 2 +- docs/topics/loaders.rst | 6 +- docs/topics/selectors.rst | 4 +- docs/utils/linkfix.py | 2 +- scrapy/http/request/form.py | 6 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/logformatter.py | 2 +- tests/test_cmdline/__init__.py | 2 +- tests/test_downloader_handlers.py | 6 +- tests/test_downloadermiddleware_cookies.py | 16 +- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_redirect.py | 6 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_exporters.py | 84 +++---- tests/test_feedexport.py | 8 +- tests/test_http_headers.py | 6 +- tests/test_http_request.py | 90 ++++---- tests/test_http_response.py | 72 +++--- tests/test_item.py | 42 ++-- tests/test_linkextractors.py | 180 +++++++-------- tests/test_loader.py | 124 +++++----- tests/test_loader_deprecated.py | 226 +++++++++---------- tests/test_logformatter.py | 12 +- tests/test_mail.py | 8 +- tests/test_responsetypes.py | 10 +- tests/test_robotstxt_interface.py | 8 +- tests/test_selector.py | 32 +-- tests/test_spider.py | 12 +- tests/test_utils_iterators.py | 90 ++++---- tests/test_utils_python.py | 16 +- tests/test_utils_reqser.py | 2 +- tests/test_utils_template.py | 4 +- 32 files changed, 545 insertions(+), 545 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 192123473..640660943 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -17,7 +17,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): if node.tagname == 'index': # index entries for setting directives look like: - # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] return entry_type == 'pair' and info.endswith('; setting') return False diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index d0eeb4097..29d9c5805 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -237,10 +237,10 @@ metadata. Here is an example:: >>> from scrapy.loader import ItemLoader >>> il = ItemLoader(item=Product()) ->>> il.add_value('name', [u'Welcome to my', u'website']) ->>> il.add_value('price', [u'€', u'1000']) +>>> il.add_value('name', ['Welcome to my', 'website']) +>>> il.add_value('price', ['€', '1000']) >>> il.load_item() -{'name': u'Welcome to my website', 'price': u'1000'} +{'name': 'Welcome to my website', 'price': '1000'} The precedence order, for both input and output processors, is as follows: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index bb46ea80f..5014df6ac 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit: >>> from scrapy import Selector ->>> doc = u""" +>>> doc = """ ...
...
    ...
  • first item
    • @@ -765,7 +765,7 @@ extracting text elements for example. Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops:: - >>> doc = u""" + >>> doc = """ ...
    ... Kenmore White 17" Microwave ... Kenmore 17" Microwave diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 9acfc3b23..95a3f17d5 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -23,7 +23,7 @@ def main(): _contents = None # A regex that matches standard linkcheck output lines - line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') + line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0e6ceef0b..a260798ac 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={ "re": "http://exslt.org/regular-expressions"}) - values = [(k, u'' if v is None else v) + values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -168,7 +168,7 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or u'').strip() for o in selected_options] + v = [(o.get('value') or o.text or '').strip() for o in selected_options] return n, v @@ -205,7 +205,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1615d44d7..8b9f961ee 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -76,7 +76,7 @@ class LxmlParserLinkExtractor: url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) - link = Link(url, _collect_string_content(el) or u'', + link = Link(url, _collect_string_content(el) or '', nofollow=rel_has_nofollow(el.get('rel'))) links.append(link) return self._deduplicate_if_needed(links) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 219145f13..0f9e6f1cb 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -44,7 +44,7 @@ class LogFormatter: def dropped(self, item, exception, response, spider): return { 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", 'args': { 'exception': exception, 'item': item, diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index da99a6be8..591075a98 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase): 'EXTENSIONS=' + json.dumps(EXTENSIONS)) # XXX: There's gotta be a smarter way to do this... self.assertNotIn("...", settingsstr) - for char in ("'", "<", ">", 'u"'): + for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..57d4cdd6b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") @@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(response.body, b'\xbe\xd3\xbe') self.assertEqual(response.encoding, "iso-8859-7") @@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 9ccc2110b..010577415 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')}) + req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')}) + req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', cookies={'a': u'á'}) + # 3) String + req3 = Request('http://example.org', cookies={'a': 'á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')}) + req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')}) + req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', headers={'Cookie': u'a=á'}) + # 3) String + req3 = Request('http://example.org', headers={'Cookie': 'a=á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 9841d7a76..351631eb8 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase): def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128' mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None @@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') @@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 919dbed23..131332131 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_latin1_location(self): req = Request('http://scrapytest.org/first') - latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 + latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' @@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding + utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' @@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = u"""""" + html = """""" return html.format(interval, url).encode('utf-8') def test_priority_adjust(self): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index b9452a0e7..f9936baba 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = u""" + ROBOTS = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -56,7 +56,7 @@ Disallow: /some/randome/page.html self.assertIgnored(Request('http://site.local/admin/main'), middleware), self.assertIgnored(Request('http://site.local/static/'), middleware), self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), - self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) + self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 25da54a65..660c99ce1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase): def setUp(self): if self.item_class is None: raise unittest.SkipTest("item class is None") - self.i = self.item_class(name=u'John\xa3', age=u'22') + self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() @@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) - self.assertEqual(res, u'John\xa3') + self.assertEqual(res, 'John\xa3') res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) - self.assertEqual(res, u'22') + self.assertEqual(res, '22') def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')]) ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] assert isinstance(name, str) - self.assertEqual(name, u'John\xa3') + self.assertEqual(name, 'John\xa3') def test_field_custom_serializer(self): - i = self.custom_field_item_class(name=u'John\xa3', age=u'22') + i = self.custom_field_item_class(name='John\xa3', age='22') a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3') self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') @@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) self.assertEqual( exported, - {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'} ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = self.item_class(name=u'Maria', age=[i1]) - i3 = self.item_class(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = self.item_class(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=[i1]) - i3 = self.item_class(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = self.item_class(name=u'John\xa3', age=u'22') + value = self.item_class(name='John\xa3', age='22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]), + self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]), b"""\n @@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name=u'foo\xa3hoo', age='22') - i2 = dict(name=u'bar', age=i1) - i3 = self.item_class(name=u'buz', age=i2) + i1 = dict(name='foo\xa3hoo', age='22') + i2 = dict(name='bar', age=i1) + i3 = self.item_class(name='buz', age=i2) self.assertExportResult( i3, @@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name=u'foo') - i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = self.item_class(name=u'buz', age=[i1, i2]) + i1 = dict(name='foo') + i2 = dict(name='bar', v2={"egg": ["spam"]}) + i3 = self.item_class(name='buz', age=[i1, i2]) self.assertExportResult( i3, @@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} + _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} def _get_exporter(self, **kwargs): return JsonLinesItemExporter(self.output, **kwargs) @@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name=u'Joseph\xa3', age='22') - i2 = self.item_class(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name=u'Joseph\xa3', age='22') - i2 = self.item_class(name=u'Maria', age=i1) - i3 = dict(name=u'Jesus', age=i2) + i1 = dict(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = dict(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) def test_nonstring_types_item(self): @@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase): else: return super(CustomItemExporter, self).serialize_field(field, name, value) - i = self.item_class(name=u'John', age='22') + i = self.item_class(name='John', age='22') a = ItemAdapter(i) ie = CustomItemExporter() self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') - i2 = {'name': u'John', 'age': '22'} + i2 = {'name': 'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c49b2e92f..b57349848 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({'foo': u'Test\xd6'})] + items = [dict({'foo': 'Test\xd6'})] formats = { 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), @@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'})] formats = { 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})] formats = { 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] + items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})] formats = { 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), '[{"foo": "FOO1"}]'.encode('utf-8')], diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index cf3fc8496..64ff7a73d 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase): assert h.getlist('X-Forwarded-For') is not hlist def test_encode_utf8(self): - h = Headers({u'key': u'\xa3'}, encoding='utf-8') + h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] self.assertEqual(val[0], b'\xc2\xa3') def test_encode_latin1(self): - h = Headers({u'key': u'\xa3'}, encoding='latin1') + h = Headers({'key': '\xa3'}, encoding='latin1') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xa3') def test_encode_multiple(self): - h = Headers({u'key': [u'\xa3']}, encoding='utf-8') + h = Headers({'key': ['\xa3']}, encoding='utf-8') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xc2\xa3') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 63014b22d..f5cf4e798 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase): self.assertFalse(p.headers is r.headers) # headers must not be unicode - h = Headers({'key1': u'val1', u'key2': 'val2'}) - h[u'newkey'] = u'newval' + h = Headers({'key1': 'val1', 'key2': 'val2'}) + h['newkey'] = 'newval' for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: @@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase): self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") def test_url_encoding(self): - r = self.request_class(url=u"http://www.scrapy.org/price/£") + r = self.request_class(url="http://www.scrapy.org/price/£") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_query(self): - r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ") + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") # should be same as above - r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1") self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") def test_url_encoding_nonutf8_untouched(self): @@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase): # characters. Otherwise, in the future the IRI will be mapped to # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3") + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") - r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3") + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html") + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") def test_body(self): @@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase): assert isinstance(r2.body, bytes) self.assertEqual(r2.encoding, 'utf-8') # default encoding - r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') + r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) self.assertEqual(r3.body, b"Price: \xc2\xa3100") - r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1') + r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1') assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") @@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase): r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") # unicode url - r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value") + r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") def test_copy(self): @@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase): assert r4.dont_filter is False def test_method_always_str(self): - r = self.request_class("http://www.example.com", method=u"POST") + r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) def test_immutable_attributes(self): @@ -381,7 +381,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_textual_data(self): # using default encoding (utf-8) - data = {u'µ one': u'two', u'price': u'£ 100'} + data = {'µ one': 'two', 'price': '£ 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -390,7 +390,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'} + data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -406,14 +406,14 @@ class FormRequestTest(RequestTest): self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') def test_custom_encoding_textual_data(self): - data = {'price': u'£ 100'} + data = {'price': '£ 100'} r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') def test_multi_key_values(self): # using multiples values for a single key - data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} + data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') @@ -450,10 +450,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -471,14 +471,14 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_unicode(self): response = _buildresponse( - u""" + """ @@ -490,10 +490,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_duplicate_form_key(self): response = _buildresponse( @@ -685,7 +685,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, clickdata={u'name': u'clickable', u'value': u'clicked2'} + response, clickdata={'name': 'clickable', 'value': 'clicked2'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -694,21 +694,21 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_clickdata(self): response = _buildresponse( - u"""
    + """
    """) req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a3'} + response, clickdata={'name': 'price in \u00a3'} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs[u'price in \u00a3']) + self.assertTrue(fs['price in \u00a3']) def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( - u"""
    + """ @@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
    """, encoding='latin1') req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a5'} + response, clickdata={'name': 'price in \u00a5'} ) fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertTrue(fs[u'price in \u00a5']) + self.assertTrue(fs['price in \u00a5']) def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -733,7 +733,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, formname='form2', clickdata={u'name': u'clickable'} + response, formname='form2', clickdata={'name': 'clickable'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_xpath(self): response = _buildresponse(b'
    ') - r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']") fs = _qs(r) self.assertEqual(fs, {}) - xpath = u"//form[@name='\u03b1']" + xpath = "//form[@name='\u03b1']" self.assertRaisesRegex(ValueError, re.escape(xpath), self.request_class.from_response, response, formxpath=xpath) @@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=('value',)) self._test_request(params=('username', 'password'), methodname='login') self._test_request(params=('response', ), methodresponse='login') - self._test_request(params=(u'pas£',), encoding='utf-8') + self._test_request(params=('pas£',), encoding='utf-8') self._test_request(params=(None,), allow_none=1) self.assertRaises(TypeError, self._test_request) self.assertRaises(TypeError, self._test_request, params=(None,)) def test_latin1(self): - self._test_request(params=(u'pas£',), encoding='latin1') + self._test_request(params=('pas£',), encoding='latin1') class JsonRequestTest(RequestTest): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e0ca3c0e6..56d017de6 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest): def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class(u"http://www.example.com/") + resp = self.response_class("http://www.example.com/") self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str - resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/", encoding='utf-8') assert isinstance(resp.url, str) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8') self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' '\u0442\u0435\u043a\u0441\u0442') - self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') + self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body') original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') @@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest): def test_encoding(self): r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") + r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3") r3 = self.response_class("http://www.example.com", body=b"\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") @@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r5._headers_encoding(), None) self._assert_response_encoding(r5, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' - self._assert_response_values(r1, 'utf-8', u"\xa3") - self._assert_response_values(r2, 'utf-8', u"\xa3") - self._assert_response_values(r3, 'iso-8859-1', u"\xa3") - self._assert_response_values(r6, 'gb18030', u"\u2015") - self._assert_response_values(r7, 'gb18030', u"\u2015") + self._assert_response_values(r1, 'utf-8', "\xa3") + self._assert_response_values(r2, 'utf-8', "\xa3") + self._assert_response_values(r3, 'iso-8859-1', "\xa3") + self._assert_response_values(r6, 'gb18030', "\u2015") + self._assert_response_values(r7, 'gb18030', "\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, 'utf-8', u"\xa3") + self._assert_response_values(r, 'utf-8', "\xa3") def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" r = self.response_class("http://www.example.com", body=b'\xff\xfeh\x00i\x00', encoding='utf-16') - self._assert_response_values(r, 'utf-16', u"hi") + self._assert_response_values(r, 'utf-16', "hi") def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): r6 = self.response_class("http://www.example.com", @@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest): body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') self.assertIn(r6.text, { - u'WORD\ufffd\ufffd', # w3lib < 1.19.0 - u'WORD\ufffd', # w3lib >= 1.19.0 + 'WORD\ufffd\ufffd', # w3lib < 1.19.0 + 'WORD\ufffd', # w3lib >= 1.19.0 }) def test_bom_is_removed_from_body(self): @@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.text, repr(r.text) - assert u'PREFIX' in r.text, repr(r.text) - assert u'SUFFIX' in r.text, repr(r.text) + # but it should always contain the unicode replacement char ('\ufffd') + assert '\ufffd' in r.text, repr(r.text) + assert 'PREFIX' in r.text, repr(r.text) + assert 'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', body=b'\xf0value') - assert u'value' in r.text, repr(r.text) + assert 'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert u'\ufffd' in r.text, repr(r.text) + # assert '\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page" @@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest): self.assertEqual( response.selector.xpath("//title/text()").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.css("title::text").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.re("Some (.*)"), - [u'page'] + ['page'] ) def test_selector_shortcuts(self): @@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body=u'click me'.encode('utf8') + body='click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body=u'click me'.encode('cp1251') + body='click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], @@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest): def test_body_as_unicode_deprecation_warning(self): with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), u'Hello') + r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') + self.assertEqual(r1.body_as_unicode(), 'Hello') self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) @@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest): self.assertEqual( response.selector.xpath("//elem/text()").getall(), - [u'value'] + ['value'] ) def test_selector_shortcuts(self): diff --git a/tests/test_item.py b/tests/test_item.py index 60468971c..0ce78f8c0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'name' - self.assertEqual(i['name'], u'name') + i['name'] = 'name' + self.assertEqual(i['name'], 'name') def test_init(self): class TestItem(Item): @@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.__getitem__, 'name') - i2 = TestItem(name=u'john doe') - self.assertEqual(i2['name'], u'john doe') + i2 = TestItem(name='john doe') + self.assertEqual(i2['name'], 'john doe') - i3 = TestItem({'name': u'john doe'}) - self.assertEqual(i3['name'], u'john doe') + i3 = TestItem({'name': 'john doe'}) + self.assertEqual(i3['name'], 'john doe') i4 = TestItem(i3) - self.assertEqual(i4['name'], u'john doe') + self.assertEqual(i4['name'], 'john doe') - self.assertRaises(KeyError, TestItem, {'name': u'john doe', - 'other': u'foo'}) + self.assertRaises(KeyError, TestItem, {'name': 'john doe', + 'other': 'foo'}) def test_invalid_field(self): class TestItem(Item): @@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase): number = Field() i = TestItem() - i['name'] = u'John Doe' + i['name'] = 'John Doe' i['number'] = 123 itemrepr = repr(i) @@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.get_name) - i['name'] = u'lala' - self.assertEqual(i.get_name(), u'lala') - i.change_name(u'other') + i['name'] = 'lala' + self.assertEqual(i.get_name(), 'lala') + i.change_name('other') self.assertEqual(i.get_name(), 'other') def test_metaclass(self): @@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase): values = Field() i = TestItem() - i['name'] = u'John' + i['name'] = 'John' self.assertEqual(list(i.keys()), ['name']) self.assertEqual(list(i.values()), ['John']) - i['keys'] = u'Keys' - i['values'] = u'Values' + i['keys'] = 'Keys' + i['values'] = 'Values' self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name']) - self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John']) + self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John']) def test_metaclass_with_fields_attribute(self): class TestItem(Item): fields = {'new': Field(default='X')} - item = TestItem(new=u'New') + item = TestItem(new='New') self.assertSortedEqual(list(item.keys()), ['new']) - self.assertSortedEqual(list(item.values()), [u'New']) + self.assertSortedEqual(list(item.values()), ['New']) def test_metaclass_inheritance(self): class ParentItem(Item): @@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'John' - self.assertEqual(dict(i), {'name': u'John'}) + i['name'] = 'John' + self.assertEqual(dict(i), {'name': 'John'}) def test_copy(self): class TestItem(Item): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 8d4538eed..a0bafa5e5 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -31,31 +31,31 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) def test_extract_filter_allow(self): lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=('sample', ), unique=False) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) @@ -63,10 +63,10 @@ class Base: lx = self.extractor_cls(allow=('sample', ), unique=False, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') ]) @@ -74,22 +74,22 @@ class Base: lx = self.extractor_cls(allow=('sample',), unique=True, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=('google.com', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_extraction_using_single_values(self): @@ -97,27 +97,27 @@ class Base: lx = self.extractor_cls(allow='sample') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(allow_domains='google.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) lx = self.extractor_cls(deny_domains='example.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_nofollow(self): @@ -145,11 +145,11 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/about.html', text=u'About us'), - Link(url='http://example.org/follow.html', text=u'Follow this link'), - Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), - Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://example.org/about.html', text='About us'), + Link(url='http://example.org/follow.html', text='Follow this link'), + Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True), + Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ]) def test_matches(self): @@ -183,8 +183,8 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_restrict_xpaths_encoding(self): @@ -202,14 +202,14 @@ class Base: lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + [Link(url='http://example.org/about.html', text='About us\xa3')]) def test_restrict_xpaths_with_html_entities(self): html = b'

    text

    ' response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')]) + [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')]) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" @@ -217,22 +217,22 @@ class Base: response = HtmlResponse("http://example.org", body=body, encoding='gb18030') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c', fragment='', nofollow=False)]) def test_restrict_css(self): lx = self.extractor_cls(restrict_css=('#subwrapper a',)) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2') + Link(url='http://example.com/sample2.html', text='sample 2') ]) def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), restrict_css=('#subwrapper + a', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_area_tag_with_unicode_present(self): @@ -243,7 +243,7 @@ class Base: lx.extract_links(response) lx.extract_links(response) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'', + [Link(url='http://example.org/foo', text='', fragment='', nofollow=False)]) def test_encoded_url(self): @@ -251,7 +251,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_encoded_url_in_restricted_xpath(self): @@ -259,7 +259,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_ignored_extensions(self): @@ -268,7 +268,7 @@ class Base: response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), + Link(url='http://example.org/page.html', text='asd'), ]) # override denied extensions @@ -308,25 +308,25 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) @@ -344,24 +344,24 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags="area") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample1.html', text=''), ]) lx = self.extractor_cls(tags="a") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=()) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), ]) def test_tags_attrs(self): @@ -375,14 +375,14 @@ class Base: lx = self.extractor_cls(tags='div', attrs='data-url') self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) def test_xhtml(self): @@ -420,13 +420,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -436,13 +436,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -455,8 +455,8 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_ftp_links(self): @@ -467,7 +467,7 @@ class Base: response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False), ]) def test_pickle_extractor(self): @@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_link_restrict_text(self): @@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text='dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Unique regex test lx = self.extractor_cls(restrict_text=r'of.*dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Multiple regex test lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) def test_restrict_xpaths_with_html_entities(self): diff --git a/tests/test_loader.py b/tests/test_loader.py index 581183625..2ed6f365f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) + self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo']) def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) class InitializationTestMixin: @@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, l.get_css, '#name::text') def test_init_method_with_selector(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_selector_css(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) # combining/accumulating CSS selectors and XPath expressions l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta']) l.add_xpath('url', '//img/@src') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png']) def test_add_xpath_re(self): l = TestItemLoader(response=self.response) l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_replace_xpath(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//p/text()') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_xpath('name', ['//p/text()', '//div/text()']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) def test_get_xpath(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph']) - self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph') - self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_xpath('//p/text()'), ['paragraph']) + self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph') + self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta']) + self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta']) def test_replace_xpath_multi_fields(self): l = TestItemLoader(response=self.response) l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_add_css_re(self): l = TestItemLoader(response=self.response) l.add_css('name', 'div::text', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) def test_replace_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css('name', 'p::text') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_css('name', ['p::text', 'div::text']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) l.replace_css('url', 'img::attr(src)') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_get_css(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_css('p::text'), [u'paragraph']) - self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph') - self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_css('p::text'), ['paragraph']) + self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph') + self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta']) + self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta']) self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']), - [u'http://www.scrapy.org', u'/images/logo.png']) + ['http://www.scrapy.org', '/images/logo.png']) def test_replace_css_multi_fields(self): l = TestItemLoader(response=self.response) l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_replace_css_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') - self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['scrapy.org']) class SubselectorLoaderTest(unittest.TestCase): @@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath('a') l.add_xpath('url', '//footer/a/@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) nl1.replace_xpath('url', 'img/@src') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) nl2.replace_xpath('url', '@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase): l.add_xpath('url', '//footer/a/@href') self.assertEqual(l.get_output_value('url'), [ - u'/images/logo.png', - u'http://www.scrapy.org', - u'homepage', - u'http://www.scrapy.org', + '/images/logo.png', + 'http://www.scrapy.org', + 'homepage', + 'http://www.scrapy.org', ]) def test_nested_load_item(self): @@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item['name'], [u'marta']) - self.assertEqual(item['url'], [u'http://www.scrapy.org']) - self.assertEqual(item['image'], [u'/images/logo.png']) + self.assertEqual(item['name'], ['marta']) + self.assertEqual(item['url'], ['http://www.scrapy.org']) + self.assertEqual(item['image'], ['/images/logo.png']) # Functions as processors diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d0a59e8cd..eb14de14f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) def test_load_item_ignore_none_field_values(self): def validate_sku(value): @@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase): price_out = Compose(TakeFirst(), float) sku_out = Compose(TakeFirst(), validate_sku) - valid_fragment = u'SKU: 1234' - invalid_fragment = u'SKU: not available' + valid_fragment = 'SKU: 1234' + invalid_fragment = 'SKU: not available' sku_re = 'SKU: (.+)' il = MyLoader(item={}) # Should not return "sku: None". il.add_value('sku', [invalid_fragment], re=sku_re) # Should not ignore empty values. - il.add_value('name', u'') - il.add_value('price', [u'0']) + il.add_value('name', '') + il.add_value('price', ['0']) self.assertEqual(il.load_item(), { - 'name': u'', + 'name': '', 'price': 0.0, }) il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], u'1234') + self.assertEqual(il.load_item()['sku'], '1234') def test_self_referencing_loader(self): class MyLoader(ItemLoader): @@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value(self): il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.add_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.add_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe']) + self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe']) # test add object value il.add_value('summary', {'key': 1}) self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) - il.add_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) + il.add_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim']) def test_add_zero(self): il = NameItemLoader() @@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase): def test_replace_value(self): il = TestItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.replace_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Pepe']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.replace_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Pepe']) + self.assertEqual(il.get_output_value('name'), ['Pepe']) - il.replace_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Jim']) + il.replace_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Jim']) def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) - self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) - self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) + self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper)) + self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$')) + self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')) - il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') - self.assertEqual([u'foo'], il.get_collected_values('name')) - il.replace_value('name', u'name:bar', re=u'name:(.*)$') - self.assertEqual([u'bar'], il.get_collected_values('name')) + il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$') + self.assertEqual(['foo'], il.get_collected_values('name')) + il.replace_value('name', 'name:bar', re='name:(.*)$') + self.assertEqual(['bar'], il.get_collected_values('name')) def test_iter_on_input_processor_input(self): class NameFirstItemLoader(NameItemLoader): name_in = TakeFirst() il = NameFirstItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', u'marta') - il.add_value('name', [u'jose', u'pedro']) - self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) + il.add_value('name', 'marta') + il.add_value('name', ['jose', 'pedro']) + self.assertEqual(il.get_collected_values('name'), ['marta', 'jose']) def test_map_compose_filter(self): def filter_world(x): @@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Mart']) item = il.load_item() - self.assertEqual(item['name'], [u'Mart']) + self.assertEqual(item['name'], ['Mart']) def test_default_input_processor(self): il = DefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_inherited_default_input_processor(self): class InheritDefaultedItemLoader(DefaultedItemLoader): pass il = InheritDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_input_processor_inheritance(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'HTTP://scrapy.ORG') + self.assertEqual(il.get_output_value('url'), ['http://scrapy.org']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) class ChildChildItemLoader(ChildItemLoader): url_in = MapCompose(lambda v: v.upper()) summary_in = MapCompose(lambda v: v) il = ChildChildItemLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'http://scrapy.org') + self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) def test_empty_map_compose(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_identity_input_processor(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = Identity() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mARTA']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mARTA']) def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'MART']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['MART']) def test_output_processor_using_function(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): - name_out = u" ".join + name_out = " ".join il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') def test_output_processor_error(self): class TestItemLoader(ItemLoader): @@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = MapCompose(float) il = TestItemLoader() - il.add_value('name', [u'$10']) + il.add_value('name', ['$10']) try: - float(u'$10') + float('$10') except Exception as e: expected_exc_str = str(e) @@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_using_classes(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): name_out = Join() il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') class TakeFirstItemLoader(TestItemLoader): name_out = Join("
    ") il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar
    Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar
    Ta') def test_default_output_processor(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class LalaItemLoader(TestItemLoader): default_output_processor = Identity() il = LalaItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) def test_loader_context_on_declaration(self): class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key=u'val') + url_in = MapCompose(processor_with_args, key='val') il = ChildItemLoader() - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_instantiation(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor_with_args) - il = ChildItemLoader(key=u'val') - il.add_value('url', u'text') + il = ChildItemLoader(key='val') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_assign(self): @@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase): url_in = MapCompose(processor_with_args) il = ChildItemLoader() - il.context['key'] = u'val' - il.add_value('url', u'text') + il.context['key'] = 'val' + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_item_passed_to_input_processor_functions(self): @@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase): it = TestItem(name='marta') il = ChildItemLoader(item=it) - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['marta']) def test_compose_processor(self): @@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', [u'marta', u'other']) - self.assertEqual(il.get_output_value('name'), u'Mart') + il.add_value('name', ['marta', 'other']) + self.assertEqual(il.get_output_value('name'), 'Mart') item = il.load_item() - self.assertEqual(item['name'], u'Mart') + self.assertEqual(item['name'], 'Mart') def test_partial_processor(self): def join(values, sep=None, loader_context=None, ignored=None): @@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase): summary_out = Compose(partial(join, ignored='foo')) il = TestItemLoader() - il.add_value('name', [u'rabbit', u'hole']) - il.add_value('url', [u'rabbit', u'hole']) - il.add_value('summary', [u'rabbit', u'hole']) + il.add_value('name', ['rabbit', 'hole']) + il.add_value('url', ['rabbit', 'hole']) + il.add_value('summary', ['rabbit', 'hole']) item = il.load_item() - self.assertEqual(item['name'], u'rabbit+hole') - self.assertEqual(item['url'], u'rabbit.hole') - self.assertEqual(item['summary'], u'rabbithole') + self.assertEqual(item['name'], 'rabbit+hole') + self.assertEqual(item['url'], 'rabbit.hole') + self.assertEqual(item['summary'], 'rabbithole') def test_error_input_processor(self): class TestItem(Item): @@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other']) + ['marta', 'other']) def test_error_output_processor(self): class TestItem(Item): @@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(Join(), float) il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') with self.assertRaises(ValueError): il.load_item() @@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other'], Compose(float)) + ['marta', 'other'], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), u' hello world') - self.assertEqual(proc(['hello', 'world']), u'hello world') + self.assertEqual(proc(['', 'hello', 'world']), ' hello world') + self.assertEqual(proc(['hello', 'world']), 'hello world') self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): @@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase): def filter_world(x): return None if x == 'world' else x proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), - [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 7064337ad..b771e7d79 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase): def test_dropped(self): item = {} - exception = Exception(u"\u2018") + exception = Exception("\u2018") response = Response("http://www.example.com") logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Dropped: \u2018", '{}']) + self.assertEqual(lines, ["Dropped: \u2018", '{}']) def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, u"Error processing {'key': 'value'}") + self.assertEqual(logline, "Error processing {'key': 'value'}") def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase): def test_scraped(self): item = CustomItem() - item['name'] = u'\xa3' + item['name'] = '\xa3' response = Response("http://www.example.com") logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3']) class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) CRAWLEDMSG = ( - u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" + "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) log_args = kwargs['args'] log_args['flags'] = str(request.flags) diff --git a/tests/test_mail.py b/tests/test_mail.py index 53dbc0686..9b248fbfa 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase): self.catched_msg = dict(**kwargs) def test_send_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, charset='utf-8', _callback=self._catch_mail_sent) @@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"') def test_send_attach_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' attach = BytesIO() attach.write(body.encode('utf-8')) attach.seek(0) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index dd19a69d5..a175f88ca 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b'attachment; filename=data.xml', XmlResponse), - (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response), - (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), - (u'attachment;filename=data高.doc'.encode('gbk'), Response), - (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), - (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), + ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response), + ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), + ('attachment;filename=data高.doc'.encode('gbk'), Response), + ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), + ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), ] for source, cls in mappings: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 24aaaf7ec..9d8c201dd 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -93,7 +93,7 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) def test_unicode_url_and_useragent(self): - robotstxt_robotstxt_body = u""" + robotstxt_robotstxt_body = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -107,11 +107,11 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/", "*")) self.assertFalse(rp.allowed("https://site.local/admin/", "*")) self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt")) + self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*")) - self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*")) + self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt")) + self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")) class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): diff --git a/tests/test_selector.py b/tests/test_selector.py index 00e663c11..62036ad8c 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase): ) self.assertEqual( [x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a'] + ['a'] ) self.assertEqual( [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0'] + ['12.0'] ) self.assertEqual( sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules'] + ['xpathrules'] ) self.assertEqual( [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12'] + ['12'] ) def test_root_base_url(self): @@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + # '\xa3' = pound symbol in unicode + # '\xc2\xa3' = pound symbol in utf-8 + # '\xa3' = pound symbol in latin-1 (iso-8859-1) - meta = u'' - head = u'' + meta + u'' - body_content = u'\xa3' - body = u'' + body_content + u'' - html = u'' + head + body + u'' + meta = '' + head = '' + meta + '' + body_content = '\xa3' + body = '' + body_content + '' + html = '' + head + body + '' encoding = 'utf-8' html_utf8 = html.encode(encoding) headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): - Selector(TextResponse(url='http://example.com', body=b''), text=u'') + Selector(TextResponse(url='http://example.com', body=b''), text='') diff --git a/tests/test_spider.py b/tests/test_spider.py index 83c10a3c3..78157a9b9 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest): output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ - {'loc': [u'http://www.example.com/Special-Offers.html'], - 'updated': [u'2009-08-16'], - 'custom': [u'fuu'], - 'other': [u'bar']}, + {'loc': ['http://www.example.com/Special-Offers.html'], + 'updated': ['2009-08-16'], + 'custom': ['fuu'], + 'other': ['bar']}, {'loc': [], - 'updated': [u'2009-08-16'], - 'other': [u'foo'], + 'updated': ['2009-08-16'], + 'other': ['foo'], 'custom': []}, ], iterator) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index d17bb2cbc..298178f08 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = u""" + body = """ <þingflokkar> <þingflokkur id="26"> @@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase): XmlResponse(url="http://example.com", body=body, encoding='utf-8'), ): attrs = [] - for x in self.xmliter(r, u'þingflokkur'): + for x in self.xmliter(r, 'þingflokkur'): attrs.append((x.attrib['id'], - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), - x.xpath(u'./tímabil/fyrstaþing/text()').getall())) + x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath('./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [(u'26', [u'-'], [u'80']), - (u'21', [u'Ab'], [u'76']), - (u'27', [u'A'], [u'27'])]) + [('26', ['-'], ['80']), + ('21', ['Ab'], ['76']), + ('27', ['A'], ['27'])]) def test_xmliter_text(self): body = ( @@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase): ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], - [[u'one'], [u'two']]) + [['one'], ['two']]) def test_xmliter_namespaces(self): body = b""" @@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), - u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' + 'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase): result = [row for row in csv] self.assertEqual(result, - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: @@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') @@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual([row for row in csv1], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual([row for row in csv2], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): body = get_testdata('feeds', 'feed-sample6.csv') @@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"}, - {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"}, - {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"}, - {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) + [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"}, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') @@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() @@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': 'foo\nbar'}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + {'id': '1', 'name': 'latin1', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'}, ] ) @@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + {'id': '1', 'name': 'cp852', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, ] ) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..3f93f509e 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase): class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e') def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') + self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e') def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_unicode, 423) @@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( to_unicode(b'a\xedb', 'utf-8', errors='replace'), - u'a\ufffdb' + 'a\ufffdb' ) class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49') + self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49') def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49') + self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49') def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') @@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( - to_bytes(u'a\ufffdb', 'latin-1', errors='replace'), + to_bytes('a\ufffdb', 'latin-1', errors='replace'), b'a?b' ) @@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase): assert binary_is_text(b"hello") def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text(u"hello".encode('utf-16')) + assert binary_is_text("hello".encode('utf-16')) def test_one_with_encoding(self): assert binary_is_text(b"
    Price \xa3
    ") diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 450e4bdca..de94ec960 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase): method="POST", body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, - cookies={'currency': u'руб'}, + cookies={'currency': 'руб'}, encoding='latin-1', priority=20, meta={'a': 'b'}, diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5a52dd695..5ff2e41ef 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): def test_simple_render(self): context = dict(project_name='proj', name='spi', classname='TheSpider') - template = u'from ${project_name}.spiders.${name} import ${classname}' - rendered = u'from proj.spiders.spi import TheSpider' + template = 'from ${project_name}.spiders.${name} import ${classname}' + rendered = 'from proj.spiders.spi import TheSpider' template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') render_path = os.path.join(self.tmp_path, 'templ.py') From 6f4ccec5675c00b0ec7877a0b1fcc234d5983490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 14:03:14 +0200 Subject: [PATCH 064/119] Cover our deprecation policy in the documentation --- docs/contributing.rst | 5 +++++ docs/versioning.rst | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7b901dd00..525ad3497 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches diff --git a/docs/versioning.rst b/docs/versioning.rst index 227085f02..57643ea9a 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,7 +1,7 @@ .. _versioning: ============================ -Versioning and API Stability +Versioning and API stability ============================ Versioning @@ -34,7 +34,7 @@ For example: production) -API Stability +API stability ============= API stability was one of the major goals for the *1.0* release. @@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the same way. +.. _deprecation-policy: + +Deprecation policy +================== + +We aim to maintain support for deprecated Scrapy features for at least 1 year. + +For example, if a feature is deprecated in a Scrapy version released on +June 15th 2020, that feature should continue to work in versions released on +June 14th 2021 or before that. + +Any new Scrapy release after a year *may* remove support for that deprecated +feature. + +All deprecated features removed in a Scrapy release are explicitly mentioned in +the :ref:`release notes `. + + .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases From 3600582f56071dd9f40f31ed44e09294a78dc13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:05:56 +0200 Subject: [PATCH 065/119] Cover Scrapy 2.2.1 and 2.3 in the release notes (#4708) --- docs/news.rst | 133 ++++++++++++++++++++++++++++++++ docs/topics/commands.rst | 2 + docs/topics/developer-tools.rst | 6 +- docs/topics/feed-exports.rst | 36 ++++++++- scrapy/utils/curl.py | 3 +- 5 files changed, 176 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 80d130e4a..850b323ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,139 @@ Release notes ============= +.. _release-2.3.0: + +Scrapy 2.3.0 (2020-08-04) +------------------------- + +Highlights: + +* :ref:`Feed exports ` now support :ref:`Google Cloud + Storage ` as a storage backend + +* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver + output items in batches of up to the specified number of items. + + It also serves as a workaround for :ref:`delayed file delivery + `, which causes Scrapy to only start item delivery + after the crawl has finished when using certain storage backends + (:ref:`S3 `, :ref:`FTP `, + and now :ref:`GCS `). + +* The base implementation of :ref:`item loaders ` has been + moved into a separate library, :doc:`itemloaders `, + allowing usage from outside Scrapy and a separate release schedule + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* Removed the following classes and their parent modules from + ``scrapy.linkextractors``: + + * ``htmlparser.HtmlParserLinkExtractor`` + * ``regex.RegexLinkExtractor`` + * ``sgml.BaseSgmlLinkExtractor`` + * ``sgml.SgmlLinkExtractor`` + + Use + :class:`LinkExtractor ` + instead (:issue:`4356`, :issue:`4679`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated + (:issue:`4683`) + + +New features +~~~~~~~~~~~~ + +* :ref:`Feed exports ` support :ref:`Google Cloud + Storage ` (:issue:`685`, :issue:`3608`) + +* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries + (:issue:`4250`, :issue:`4434`) + +* The :command:`parse` command now allows specifying an output file + (:issue:`4317`, :issue:`4377`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support + ``--data-raw`` (:issue:`4612`) + +* A ``parse`` callback may now be used in built-in spider subclasses, such + as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`, + :issue:`781`, :issue:`4254` ) + + +Bug fixes +~~~~~~~~~ + +* Fixed the :ref:`CSV exporting ` of + :ref:`dataclass items ` and :ref:`attr.s items + ` (:issue:`4667`, :issue:`4668`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request + method to ``POST`` when a request body is specified and no request method + is specified (:issue:`4612`) + +* The processing of ANSI escape sequences in enabled in Windows 10.0.14393 + and later, where it is required for colored output (:issue:`4393`, + :issue:`4403`) + + +Documentation +~~~~~~~~~~~~~ + +* Updated the `OpenSSL cipher list format`_ link in the documentation about + the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`) + +* Simplified the code example in :ref:`topics-loaders-dataclass` + (:issue:`4652`) + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* The base implementation of :ref:`item loaders ` has been + moved into :doc:`itemloaders ` (:issue:`4005`, + :issue:`4516`) + +* Fixed a silenced error in some scheduler tests (:issue:`4644`, + :issue:`4645`) + +* Renewed the localhost certificate used for SSL tests (:issue:`4650`) + +* Removed cookie-handling code specific to Python 2 (:issue:`4682`) + +* Stopped using Python 2 unicode literal syntax (:issue:`4704`) + +* Stopped using a backlash for line continuation (:issue:`4673`) + +* Removed unneeded entries from the MyPy exception list (:issue:`4690`) + +* Automated tests now pass on Windows as part of our continuous integration + system (:issue:`4458`) + +* Automated tests now pass on the latest PyPy version for supported Python + versions in our continuous integration system (:issue:`4504`) + + +.. _release-2.2.1: + +Scrapy 2.2.1 (2020-07-17) +------------------------- + +* The :command:`startproject` command no longer makes unintended changes to + the permissions of files in the destination folder, such as removing + execution permissions (:issue:`4662`, :issue:`4666`) + + .. _release-2.2.0: Scrapy 2.2.0 (2020-06-24) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 4fce51abc..9638a2322 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -493,6 +493,8 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file + .. versionadded:: 2.3 + .. skip: start Usage example:: diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 4e87a00f2..101aa159c 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -289,8 +289,10 @@ request:: "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") Alternatively, if you want to know the arguments needed to recreate that -request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` -function to get a dictionary with the equivalent arguments. +request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments: + +.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dd4eb3c61..37b7096f6 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -100,6 +100,7 @@ The storages backends supported out of the box are: * :ref:`topics-feed-storage-fs` * :ref:`topics-feed-storage-ftp` * :ref:`topics-feed-storage-s3` (requires botocore_) + * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) * :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are @@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-s3: S3 @@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-gcs: Google Cloud Storage (GCS) -------------------------- +.. versionadded:: 2.3 + The feeds are stored on `Google Cloud Storage`_. * URI scheme: ``gs`` @@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_. * ``gs://mybucket/path/to/export.csv`` - * Required external libraries: `google-cloud-storage `_. + * Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. @@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following * :setting:`FEED_STORAGE_GCS_ACL` * :setting:`GCS_PROJECT_ID` +This storage backend uses :ref:`delayed file delivery `. + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + + .. _topics-feed-storage-stdout: Standard output @@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process. * Required external libraries: none +.. _delayed-file-delivery: + +Delayed file delivery +--------------------- + +As indicated above, some of the described storage backends use delayed file +delivery. + +These storage backends do not upload items to the feed URI as those items are +scraped. Instead, Scrapy writes items into a temporary local file, and only +once all the file contents have been written (i.e. at the end of the crawl) is +that file uploaded to the feed URI. + +If you want item delivery to start earlier when using one of these storage +backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items +in multiple files, with the specified maximum item count per file. That way, as +soon as a file reaches the maximum item count, that file is delivered to the +feed URI, allowing item delivery to start way before the end of the crawl. + + Settings ======== diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index aa681522f..9c0efcec4 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): :param str curl_command: string containing the curl command :param bool ignore_unknown_options: If true, only a warning is emitted when - cURL options are unknown. Otherwise raises an error. (default: True) + cURL options are unknown. Otherwise + raises an error. (default: True) :return: dictionary of Request kwargs """ From 1278e76d9093b1c5c9ec810768d1066772a8a134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:07:02 +0200 Subject: [PATCH 066/119] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92=202?= =?UTF-8?q?.3.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8d4d74bc5..3c1c8f891 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0 +current_version = 2.3.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index ccbccc3dc..276cbf9e2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.2.0 +2.3.0 From 4ee538e44b2650c49054b1f7f4c87ac70350471a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:34:11 +0200 Subject: [PATCH 067/119] Update unicode references from Python 2 times in the documentation (#4703) --- docs/topics/exporters.rst | 8 ++------ docs/topics/loaders.rst | 4 ++-- docs/topics/request-response.rst | 30 ++++++++++++++++-------------- docs/topics/selectors.rst | 7 ++++--- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e5c99e5b1..8c84b85fc 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -166,8 +166,7 @@ BaseItemExporter By default, this method looks for a serializer :ref:`declared in the item field ` and returns the result of applying that serializer to the value. If no serializer is found, it returns the - value unchanged except for ``unicode`` values which are encoded to - ``str`` using the encoding declared in the :attr:`encoding` attribute. + value unchanged. :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty @@ -217,10 +216,7 @@ BaseItemExporter .. attribute:: encoding - The encoding that will be used to encode unicode values. This only - affects unicode values (which are always serialized to str using this - encoding). Other value types are passed unchanged to the specific - serialization library. + The output character encoding. .. attribute:: indent diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 29d9c5805..c0f534493 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example:: default_output_processor = TakeFirst() - name_in = MapCompose(unicode.title) + name_in = MapCompose(str.title) name_out = Join() - price_in = MapCompose(unicode.strip) + price_in = MapCompose(str.strip) # ... diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index fbd8e4b73..1dffd1d55 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,12 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str or unicode + :param body: the request body. If a string is passed, then it's encoded as + bytes using the ``encoding`` passed (which defaults to ``utf-8``). If + ``body`` is not given, an empty bytes object is stored. Regardless of the + type of this argument, the final value stored will be a bytes object + (never a string or ``None``). + :type body: bytes or str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +106,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes (if given as a string). :type encoding: string :param priority: the priority of this request (defaults to ``0``). @@ -721,7 +721,7 @@ Response objects .. attribute:: Response.body The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use + is always a bytes object. If you want the string version use :attr:`TextResponse.text` (only available in :class:`TextResponse` and subclasses). @@ -842,9 +842,9 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode + response. If you create a :class:`TextResponse` object with a string as body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the + is always a bytes object). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. :type encoding: string @@ -853,7 +853,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body, as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -861,9 +861,11 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + ``str(response.body)`` is not a correct way to convert the response + body into a string: + + >>> str(b'body') + "b'body'" .. attribute:: TextResponse.encoding diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 5014df6ac..9e2c6ba42 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or -markup as an unicode string (in ``text`` argument). +markup as a string (in ``text`` argument). + Usually there is no need to construct Scrapy selectors manually: ``response`` object is available in Spider callbacks, so in most cases it is more convenient to use ``response.css()`` and ``response.xpath()`` @@ -383,7 +384,7 @@ Using selectors with regular expressions :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or -``.css()`` methods, ``.re()`` returns a list of unicode strings. So you +``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. Here's an example used to extract image names from the :ref:`HTML code @@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: sel.xpath("//h1") 2. Extract the text of all ``

    `` elements from an HTML response body, - returning a list of unicode strings:: + returning a list of strings:: sel.xpath("//h1").getall() # this includes the h1 tag sel.xpath("//h1/text()").getall() # this excludes the h1 tag From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Tue, 4 Aug 2020 20:42:01 +0200 Subject: [PATCH 068/119] Change super syntax (#4707) --- extras/qpsclient.py | 2 +- scrapy/commands/view.py | 2 +- scrapy/contracts/default.py | 2 +- scrapy/core/downloader/contextfactory.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 10 +++++----- scrapy/core/downloader/tls.py | 2 +- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/exceptions.py | 4 ++-- scrapy/exporters.py | 2 +- scrapy/http/headers.py | 8 ++++---- scrapy/http/request/form.py | 2 +- scrapy/http/request/json_request.py | 4 ++-- scrapy/http/request/rpc.py | 2 +- scrapy/http/response/text.py | 10 +++++----- scrapy/item.py | 10 +++++----- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 3 +-- scrapy/resolver.py | 10 +++++----- scrapy/selector/unified.py | 2 +- scrapy/settings/__init__.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spiders/crawl.py | 4 ++-- scrapy/spiders/init.py | 2 +- scrapy/spiders/sitemap.py | 2 +- scrapy/squeues.py | 12 ++++++------ scrapy/statscollectors.py | 2 +- scrapy/utils/datatypes.py | 14 +++++++------- scrapy/utils/deprecate.py | 8 ++++---- scrapy/utils/log.py | 2 +- scrapy/utils/serialize.py | 2 +- scrapy/utils/testsite.py | 4 ++-- tests/spiders.py | 20 ++++++++++---------- tests/test_command_parse.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_contracts.py | 2 +- tests/test_downloader_handlers.py | 4 ++-- tests/test_downloadermiddleware_httpcache.py | 4 ++-- tests/test_downloadermiddleware_robotstxt.py | 4 ++-- tests/test_exporters.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_http_response.py | 2 +- tests/test_item.py | 2 +- tests/test_linkextractors.py | 2 +- tests/test_loader.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_media.py | 12 ++++++------ tests/test_request_left.py | 2 +- tests/test_robotstxt_interface.py | 8 ++++---- tests/test_scheduler.py | 4 ++-- tests/test_spidermiddleware_httperror.py | 2 +- 56 files changed, 117 insertions(+), 118 deletions(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 7554f7eec..fe1f96cbb 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -27,7 +27,7 @@ class QPSSpider(Spider): slots = 1 def __init__(self, *a, **kw): - super(QPSSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) if self.qps is not None: self.qps = float(self.qps) self.download_delay = 1 / self.qps diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 908bee966..c8f873334 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): - super(Command, self).add_options(parser) + super().add_options(parser) parser.remove_option("--headers") def _print_response(self, response, opts): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 34f0d36d4..cfdcc7c25 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -56,7 +56,7 @@ class ReturnsContract(Contract): } def __init__(self, *args, **kwargs): - super(ReturnsContract, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: raise ValueError( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..8a7d656a1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): - super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging if tls_ciphers: @@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) # # * getattr() for `_ssl_method` attribute for context factories - # not calling super(..., self).__init__ + # not calling super().__init__ return CertificateOptions( verify=False, method=getattr(self, 'method', getattr(self, '_ssl_method', None)), diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..fb04d1fb7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf - super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) + super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() self._tunneledHost = host self._tunneledPort = port @@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def connect(self, protocolFactory): self._protocolFactory = protocolFactory - connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) + connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) connectDeferred.addErrback(self.connectFailed) return self._tunnelReadyDeferred @@ -215,7 +215,7 @@ class TunnelingAgent(Agent): def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): - super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory @@ -235,7 +235,7 @@ class TunnelingAgent(Agent): # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy key = key + self._proxyConf - return super(TunnelingAgent, self)._requestWithEndpoint( + return super()._requestWithEndpoint( key=key, endpoint=endpoint, method=method, @@ -249,7 +249,7 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): - super(ScrapyProxyAgent, self).__init__( + super().__init__( reactor=reactor, connectTimeout=connectTimeout, bindAddress=bindAddress, diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e43a3c83e..d9f3750d5 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): """ def __init__(self, hostname, ctx, verbose_logging=False): - super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + super().__init__(hostname, ctx) self.verbose_logging = verbose_logging def _identityVerifyingInfoCallback(self, connection, where, ret): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 35264a92b..5a99b96be 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): - super(SpiderMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f43771e2..48f19424c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner): """ def __init__(self, settings=None, install_root_handler=True): - super(CrawlerProcess, self).__init__(settings) + super().__init__(settings) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 366d60dcb..4053fecc5 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = 'METAREFRESH_ENABLED' def __init__(self, settings): - super(MetaRefreshMiddleware, self).__init__(settings) + super().__init__(settings) self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 45f152321..0c410f035 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -37,7 +37,7 @@ class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" def __init__(self, reason='cancelled'): - super(CloseSpider, self).__init__() + super().__init__() self.reason = reason @@ -74,7 +74,7 @@ class UsageError(Exception): def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) - super(UsageError, self).__init__(*a, **kw) + super().__init__(*a, **kw) class ScrapyDeprecationWarning(Warning): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0aba1c904..95518b3ac 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter): def _configure(self, options, dont_fail=False): self.binary = options.pop('binary', True) - super(PythonItemExporter, self)._configure(options, dont_fail) + super()._configure(options, dont_fail) if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index dcaaeddfa..6bf9e5346 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -8,7 +8,7 @@ class Headers(CaselessDict): def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding - super(Headers, self).__init__(seq) + super().__init__(seq) def normkey(self, key): """Normalize key to bytes""" @@ -37,19 +37,19 @@ class Headers(CaselessDict): def __getitem__(self, key): try: - return super(Headers, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] except IndexError: return None def get(self, key, def_val=None): try: - return super(Headers, self).get(key, def_val)[-1] + return super().get(key, def_val)[-1] except IndexError: return None def getlist(self, key, def_val=None): try: - return super(Headers, self).__getitem__(key) + return super().__getitem__(key) except KeyError: if def_val is not None: return self.normvalue(def_val) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a260798ac..59af81321 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,7 @@ class FormRequest(Request): if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' - super(FormRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f08b25280..eae3f9f6b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -32,7 +32,7 @@ class JsonRequest(Request): if 'method' not in kwargs: kwargs['method'] = 'POST' - super(JsonRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -47,7 +47,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - return super(JsonRequest, self).replace(*args, **kwargs) + return super().replace(*args, **kwargs) def _dumps(self, data): """Convert to JSON """ diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 811d3ad6b..c70912e49 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -31,5 +31,5 @@ class XmlRpcRequest(Request): if encoding is not None: kwargs['encoding'] = encoding - super(XmlRpcRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'text/xml') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0f300c8da..a7bb34d48 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,13 +35,13 @@ class TextResponse(Response): self._cached_benc = None self._cached_ubody = None self._cached_selector = None - super(TextResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def _set_url(self, url): if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: - super(TextResponse, self)._set_url(url) + super()._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection @@ -51,7 +51,7 @@ class TextResponse(Response): type(self).__name__) self._body = body.encode(self._encoding) else: - super(TextResponse, self)._set_body(body) + super()._set_body(body) def replace(self, *args, **kwargs): kwargs.setdefault('encoding', self.encoding) @@ -166,7 +166,7 @@ class TextResponse(Response): elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding - return super(TextResponse, self).follow( + return super().follow( url=url, callback=callback, method=method, @@ -226,7 +226,7 @@ class TextResponse(Response): for sel in selectors: with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) - return super(TextResponse, self).follow_all( + return super().follow_all( urls=urls, callback=callback, method=method, diff --git a/scrapy/item.py b/scrapy/item.py index 4ab83d1a0..c262a153c 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta): if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(BaseItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) class Field(dict): @@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta): def __new__(mcs, class_name, bases, attrs): classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) - _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) + _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs) fields = getattr(_class, 'fields', {}) new_attrs = {} @@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta): new_attrs['_class'] = _class if classcell is not None: new_attrs['__classcell__'] = classcell - return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) + return super().__new__(mcs, class_name, bases, new_attrs) class DictItem(MutableMapping, BaseItem): @@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem): if issubclass(cls, DictItem) and not issubclass(cls, Item): warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(DictItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): self._values = {} @@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) - super(DictItem, self).__setattr__(name, value) + super().__setattr__(name, value) def __len__(self): return len(self._values) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 984a5c4e1..08a6ca1e8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -65,7 +65,7 @@ class FilteringLinkExtractor: warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super(FilteringLinkExtractor, cls).__new__(cls) + return super().__new__(cls) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8b9f961ee..e941c4321 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super(LxmlLinkExtractor, self).__init__( + super().__init__( link_extractor=lx, allow=allow, deny=deny, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 487382a38..6bc5d46eb 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) + super().__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 46f2bfb58..e2dd70215 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): - super(ImagesPipeline, self).__init__(store_uri, settings=settings, - download_func=download_func) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: settings = Settings(settings) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..f191deac6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver): """ def __init__(self, reactor, cache_size, timeout): - super(CachingThreadedResolver, self).__init__(reactor) + super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver): # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) - d = super(CachingThreadedResolver, self).getHostByName(name, timeout) + d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d @@ -80,16 +80,16 @@ class CachingHostnameResolver: class CachingResolutionReceiver(resolutionReceiver): def resolutionBegan(self, resolution): - super(CachingResolutionReceiver, self).resolutionBegan(resolution) + super().resolutionBegan(resolution) self.resolution = resolution self.resolved = False def addressResolved(self, address): - super(CachingResolutionReceiver, self).addressResolved(address) + super().addressResolved(address) self.resolved = True def resolutionComplete(self): - super(CachingResolutionReceiver, self).resolutionComplete() + super().resolutionComplete() if self.resolved: dnscache[hostName] = self.resolution diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 85a9bb526..f12c61081 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault('base_url', response.url) self.response = response - super(Selector, self).__init__(text=text, type=st, root=root, **kwargs) + super().__init__(text=text, type=st, root=root, **kwargs) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ff8317cd1..b8ae32d7c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -439,7 +439,7 @@ class Settings(BaseSettings): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user - super(Settings, self).__init__() + super().__init__() self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..db9d0f2ae 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -15,7 +15,7 @@ class HttpError(IgnoreRequest): def __init__(self, response, *args, **kwargs): self.response = response - super(HttpError, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class HttpErrorMiddleware: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index cb7260892..c9fbce08d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -75,7 +75,7 @@ class CrawlSpider(Spider): rules = () def __init__(self, *a, **kw): - super(CrawlSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._compile_rules() def _parse(self, response, **kwargs): @@ -145,6 +145,6 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) return spider diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index fd41133ea..fe8c94e78 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -6,7 +6,7 @@ class InitSpider(Spider): """Base Spider with initialization facilities""" def start_requests(self): - self._postinit_reqs = super(InitSpider, self).start_requests() + self._postinit_reqs = super().start_requests() return iterate_spider_output(self.init_request()) def initialized(self, response=None): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c5360bfa7..1f72e76b7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -18,7 +18,7 @@ class SitemapSpider(Spider): sitemap_alternate_links = False def __init__(self, *a, **kw): - super(SitemapSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, str): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index c7ad4d53d..77ffda6f7 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -20,7 +20,7 @@ def _with_mkdir(queue_class): if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def push(self, obj): s = serialize(obj) - super(SerializableQueue, self).push(s) + super().push(s) def pop(self): - s = super(SerializableQueue, self).pop() + s = super().pop() if s: return deserialize(s) @@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class): def __init__(self, crawler, key): self.spider = crawler.spider - super(ScrapyRequestQueue, self).__init__(key) + super().__init__(key) @classmethod def from_crawler(cls, crawler, key, *args, **kwargs): @@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class): def push(self, request): request = request_to_dict(request, self.spider) - return super(ScrapyRequestQueue, self).push(request) + return super().push(request) def pop(self): - request = super(ScrapyRequestQueue, self).pop() + request = super().pop() if not request: return None diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 579c60180..ba7d1a6bf 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -54,7 +54,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler): - super(MemoryStatsCollector, self).__init__(crawler) + super().__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2a92d0588..e31284a7f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -15,7 +15,7 @@ class CaselessDict(dict): __slots__ = () def __init__(self, seq=None): - super(CaselessDict, self).__init__() + super().__init__() if seq: self.update(seq) @@ -53,7 +53,7 @@ class CaselessDict(dict): def update(self, seq): seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) - super(CaselessDict, self).update(iseq) + super().update(iseq) @classmethod def fromkeys(cls, keys, value=None): @@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict): """ def __init__(self, limit=None): - super(LocalCache, self).__init__() + super().__init__() self.limit = limit def __setitem__(self, key, value): if self.limit: while len(self) >= self.limit: self.popitem(last=False) - super(LocalCache, self).__setitem__(key, value) + super().__setitem__(key, value) class LocalWeakReferencedCache(weakref.WeakKeyDictionary): @@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ def __init__(self, limit=None): - super(LocalWeakReferencedCache, self).__init__() + super().__init__() self.data = LocalCache(limit=limit) def __setitem__(self, key, value): try: - super(LocalWeakReferencedCache, self).__setitem__(key, value) + super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching def __getitem__(self, key): try: - return super(LocalWeakReferencedCache, self).__getitem__(key) + return super().__getitem__(key) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3dbea5fee..3c8e3c8b5 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,7 +57,7 @@ def create_deprecated_class( warned_on_subclass = False def __new__(metacls, name, bases, clsdict_): - cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_) + cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -73,7 +73,7 @@ def create_deprecated_class( if warn_once: msg += ' (warning only on first subclass, there may be others)' warnings.warn(msg, warn_category, stacklevel=2) - super(DeprecatedClass, cls).__init__(name, bases, clsdict_) + super().__init__(name, bases, clsdict_) # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks @@ -88,7 +88,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super(DeprecatedClass, cls).__subclasscheck__(sub) + return super().__subclasscheck__(sub) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -102,7 +102,7 @@ def create_deprecated_class( msg = instance_warn_message.format(cls=_clspath(cls, old_class_path), new=_clspath(new_class, new_class_path)) warnings.warn(msg, warn_category, stacklevel=2) - return super(DeprecatedClass, cls).__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {}) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..1d6a2c39d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" def __init__(self, crawler, *args, **kwargs): - super(LogCounterHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.crawler = crawler def emit(self, record): diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index dc9604578..cc3263602 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif isinstance(o, Response): return "<%s %s %s>" % (type(o).__name__, o.status, o.url) else: - return super(ScrapyJSONEncoder, self).default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 66930ad2c..397e54703 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -7,12 +7,12 @@ class SiteTest: def setUp(self): from twisted.internet import reactor - super(SiteTest, self).setUp() + super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.baseurl = "http://localhost:%d/" % self.site.getHost().port def tearDown(self): - super(SiteTest, self).tearDown() + super().tearDown() self.site.stopListening() def url(self, path): diff --git a/tests/spiders.py b/tests/spiders.py index 3eb681819..63bd726fb 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): - super(MockServerSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.mockserver = mockserver @@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider): name = 'meta' def __init__(self, *args, **kwargs): - super(MetaSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.meta = {} def closed(self, reason): @@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): - super(FollowAllSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} @@ -60,7 +60,7 @@ class DelaySpider(MetaSpider): name = 'delay' def __init__(self, n=1, b=0, *args, **kwargs): - super(DelaySpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.n = n self.b = b self.t1 = self.t2 = self.t2_err = 0 @@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider): name = 'simple' def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(SimpleSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [url] def parse(self, response): @@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider): name = 'item' def parse(self, response): - for request in super(ItemSpider, self).parse(response): + for request in super().parse(response): yield request yield Item() yield {} @@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider): raise self.exception_cls('Expected exception') def parse(self, response): - for request in super(ErrorSpider, self).parse(response): + for request in super().parse(response): yield request self.raise_exception() @@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): fail_yielding = False def __init__(self, *a, **kw): - super(BrokenStartRequestsSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self.seedsseen = [] def start_requests(self): @@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get('seed')) - for req in super(BrokenStartRequestsSpider, self).parse(response): + for req in super().parse(response): yield req @@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def parse(self, response): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5754a5478..e115f420f 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' def setUp(self): - super(ParseCommandTest, self).setUp() + super().setUp() self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: diff --git a/tests/test_commands.py b/tests/test_commands.py index 42091ab00..8938156fc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None): class StartprojectTemplatesTest(ProjectTest): def setUp(self): - super(StartprojectTemplatesTest, self).setUp() + super().setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') @@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest): class CommandTest(ProjectTest): def setUp(self): - super(CommandTest, self).setUp() + super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 99120b128..2e7e3ccc4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase): name = 'test_same_url' def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def start_requests(s): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57d4cdd6b..13063d106 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https11InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https11InvalidDNSPattern, self).setUp() + super().setUp() class Https11CustomCiphers(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9b77c97a8..299fb0eb8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) - return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded @@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_GZIP', True) - return super(FilesystemStorageTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) class DummyPolicyTest(_BaseTest): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f9936baba..858138f81 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): skip = "Rerp parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithRerpTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') @@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): skip = "Reppy parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithReppyTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 660c99ce1..6c25a0064 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase): if name == 'age': return str(int(value) + 1) else: - return super(CustomItemExporter, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index f5cf4e798..0a303dbe2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest): def setUp(self): warnings.simplefilter("always") - super(JsonRequestTest, self).setUp() + super().setUp() def test_data(self): r1 = self.request_class(url="http://www.example.com/") @@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() - super(JsonRequestTest, self).tearDown() + super().tearDown() if __name__ == "__main__": diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 56d017de6..f831ef5dc 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest): response_class = TextResponse def test_replace(self): - super(TextResponseTest, self).test_replace() + super().test_replace() r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") diff --git a/tests/test_item.py b/tests/test_item.py index 0ce78f8c0..66fa761f0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase): # requirement. When not done properly raises an error: # TypeError: __class__ set to # defining 'MyItem' as - super(MyItem, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class DictItemTest(unittest.TestCase): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a0bafa5e5..6f133d77a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ]) def test_restrict_xpaths_with_html_entities(self): - super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + super().test_restrict_xpaths_with_html_entities() def test_filteringlinkextractor_deprecation_warning(self): """Make sure the FilteringLinkExtractor deprecation warning is not diff --git a/tests/test_loader.py b/tests/test_loader.py index 2ed6f365f..b0bc82f4e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase): temp = Field() def __init__(self, *args, **kwargs): - super(TempItem, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index eb14de14f..624dd9ab8 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase): class TempDict(dict): def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index b771e7d79..41ff3651d 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase): class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) + kwargs = super().crawled(request, response, spider) CRAWLEDMSG = ( "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3364d2258..b2b75ef20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager): return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): - super(TestMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process'): self.methods['process'].append(mw.process) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 19ff00350..4f130c0c9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase): class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipeline, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def download(self, request, info): self._mockcalled.append('download') - return super(MockedMediaPipeline, self).download(request, info) + return super().download(request, info) def media_to_download(self, request, info): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') - return super(MockedMediaPipeline, self).media_to_download(request, info) + return super().media_to_download(request, info) def get_media_requests(self, item, info): self._mockcalled.append('get_media_requests') @@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline): def media_downloaded(self, response, request, info): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipeline, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): self._mockcalled.append('media_failed') - return super(MockedMediaPipeline, self).media_failed(failure, request, info) + return super().media_failed(failure, request, info) def item_completed(self, results, item, info): self._mockcalled.append('item_completed') - item = super(MockedMediaPipeline, self).item_completed(results, item, info) + item = super().item_completed(results, item, info) item['results'] = results return item diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5cfef8e7d..373b2e49c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider): name = 'signal_catcher' def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9d8c201dd..4b15d0fab 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -117,7 +117,7 @@ class BaseRobotParserTest: class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser - super(PythonRobotParserTest, self)._setUp(PythonRobotParser) + super()._setUp(PythonRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.") @@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ReppyRobotParser - super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) + super()._setUp(ReppyRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Reppy does not support order based directives precedence.") @@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import RerpRobotParser - super(RerpRobotParserTest, self)._setUp(RerpRobotParser) + super()._setUp(RerpRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("Rerp does not support length based directives precedence.") @@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ProtegoRobotParser - super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser) + super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2b6cb0902..512a7460e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,7 +53,7 @@ class MockCrawler(Crawler): JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', ) - super(MockCrawler, self).__init__(Spider, settings) + super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -296,7 +296,7 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls - super(StartUrlsSpider, self).__init__(name='StartUrlsSpider') + super().__init__(name='StartUrlsSpider') def parse(self, response): pass diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..e449cd706 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider): bypass_status_codes = set() def __init__(self, *args, **kwargs): - super(_HttpErrorSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [ self.mockserver.url("/status?n=200"), self.mockserver.url("/status?n=404"), From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:11:59 +0530 Subject: [PATCH 069/119] deprecated weakkeycache by specifying in __init__ --- scrapy/utils/python.py | 4 +++- scrapy/utils/tester.py | 3 +++ tests/test_utils_python.py | 22 +++++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index c8f921ff3..4756b07b6 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,10 +7,12 @@ import inspect import re import sys import weakref +import warnings from functools import partial, wraps from itertools import chain from scrapy.utils.decorators import deprecated +from scrapy.exceptions import ScrapyDeprecationWarning def flatten(x): @@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes): return True -@deprecated class WeakKeyCache: def __init__(self, default_factory): + warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py new file mode 100644 index 000000000..691e9bc1a --- /dev/null +++ b/scrapy/utils/tester.py @@ -0,0 +1,3 @@ +from scrapy.utils.decorators import deprecated + + diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5a53d89e4..ebce3c079 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,15 +1,18 @@ import functools +import gc import operator import platform import unittest +from itertools import count from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) + __doctests__ = ['scrapy.utils.python'] @@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) + def test_weakkeycache(self): + class _Weakme: + pass + + _values = count() + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() + v = wk[k] + self.assertEqual(v, wk[k]) + self.assertNotEqual(v, wk[_Weakme()]) + self.assertEqual(v, wk[k]) + del k + for _ in range(100): + if wk._weakdict: + gc.collect() + self.assertFalse(len(wk._weakdict)) + def test_get_func_args(self): def f1(a, b, c): pass From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:14:04 +0530 Subject: [PATCH 070/119] deleted tester.py --- scrapy/utils/tester.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py deleted file mode 100644 index 691e9bc1a..000000000 --- a/scrapy/utils/tester.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.utils.decorators import deprecated - - From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 16:13:52 +0530 Subject: [PATCH 071/119] aesthetic fixes --- scrapy/utils/python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 4756b07b6..59f1b8371 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,13 +6,13 @@ import gc import inspect import re import sys -import weakref import warnings +import weakref from functools import partial, wraps from itertools import chain -from scrapy.utils.decorators import deprecated from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import deprecated def flatten(x): @@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes): class WeakKeyCache: def __init__(self, default_factory): - warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() From 4dc09f09aa9698b02f2cbf2e3001202388eba043 Mon Sep 17 00:00:00 2001 From: linchiwei123 <40888469+linchiwei123@users.noreply.github.com> Date: Wed, 5 Aug 2020 22:23:19 +0800 Subject: [PATCH 072/119] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 58090f7a2..d0880051f 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ install_requires = [ 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', - 'lxml>=3.5.0', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', From aefd43a6c6b8d2e32dfb9bd0c94529805ce5037e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Aug 2020 12:52:54 +0200 Subject: [PATCH 073/119] Upgrade minimum dependencies for Python 3.6 support --- tests/requirements-py3.txt | 3 +-- tox.ini | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 00c56084d..4fa58d11b 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,7 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy; python_version >= '3.6' -mitmproxy<4.0.0; python_version < '3.6' +mitmproxy >=4, <5 # The latest version does not support some pinned dependencies # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines diff --git a/tox.ini b/tox.ini index 11882c03f..4f5531aea 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ deps = -rtests/requirements-py3.txt # Extras boto3>=1.13.0 - botocore>=1.3.23 + botocore>=1.4.87 Pillow>=3.4.2 passenv = S3_TEST_FILE_URI @@ -76,7 +76,7 @@ deps = zope.interface==4.1.3 -rtests/requirements-py3.txt # Extras - botocore==1.3.23 + botocore==1.4.87 google-cloud-storage==1.29.0 Pillow==3.4.2 From 1c4b4cc6b046e121be33a57031a0900ca1347e16 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Tue, 11 Aug 2020 17:42:44 +0530 Subject: [PATCH 074/119] Support defining file path based on item in media pipelines (#4686) --- docs/topics/media-pipeline.rst | 24 ++++--- scrapy/pipelines/files.py | 16 ++--- scrapy/pipelines/images.py | 14 ++-- scrapy/pipelines/media.py | 69 +++++++++++++++--- tests/test_pipeline_files.py | 13 ++++ tests/test_pipeline_media.py | 123 ++++++++++++++++++++++++++++++++- 6 files changed, 225 insertions(+), 34 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..487e26b8e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. @@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6bc5d46eb..5a2184681 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): def _onsuccess(result): if not result: return # returning None force download @@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline): checksum = result.get('checksum', None) return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} - path = self.file_path(request, info=info) + path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) dfd.addCallbacks(_onsuccess, lambda _: None) dfd.addErrback( @@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): referer = referer_str(request) if response.status != 200: @@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, status) try: - path = self.file_path(request, response=response, info=info) - checksum = self.file_downloaded(response, request, info) + path = self.file_path(request, response=response, info=info, item=item) + checksum = self.file_downloaded(response, request, info, item=item) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' @@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline): urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u) for u in urls] - def file_downloaded(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def file_downloaded(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = md5sum(buf) buf.seek(0) @@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_ext = os.path.splitext(request.url)[1] # Handles empty and wild extensions by trying to guess the diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e2dd70215..0a67a0b1d 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info): - return self.image_downloaded(response, request, info) + def file_downloaded(self, response, request, info, *, item=None): + return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info): + def image_downloaded(self, response, request, info, *, item=None): checksum = None - for path, image, buf in self.get_images(response, request, info): + for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) checksum = md5sum(buf) @@ -119,8 +119,8 @@ class ImagesPipeline(FilesPipeline): headers={'Content-Type': 'image/jpeg'}) return checksum - def get_images(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def get_images(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size @@ -166,7 +166,7 @@ class ImagesPipeline(FilesPipeline): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return 'full/%s.jpg' % (image_guid) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index aa65f4f0e..2439de9a5 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,12 +1,16 @@ import functools import logging from collections import defaultdict +from inspect import signature +from warnings import warn + from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -27,6 +31,7 @@ class MediaPipeline: def __init__(self, download_func=None, settings=None): self.download_func = download_func + self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -38,6 +43,9 @@ class MediaPipeline: ) self._handle_statuses(self.allow_redirects) + # Check if deprecated methods are being used and make them compatible + self._make_compatible() + def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -77,11 +85,11 @@ class MediaPipeline: def process_item(self, item, spider): info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) - dlist = [self._process_request(r, info) for r in requests] + dlist = [self._process_request(r, info, item) for r in requests] dfd = DeferredList(dlist, consumeErrors=1) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info): + def _process_request(self, request, info, item): fp = request_fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback @@ -102,34 +110,73 @@ class MediaPipeline: # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info) - dfd.addCallback(self._check_media_to_download, request, info) + dfd = mustbe_deferred(self.media_to_download, request, info, item=item) + dfd.addCallback(self._check_media_to_download, request, info, item=item) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(lambda f: logger.error( f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider}) ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _make_compatible(self): + """Make overridable methods of MediaPipeline and subclasses backwards compatible""" + methods = [ + "file_path", "media_to_download", "media_downloaded", + "file_downloaded", "image_downloaded", "get_images" + ] + + for method_name in methods: + method = getattr(self, method_name, None) + if callable(method): + setattr(self, method_name, self._compatible(method)) + + def _compatible(self, func): + """Wrapper for overridable methods to allow backwards compatibility""" + self._check_signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if self._expects_item[func.__name__]: + return func(*args, **kwargs) + + kwargs.pop('item', None) + return func(*args, **kwargs) + + return wrapper + + def _check_signature(self, func): + sig = signature(func) + self._expects_item[func.__name__] = True + + if 'item' not in sig.parameters: + old_params = str(sig)[1:-1] + new_params = old_params + ", *, item=None" + warn('%s(self, %s) is deprecated, ' + 'please use %s(self, %s)' + % (func.__name__, old_params, func.__name__, new_params), + ScrapyDeprecationWarning, stacklevel=2) + self._expects_item[func.__name__] = False + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - def _check_media_to_download(self, result, request, info): + def _check_media_to_download(self, result, request, info, item): if result is not None: return result if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) return dfd @@ -171,7 +218,7 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): """Check request before starting download""" pass @@ -179,7 +226,7 @@ class MediaPipeline: """Returns the media requests to download""" pass - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" return response @@ -199,3 +246,7 @@ class MediaPipeline: extra={'spider': info.spider} ) return item + + def file_path(self, request, response=None, info=None, *, item=None): + """Returns the path where downloaded media should be stored""" + pass diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a023dfcc8..b19b4ff2a 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase): for p in patchers: p.stop() + def test_file_path_from_item(self): + """ + Custom file path based on item data, overriding default implementation + """ + class CustomFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, item=None): + return 'full/%s' % item.get('path') + + file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(file_path(request, item=item), 'full/path-to-store-file') + class FilesPipelineTestCaseFieldsMixin: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 4f130c0c9..6afd47497 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info @@ -169,7 +171,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('download') return super().download(request, info) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') @@ -179,7 +181,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('get_media_requests') return item.get('requests') - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): self._mockcalled.append('media_downloaded') return super().media_downloaded(response, request, info) @@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ['get_media_requests', 'media_to_download', 'item_completed']) +class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): + + def __init__(self, *args, **kwargs): + super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + self._mockcalled = [] + + def get_media_requests(self, item, info): + item_url = item['image_urls'][0] + return Request( + item_url, + meta={'response': Response(item_url, status=200, body=b'data')} + ) + + def inc_stats(self, *args, **kwargs): + return True + + def media_to_download(self, request, info): + self._mockcalled.append('media_to_download') + return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + + def media_downloaded(self, response, request, info): + self._mockcalled.append('media_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + + def file_downloaded(self, response, request, info): + self._mockcalled.append('file_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + + def file_path(self, request, response=None, info=None): + self._mockcalled.append('file_path') + return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + + def get_images(self, response, request, info): + self._mockcalled.append('get_images') + return [] + + def image_downloaded(self, response, request, info): + self._mockcalled.append('image_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + + +class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + + def setUp(self): + self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe.open_spider(None) + self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) + + def _assert_method_called_with_warnings(self, method, message, warnings): + self.assertIn(method, self.pipe._mockcalled) + warningShown = False + for warning in warnings: + if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning: + warningShown = True + self.assertTrue(warningShown) + + @inlineCallbacks + def test_media_to_download_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_to_download(self, request, info) is deprecated, ' + 'please use media_to_download(self, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_to_download', message, warnings) + + @inlineCallbacks + def test_media_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_downloaded(self, response, request, info) is deprecated, ' + 'please use media_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_downloaded', message, warnings) + + @inlineCallbacks + def test_file_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_downloaded(self, response, request, info) is deprecated, ' + 'please use file_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('file_downloaded', message, warnings) + + @inlineCallbacks + def test_file_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_path(self, request, response=None, info=None) is deprecated, ' + 'please use file_path(self, request, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('file_path', message, warnings) + + @inlineCallbacks + def test_get_images_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'get_images(self, response, request, info) is deprecated, ' + 'please use get_images(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('get_images', message, warnings) + + @inlineCallbacks + def test_image_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'image_downloaded(self, response, request, info) is deprecated, ' + 'please use image_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('image_downloaded', message, warnings) + + class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): From 394631fc0a731a0b8b0108edc1711c0c87e2ca15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:08:09 +0200 Subject: [PATCH 075/119] Restore 3.5 support for mitmproxy-based tests --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 4fa58d11b..b67c08403 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,8 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -mitmproxy >=4, <5 # The latest version does not support some pinned dependencies +mitmproxy >=4, <5; python_version < '3.6' # The latest version does not support some pinned dependencies +mitmproxy <4; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From 8e393a0b218b56cc8a70f7bda277969aa026790d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:29:51 +0200 Subject: [PATCH 076/119] Do not change the mitmproxy version for no-3.6 Python versions --- tests/requirements-py3.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b67c08403..b425ce771 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,8 +1,9 @@ # Tests requirements attrs -dataclasses; python_version == '3.6' -mitmproxy >=4, <5; python_version < '3.6' # The latest version does not support some pinned dependencies -mitmproxy <4; python_version < '3.6' +dataclasses; python_version ==3.6 +mitmproxy; python_version >=3.7 +mitmproxy >=4, <5; python_version >=3.6, <3.7 +mitmproxy <4; python_version <3.6 # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From b1de55d37d6a1b84e92df2d93d83e247e5d0e0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 12:34:40 +0200 Subject: [PATCH 077/119] Fix marker syntax --- tests/requirements-py3.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b425ce771..b51177abb 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,9 +1,9 @@ # Tests requirements attrs -dataclasses; python_version ==3.6 -mitmproxy; python_version >=3.7 -mitmproxy >=4, <5; python_version >=3.6, <3.7 -mitmproxy <4; python_version <3.6 +dataclasses; python_version == '3.6' +mitmproxy; python_version >= '3.7' +mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' +mitmproxy < 4; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines From 125a058340cf77a70fe605b0317b3c517f637c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 17:07:21 +0200 Subject: [PATCH 078/119] Do not let umask affect the permissions of startproject-generated files --- scrapy/utils/template.py | 6 +++-- tests/test_commands.py | 57 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 96ff4b09b..f068be737 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -12,10 +12,12 @@ def render_templatefile(path, **kwargs): content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path + + if path.endswith('.tmpl'): + os.rename(path, render_path) + with open(render_path, 'wb') as fp: fp.write(content.encode('utf8')) - if path.endswith('.tmpl'): - os.remove(path) CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/tests/test_commands.py b/tests/test_commands.py index 8938156fc..be88e3511 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -126,9 +126,13 @@ class StartprojectTest(ProjectTest): def get_permissions_dict(path, renamings=None, ignore=None): + + def get_permissions(path): + return oct(os.stat(path).st_mode) + renamings = renamings or tuple() permissions_dict = { - '.': os.stat(path).st_mode, + '.': get_permissions(path), } for root, dirs, files in os.walk(path): nodes = list(chain(dirs, files)) @@ -143,13 +147,15 @@ def get_permissions_dict(path, renamings=None, ignore=None): search_string, replacement ) - permissions = os.stat(absolute_path).st_mode + permissions = get_permissions(absolute_path) permissions_dict[relative_path] = permissions return permissions_dict class StartprojectTemplatesTest(ProjectTest): + maxDiff = None + def setUp(self): super().setUp() self.tmpl = join(self.temp_path, 'templates') @@ -311,6 +317,53 @@ class StartprojectTemplatesTest(ProjectTest): self.assertEqual(actual_permissions, expected_permissions) + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = os.path.join( + scrapy_path, + 'templates', + 'project' + ) + project_name = 'umaskproject' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + class CommandTest(ProjectTest): From 4c0afb606c59e6c8eabe4bf97cdb0494cf26dec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Aug 2020 17:45:26 +0200 Subject: [PATCH 079/119] Update permission expectations --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index be88e3511..55088c605 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -297,7 +297,7 @@ class StartprojectTemplatesTest(ProjectTest): path.mkdir(mode=permissions) else: path.touch(mode=permissions) - expected_permissions[node] = path.stat().st_mode + expected_permissions[node] = oct(path.stat().st_mode) process = subprocess.Popen( ( From 5f4df622a17b1d8f9c3b9b693cb2f7119ffec27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 05:41:06 +0200 Subject: [PATCH 080/119] test_utils_iterators.py: support Windows the right way --- tests/__init__.py | 2 +- tests/test_utils_iterators.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..4253017fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb') as f: + with open(path, 'rb', newline='') as f: return f.read() diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 298178f08..50190d4d1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = "foo{}bar".format(os.linesep) - - class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) @@ -267,7 +264,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual(result, [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr @@ -283,7 +280,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): @@ -296,7 +293,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv1], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) @@ -305,7 +302,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv2], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): @@ -327,7 +324,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): @@ -353,7 +350,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): From 24ba5a71aca9b368eee21bd7d9043a7d26dba403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 06:35:09 +0200 Subject: [PATCH 081/119] Maybe the problem is not in the code after all --- .gitattributes | 1 + tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/tests/__init__.py b/tests/__init__.py index 4253017fa..12ce79fa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb', newline='') as f: + with open(path, 'rb') as f: return f.read() From 65e0abaea5eddb1dbb28b2bca9cd00e9b9409471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 09:05:51 +0200 Subject: [PATCH 082/119] Document FEED_URI_PARAMS --- docs/topics/feed-exports.rst | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 37b7096f6..0f0f258dc 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -321,13 +321,14 @@ The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition. * ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. + See :ref:`topics-feed-format` for possible values. Mandatory, no fallback setting +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` * ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING @@ -500,7 +501,7 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. +* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch. Use :ref:`printf-style string formatting ` to alter the number format. For example, to make the batch ID a 5-digit @@ -517,16 +518,74 @@ And your :command:`crawl` command line is:: The command line above can generate a directory tree like:: -->projectname --->dirname ---->1-filename2020-03-28T14-45-08.237134.json ---->2-filename2020-03-28T14-45-09.148903.json ---->3-filename2020-03-28T14-45-10.046092.json + ->projectname + -->dirname + --->1-filename2020-03-28T14-45-08.237134.json + --->2-filename2020-03-28T14-45-09.148903.json + --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains 100 items or fewer. +.. setting:: FEED_URI_PARAMS + +FEED_URI_PARAMS +--------------- + +Default: ``None`` + +A string with the import path of a function to set the parameters to apply with +:ref:`printf-style string formatting ` to the +feed URI. + +The function signature should be as follows: + +.. function:: uri_params(params, spider) + + Return a :class:`dict` of key-value pairs to apply to the feed URI using + :ref:`printf-style string formatting `. + + :param params: default key-value pairs + + Specifically: + + - ``batch_id``: ID of the file batch. See + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` + is always ``1``. + + - ``batch_time``: UTC date and time, in ISO format with ``:`` + replaced with ``-``. + + See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + - ``time``: ``batch_time``, with microseconds set to ``0``. + :type params: dict + + :param spider: source spider of the feed items + :type spider: scrapy.spiders.Spider + +For example, to include the :attr:`name ` of the +source spider in the feed URI: + +#. Define the following function somewhere in your project:: + + # myproject/utils.py + def uri_params(params, spider): + return {**params, 'spider_name': spider.name} + +#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:: + + # myproject/settings.py + FEED_URI_PARAMS = 'myproject.utils.uri_params' + +#. Use ``%(spider_name)s`` in your feed URI:: + + scrapy crawl -o "%(spider_name)s.jl" + + .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore From 756c368a6b0d2eef65f86f8418f9a7fbeff036c7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Aug 2020 22:09:24 +0500 Subject: [PATCH 083/119] Use a longer key in mitmproxy-ca.pem. --- tests/keys/mitmproxy-ca.pem | 78 +++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index 08004feca..cdef75f99 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,32 +1,50 @@ ------BEGIN RSA PRIVATE KEY----- -MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt -auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha -QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB -AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF -QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol -Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w -T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll -KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj -58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc -1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT -9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB -dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn -Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB +/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT +NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ +u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH +JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB +eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls +90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ +epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 +VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn +v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT +I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU +oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au +/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG +gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn +YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf +dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 +9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC +QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu +bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW +CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR +OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu +PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK +eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx +BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ +B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy +HzPh2taMnCA0h1fNLFFBkw88 +-----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT -CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX -DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt -aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u -YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp -w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk -Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB -/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG -CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB -gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG -+EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n -MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D -y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z -e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1 -n95u +MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM +CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX +DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt +aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW +ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 +ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg +z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI +ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE +6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ +0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC +AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD +BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG +CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA +A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd +kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe +gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY +1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj +envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj +wiKjdLjCmJgjGAQjRw5VYJ8JI0XO -----END CERTIFICATE----- From 2aa4f3cbf96ad55aa4a1fa064758338daf16b110 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Mon, 17 Aug 2020 05:39:59 -0300 Subject: [PATCH 084/119] Conditional request attribute binding for responses (#4632) --- docs/topics/signals.rst | 5 + scrapy/core/engine.py | 13 +- scrapy/core/scraper.py | 62 ++++---- setup.cfg | 3 + tests/test_request_attribute_binding.py | 202 ++++++++++++++++++++++++ 5 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 tests/test_request_attribute_binding.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 255ba9d3f..1d99d8c28 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -423,6 +423,11 @@ response_received :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: The ``request`` argument might not contain the original request that + reached the downloader, if a :ref:`topics-downloader-middleware` modifies + the :class:`~scrapy.http.Response` object and sets a specific ``request`` + attribute. + response_downloaded ~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 86a6abb23..5e0dfe37c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -243,12 +243,17 @@ class ExecutionEngine: % (type(response), response) ) if isinstance(response, Response): - response.request = request # tie request to response received - logkws = self.logformatter.crawled(request, response, spider) + if response.request is None: + response.request = request + logkws = self.logformatter.crawled(response.request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log( + signal=signals.response_received, + response=response, + request=response.request, + spider=spider, + ) return response def _on_complete(_): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1ef0790a9..20bdb22a1 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -12,7 +12,7 @@ from scrapy import signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel +from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output @@ -120,40 +120,40 @@ class Scraper: response, request, deferred = slot.next_response_request_deferred() self._scrape(response, request, spider).chainDeferred(deferred) - def _scrape(self, response, request, spider): - """Handle the downloaded response or failure through the spider - callback/errback""" - if not isinstance(response, (Response, Failure)): - raise TypeError( - "Incorrect type: expected Response or Failure, got %s: %r" - % (type(response), response) - ) - - dfd = self._scrape2(response, request, spider) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, response, spider) - dfd.addCallback(self.handle_spider_output, request, response, spider) + def _scrape(self, result, request, spider): + """ + Handle the downloaded response or failure through the spider callback/errback + """ + if not isinstance(result, (Response, Failure)): + raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) + dfd = self._scrape2(result, request, spider) # returns spider's processed output + dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addCallback(self.handle_spider_output, request, result, spider) return dfd - def _scrape2(self, request_result, request, spider): - """Handle the different cases of request's result been a Response or a - Failure""" - if not isinstance(request_result, Failure): - return self.spidermw.scrape_response( - self.call_spider, request_result, request, spider) - else: - dfd = self.call_spider(request_result, request, spider) - return dfd.addErrback( - self._log_download_errors, request_result, request, spider) + def _scrape2(self, result, request, spider): + """ + Handle the different cases of request's result been a Response or a Failure + """ + if isinstance(result, Response): + return self.spidermw.scrape_response(self.call_spider, result, request, spider) + else: # result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result, request, spider): - result.request = request - dfd = defer_result(result) - callback = request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) - warn_on_generator_with_return_value(spider, request.errback) - dfd.addCallbacks(callback=callback, - errback=request.errback, - callbackKeywords=request.cb_kwargs) + if isinstance(result, Response): + if getattr(result, "request", None) is None: + result.request = request + callback = result.request.callback or spider._parse + warn_on_generator_with_return_value(spider, callback) + dfd = defer_succeed(result) + dfd.addCallback(callback, **result.request.cb_kwargs) + else: # result is a Failure + result.request = request + warn_on_generator_with_return_value(spider, request.errback) + dfd = defer_fail(result) + dfd.addErrback(request.errback) return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, response, spider): diff --git a/setup.cfg b/setup.cfg index f8e7c0c91..3a624ec94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,6 +130,9 @@ ignore_errors = True [mypy-tests.test_pipelines] ignore_errors = True +[mypy-tests.test_request_attribute_binding] +ignore_errors = True + [mypy-tests.test_request_cb_kwargs] ignore_errors = True diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py new file mode 100644 index 000000000..b60b7c579 --- /dev/null +++ b/tests/test_request_attribute_binding.py @@ -0,0 +1,202 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy import Request, signals +from scrapy.crawler import CrawlerRunner +from scrapy.http.response import Response + +from testfixtures import LogCapture + +from tests.mockserver import MockServer +from tests.spiders import SingleRequestSpider + + +OVERRIDEN_URL = "https://example.org" + + +class ProcessResponseMiddleware: + def process_response(self, request, response, spider): + return response.replace(request=Request(OVERRIDEN_URL)) + + +class RaiseExceptionRequestMiddleware: + def process_request(self, request, spider): + 1 / 0 + return request + + +class CatchExceptionOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + request=Request(OVERRIDEN_URL), + ) + + +class CatchExceptionDoNotOverrideRequestMiddleware: + def process_exception(self, request, exception, spider): + return Response( + url="http://localhost/", + body=b"Caught " + exception.__class__.__name__.encode("utf-8"), + ) + + +class AlternativeCallbacksSpider(SingleRequestSpider): + name = "alternative_callbacks_spider" + + def alt_callback(self, response, foo=None): + self.logger.info("alt_callback was invoked with foo=%s", foo) + + +class AlternativeCallbacksMiddleware: + def process_response(self, request, response, spider): + new_request = request.replace( + url=OVERRIDEN_URL, + callback=spider.alt_callback, + cb_kwargs={"foo": "bar"}, + ) + return response.replace(request=new_request) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_response_200(self): + url = self.mockserver.url("/status?n=200") + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_response_error(self): + for status in ("404", "500"): + url = self.mockserver.url("/status?n={}".format(status)) + crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + response = failure.value.response + self.assertEqual(failure.request.url, url) + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_raise_exception(self): + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + failure = crawler.spider.meta["failure"] + self.assertEqual(failure.request.url, url) + self.assertIsInstance(failure.value, ZeroDivisionError) + + @defer.inlineCallbacks + def test_downloader_middleware_override_request_in_process_response(self): + """ + Downloader middleware which returns a response with an specific 'request' attribute. + + * The spider callback should receive the overriden response.request + * Handlers listening to the response_received signal should receive the overriden response.request + * The "crawled" log message should show the overriden response.request + """ + signal_params = {} + + def signal_handler(response, request, spider): + signal_params["response"] = response + signal_params["request"] = request + + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".ProcessResponseMiddleware": 595, + } + }) + crawler = runner.create_crawler(SingleRequestSpider) + crawler.signals.connect(signal_handler, signal=signals.response_received) + + with LogCapture() as log: + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.request.url, OVERRIDEN_URL) + + self.assertEqual(signal_params["response"].url, url) + self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) + + log.check_present( + ("scrapy.core.engine", "DEBUG", "Crawled (200) (referer: None)".format(OVERRIDEN_URL)), + ) + + @defer.inlineCallbacks + def test_downloader_middleware_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response with a specific 'request' attribute. + + The spider callback should receive the overriden response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, OVERRIDEN_URL) + + @defer.inlineCallbacks + def test_downloader_middleware_do_not_override_in_process_exception(self): + """ + An exception is raised but caught by the next middleware, which + returns a Response without a specific 'request' attribute. + + The spider callback should receive the original response.request + """ + url = self.mockserver.url("/status?n=200") + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".RaiseExceptionRequestMiddleware": 590, + __name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595, + }, + }) + crawler = runner.create_crawler(SingleRequestSpider) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + response = crawler.spider.meta["responses"][0] + self.assertEqual(response.body, b"Caught ZeroDivisionError") + self.assertEqual(response.request.url, url) + + @defer.inlineCallbacks + def test_downloader_middleware_alternative_callback(self): + """ + Downloader middleware which returns a response with a + specific 'request' attribute, with an alternative callback + """ + runner = CrawlerRunner(settings={ + "DOWNLOADER_MIDDLEWARES": { + __name__ + ".AlternativeCallbacksMiddleware": 595, + } + }) + crawler = runner.create_crawler(AlternativeCallbacksSpider) + + with LogCapture() as log: + url = self.mockserver.url("/status?n=200") + yield crawler.crawl(seed=url, mockserver=self.mockserver) + + log.check_present( + ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"), + ) From a8e08d51cd50e8f0b58a60992d310e56d4f71641 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Mon, 17 Aug 2020 14:15:52 +0530 Subject: [PATCH 085/119] Check if file is already present on running `scrapy genspider` and terminate if so (#4623) --- scrapy/commands/genspider.py | 41 ++++++++++++++++------ tests/test_commands.py | 67 +++++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 4c7548e9c..74a077d1b 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -66,16 +66,9 @@ class Command(ScrapyCommand): print("Cannot create a spider with the same name as your project") return - try: - spidercls = self.crawler_process.spider_loader.load(name) - except KeyError: - pass - else: - # if spider already exists and not --force then halt - if not opts.force: - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) - return + if not opts.force and self._spider_exists(name): + return + template_file = self._find_template(opts.template) if template_file: self._genspider(module, name, domain, opts.template, template_file) @@ -119,6 +112,34 @@ class Command(ScrapyCommand): if filename.endswith('.tmpl'): print(" %s" % splitext(filename)[0]) + def _spider_exists(self, name): + if not self.settings.get('NEWSPIDER_MODULE'): + # if run as a standalone command and file with same filename already exists + if exists(name + ".py"): + print("%s already exists" % (abspath(name + ".py"))) + return True + return False + + try: + spidercls = self.crawler_process.spider_loader.load(name) + except KeyError: + pass + else: + # if spider with same name exists + print("Spider %r already exists in module:" % name) + print(" %s" % spidercls.__module__) + return True + + # a file with the same name exists in the target directory + spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) + spiders_dir = dirname(spiders_module.__file__) + spiders_dir_abs = abspath(spiders_dir) + if exists(join(spiders_dir_abs, name + ".py")): + print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) + return True + + return False + @property def templates_dir(self): return join( diff --git a/tests/test_commands.py b/tests/test_commands.py index 8938156fc..109f006a8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -8,7 +8,7 @@ import sys import tempfile from contextlib import contextmanager from itertools import chain -from os.path import exists, join, abspath +from os.path import exists, join, abspath, getmtime from pathlib import Path from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION @@ -337,8 +337,11 @@ class GenspiderCommandTest(CommandTest): p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) + modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn("Spider %r already exists in module" % spname, out) + modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): self.test_template('basic') @@ -360,6 +363,40 @@ class GenspiderCommandTest(CommandTest): self.assertEqual(2, self.call('genspider', self.project_name)) assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + def test_same_filename_as_existing_spider(self, force=False): + file_name = 'example' + file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name) + self.assertEqual(0, self.call('genspider', file_name, 'example.com')) + assert exists(file_path) + + # change name of spider but not its file name + with open(file_path, 'r+') as spider_file: + file_data = spider_file.read() + file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = getmtime(file_path) + file_contents_before = file_data + + if force: + p, out, err = self.proc('genspider', '--force', file_name, 'example.com') + self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("%s already exists" % (file_path), out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) + class GenspiderStandaloneCommandTest(ProjectTest): @@ -367,6 +404,34 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.call('genspider', 'example', 'example.com') assert exists(join(self.temp_path, 'example.py')) + def test_same_name_as_existing_file(self, force=False): + file_name = 'example' + file_path = join(self.temp_path, file_name + '.py') + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("Created spider %r using template \'basic\' " % file_name, out) + assert exists(file_path) + modify_time_before = getmtime(file_path) + file_contents_before = open(file_path, 'r').read() + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') + self.assertIn("Created spider %r using template \'crawl\' " % file_name, out) + modify_time_after = getmtime(file_path) + self.assertNotEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertNotEqual(file_contents_after, file_contents_before) + else: + p, out, err = self.proc('genspider', file_name, 'example.com') + self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out) + modify_time_after = getmtime(file_path) + self.assertEqual(modify_time_after, modify_time_before) + file_contents_after = open(file_path, 'r').read() + self.assertEqual(file_contents_after, file_contents_before) + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) + class MiscCommandsTest(CommandTest): From 55edf8d3b8885541cdbf9d1c62d9a6bbf634e2a0 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Mon, 17 Aug 2020 18:50:52 +1000 Subject: [PATCH 086/119] Add typing hint to httpcache downloadermiddlewares (#4243) --- scrapy/downloadermiddlewares/httpcache.py | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6db57bd8b..62f1c3a29 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,4 +1,5 @@ from email.utils import formatdate +from typing import Optional, Type, TypeVar from twisted.internet import defer from twisted.internet.error import ( @@ -13,10 +14,19 @@ from twisted.internet.error import ( from twisted.web.client import ResponseFailed from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http.request import Request +from scrapy.http.response import Response +from scrapy.settings import Settings +from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector from scrapy.utils.misc import load_object +HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware") + + class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,7 +34,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) - def __init__(self, settings, stats): + def __init__(self, settings: Settings, stats: StatsCollector) -> None: if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) @@ -33,26 +43,26 @@ class HttpCacheMiddleware: self.stats = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV: o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.storage.open_spider(spider) - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: self.storage.close_spider(spider) - def process_request(self, request, spider): + def process_request(self, request: Request, spider: Spider) -> Optional[Response]: if request.meta.get('dont_cache', False): - return + return None # Skip uncacheable requests if not self.policy.should_cache_request(request): request.meta['_dont_cache'] = True # flag as uncacheable - return + return None # Look for cached response and check if expired cachedresponse = self.storage.retrieve_response(spider, request) @@ -61,7 +71,7 @@ class HttpCacheMiddleware: if self.ignore_missing: self.stats.inc_value('httpcache/ignore', spider=spider) raise IgnoreRequest("Ignored request not in cache: %s" % request) - return # first time request + return None # first time request # Return cached response only if not expired cachedresponse.flags.append('cached') @@ -73,7 +83,9 @@ class HttpCacheMiddleware: # process_response hook request.meta['cached_response'] = cachedresponse - def process_response(self, request, response, spider): + return None + + def process_response(self, request: Request, response: Response, spider: Spider) -> Response: if request.meta.get('dont_cache', False): return response @@ -85,7 +97,7 @@ class HttpCacheMiddleware: # RFC2616 requires origin server to set Date header, # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: - response.headers['Date'] = formatdate(usegmt=1) + response.headers['Date'] = formatdate(usegmt=True) # Do not validate first-hand responses cachedresponse = request.meta.pop('cached_response', None) @@ -102,13 +114,18 @@ class HttpCacheMiddleware: self._cache_response(spider, response, request, cachedresponse) return response - def process_exception(self, request, exception, spider): + def process_exception( + self, request: Request, exception: Exception, spider: Spider + ) -> Optional[Response]: cachedresponse = request.meta.pop('cached_response', None) if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS): self.stats.inc_value('httpcache/errorrecovery', spider=spider) return cachedresponse + return None - def _cache_response(self, spider, response, request, cachedresponse): + def _cache_response( + self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response] + ) -> None: if self.policy.should_cache_response(response, request): self.stats.inc_value('httpcache/store', spider=spider) self.storage.store_response(spider, request, response) From e70975f0bb4e8378bf0e00ae7a7014247cd59441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Aug 2020 15:10:08 +0200 Subject: [PATCH 087/119] Allow overwriting feeds (#4512) Co-authored-by: Yuval Hager --- docs/faq.rst | 6 +- docs/intro/overview.rst | 26 +- docs/intro/tutorial.rst | 13 +- docs/topics/feed-exports.rst | 47 +++- scrapy/commands/__init__.py | 15 +- scrapy/extensions/feedexport.py | 142 +++++++--- scrapy/utils/conf.py | 44 +++- scrapy/utils/ftp.py | 6 +- scrapy/utils/python.py | 3 +- tests/ftpserver.py | 24 ++ tests/mockserver.py | 26 ++ tests/requirements-py3.txt | 1 + tests/test_commands.py | 126 +++++++++ tests/test_feedexport.py | 442 ++++++++++++++++++++++++++++---- tests/test_utils_conf.py | 16 ++ tests/test_utils_python.py | 4 + 16 files changed, 791 insertions(+), 150 deletions(-) create mode 100644 tests/ftpserver.py diff --git a/docs/faq.rst b/docs/faq.rst index ea2c8216f..9346ec358 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -o items.json + scrapy crawl myspider -O items.json To dump into a CSV file:: - scrapy crawl myspider -o items.csv + scrapy crawl myspider -O items.csv To dump into a XML file:: - scrapy crawl myspider -o items.xml + scrapy crawl myspider -O items.xml For more information see :ref:`topics-feed-exports` diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 01986b594..dd80c7bd0 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination:: if next_page is not None: yield response.follow(next_page, self.parse) - Put this in a text file, name it to something like ``quotes_spider.py`` and run the spider using the :command:`runspider` command:: - scrapy runspider quotes_spider.py -o quotes.json + scrapy runspider quotes_spider.py -o quotes.jl +When this finishes you will have in the ``quotes.jl`` file a list of the +quotes in JSON Lines format, containing text and author, looking like this:: -When this finishes you will have in the ``quotes.json`` file a list of the -quotes in JSON format, containing text and author, looking like this (reformatted -here for better readability):: - - [{ - "author": "Jane Austen", - "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" - }, - { - "author": "Groucho Marx", - "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" - }, - { - "author": "Steve Martin", - "text": "\u201cA day without sunshine is like, you know, night.\u201d" - }, - ...] + {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"} + {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"} + {"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"} + ... What just happened? diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5f35dc936..f96c78887 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -464,16 +464,15 @@ Storing the scraped data The simplest way to store the scraped data is by using :ref:`Feed exports `, with the following command:: - scrapy crawl quotes -o quotes.json + scrapy crawl quotes -O quotes.json That will generate an ``quotes.json`` file containing all scraped items, serialized in `JSON`_. -For historic reasons, Scrapy appends to a given file instead of overwriting -its contents. If you run this command twice without removing the file -before the second time, you'll end up with a broken JSON file. - -You can also use other formats, like `JSON Lines`_:: +The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead +to append new content to any existing file. However, appending to a JSON file +makes the file contents invalid JSON. When appending to a file, consider +using a different serialization format, such as `JSON Lines`_:: scrapy crawl quotes -o quotes.jl @@ -704,7 +703,7 @@ Using spider arguments You can provide command line arguments to your spiders by using the ``-a`` option when running them:: - scrapy crawl quotes -o quotes-humor.json -a tag=humor + scrapy crawl quotes -O quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become spider attributes by default. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0f0f258dc..cd4f7cf29 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -291,6 +291,7 @@ Default: ``{}`` A dictionary in which every key is a feed URI (or a :class:`pathlib.Path` object) and each value is a nested dictionary containing configuration parameters for the specific feed. + This setting is required for enabling the feed export feature. See :ref:`topics-feed-storage-backends` for supported URI schemes. @@ -318,17 +319,43 @@ For instance:: } The following is a list of the accepted keys and the setting that is used -as a fallback value if that key is not provided for a specific feed definition. +as a fallback value if that key is not provided for a specific feed definition: + +- ``format``: the :ref:`serialization format `. + + This setting is mandatory, there is no fallback value. + +- ``batch_item_count``: falls back to + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + +- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`. + +- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. + +- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. + +- ``overwrite``: whether to overwrite the file if it already exists + (``True``) or append to its content (``False``). + + The default value depends on the :ref:`storage backend + `: + + - :ref:`topics-feed-storage-fs`: ``False`` + + - :ref:`topics-feed-storage-ftp`: ``True`` + + .. note:: Some FTP servers may not support appending to files (the + ``APPE`` FTP command). + + - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported + `_) + + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) + +- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`. + +- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`. -* ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. - Mandatory, no fallback setting -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` -* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` -* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` -* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` -* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 57ce4e522..cfd940fe7 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + if opts.output or opts.overwrite_output: + feeds = feed_process_params_from_cli( + self.settings, + opts.output, + opts.output_format, + opts.overwrite_output, + ) self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b7a4e362e..980825499 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.python import without_none_values +from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): + argument_names = get_func_args(builder) + if 'feed_options' in argument_names: + kwargs['feed_options'] = feed_options + else: + warnings.warn( + "{} does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove this " + "warning. This parameter will become mandatory in a future " + "version of Scrapy." + .format(builder.__qualname__), + category=ScrapyDeprecationWarning + ) + return builder(*preargs, uri, *args, **kwargs) + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" - def __init__(uri): - """Initialize the storage with the parameters given in the URI""" + def __init__(uri, *, feed_options=None): + """Initialize the storage with the parameters given in the URI and the + feed-specific options (see :setting:`FEEDS`)""" def open(spider): """Open the storage for the given spider. It must return a file-like @@ -64,10 +81,15 @@ class BlockingFeedStorage: @implementer(IFeedStorage) class StdoutFeedStorage: - def __init__(self, uri, _stdout=None): + def __init__(self, uri, _stdout=None, *, feed_options=None): if not _stdout: _stdout = sys.stdout.buffer self._stdout = _stdout + if feed_options and feed_options.get('overwrite', False) is True: + logger.warning('Standard output (stdout) storage does not support ' + 'overwriting. To suppress this warning, remove the ' + 'overwrite option from your FEEDS setting, or set ' + 'it to False.') def open(self, spider): return self._stdout @@ -79,14 +101,16 @@ class StdoutFeedStorage: @implementer(IFeedStorage) class FileFeedStorage: - def __init__(self, uri): + def __init__(self, uri, *, feed_options=None): self.path = file_uri_to_path(uri) + feed_options = feed_options or {} + self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' def open(self, spider): dirname = os.path.dirname(self.path) if dirname and not os.path.exists(dirname): os.makedirs(dirname) - return open(self.path, 'ab') + return open(self.path, self.write_mode) def store(self, file): file.close() @@ -94,7 +118,8 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): - def __init__(self, uri, access_key=None, secret_key=None, acl=None): + def __init__(self, uri, access_key=None, secret_key=None, acl=None, *, + feed_options=None): u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key @@ -111,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage): else: import boto self.connect_s3 = boto.connect_s3 + if feed_options and feed_options.get('overwrite', True) is False: + logger.warning('S3 does not support appending to files. To ' + 'suppress this warning, remove the overwrite ' + 'option from your FEEDS setting or set it to True.') @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], - acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None + acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, + feed_options=feed_options, ) def _store_in_thread(self, file): @@ -135,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage): kwargs = {'policy': self.acl} if self.acl else {} key.set_contents_from_file(file, **kwargs) key.close() + file.close() class GCSFeedStorage(BlockingFeedStorage): @@ -165,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False): + def __init__(self, uri, use_active_mode=False, *, feed_options=None): u = urlparse(uri) self.host = u.hostname self.port = int(u.port or '21') self.username = u.username - self.password = unquote(u.password) + self.password = unquote(u.password or '') self.path = u.path self.use_active_mode = use_active_mode + self.overwrite = not feed_options or feed_options.get('overwrite', True) @classmethod - def from_crawler(cls, crawler, uri): - return cls( - uri=uri, - use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE') + def from_crawler(cls, crawler, uri, *, feed_options=None): + return build_storage( + cls, + uri, + crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'), + feed_options=feed_options, ) def _store_in_thread(self, file): ftp_store_file( path=self.path, file=file, host=self.host, port=self.port, username=self.username, - password=self.password, use_active_mode=self.use_active_mode + password=self.password, use_active_mode=self.use_active_mode, + overwrite=self.overwrite, ) @@ -242,32 +278,32 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects - feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed in self.settings.getdict('FEEDS').items(): + for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects - self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - for uri, feed in self.feeds.items(): - if not self._storage_supported(uri): + for uri, feed_options in self.feeds.items(): + if not self._storage_supported(uri, feed_options): raise NotConfigured if not self._settings_are_valid(): raise NotConfigured - if not self._exporter_supported(feed['format']): + if not self._exporter_supported(feed_options['format']): raise NotConfigured def open_spider(self, spider): - for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params']) + for uri, feed_options in self.feeds.items(): + uri_params = self._get_uri_params(spider, feed_options['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, - feed=feed, + feed_options=feed_options, spider=spider, uri_template=uri, )) @@ -306,32 +342,32 @@ class FeedExporter: ) return d - def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): + def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template): """ Redirect the output data stream to a new file. Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start - :param feed: dict with parameters of feed + :param feed_options: dict with parameters of feed :param spider: user spider :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ - storage = self._get_storage(uri) + storage = self._get_storage(uri, feed_options) file = storage.open(spider) exporter = self._get_exporter( file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], + format=feed_options['format'], + fields_to_export=feed_options['fields'], + encoding=feed_options['encoding'], + indent=feed_options['indent'], ) slot = _FeedSlot( file=file, exporter=exporter, storage=storage, uri=uri, - format=feed['format'], - store_empty=feed['store_empty'], + format=feed_options['format'], + store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, ) @@ -355,7 +391,7 @@ class FeedExporter: slots.append(self._start_new_batch( batch_id=slot.batch_id + 1, uri=slot.uri_template % uri_params, - feed=self.feeds[slot.uri_template], + feed_options=self.feeds[slot.uri_template], spider=spider, uri_template=slot.uri_template, )) @@ -394,11 +430,11 @@ class FeedExporter: return False return True - def _storage_supported(self, uri): + def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme if scheme in self.storages: try: - self._get_storage(uri) + self._get_storage(uri, feed_options) return True except NotConfigured as e: logger.error("Disabled feed storage scheme: %(scheme)s. " @@ -416,8 +452,30 @@ class FeedExporter: def _get_exporter(self, file, format, *args, **kwargs): return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri): - return self._get_instance(self.storages[urlparse(uri).scheme], uri) + def _get_storage(self, uri, feed_options): + """Fork of create_instance specific to feed storage classes + + It supports not passing the *feed_options* parameters to classes that + do not support it, and issuing a deprecation warning instead. + """ + feedcls = self.storages[urlparse(uri).scheme] + crawler = getattr(self, 'crawler', None) + + def build_instance(builder, *preargs): + return build_storage(builder, uri, preargs=preargs) + + if crawler and hasattr(feedcls, 'from_crawler'): + instance = build_instance(feedcls.from_crawler, crawler) + method_name = 'from_crawler' + elif hasattr(feedcls, 'from_settings'): + instance = build_instance(feedcls.from_settings, self.settings) + method_name = 'from_settings' + else: + instance = build_instance(feedcls) + method_name = '__new__' + if instance is None: + raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name)) + return instance def _get_uri_params(self, spider, uri_params, slot=None): params = {} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index a83076c47..90a52b25b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output, output_format=None): +def feed_process_params_from_cli(settings, output, output_format=None, + overwrite_output=None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary @@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None): def check_valid_format(output_format): if output_format not in valid_output_formats: - raise UsageError("Unrecognized output format '%s', set one after a" - " colon using the -o option (i.e. -o :)" - " or as a file extension, from the supported list %s" % - (output_format, tuple(valid_output_formats))) + raise UsageError( + "Unrecognized output format '%s'. Set a supported one (%s) " + "after a colon at the end of the output URI (i.e. -o/-O " + ":) or as a file extension." % ( + output_format, + tuple(valid_output_formats), + ) + ) + + overwrite = False + if overwrite_output: + if output: + raise UsageError( + "Please use only one of -o/--output and -O/--overwrite-output" + ) + output = overwrite_output + overwrite = True if output_format: if len(output) == 1: check_valid_format(output_format) - warnings.warn('The -t command line option is deprecated in favor' - ' of specifying the output format within the -o' - ' option, please check the -o option docs for more details', - category=ScrapyDeprecationWarning, stacklevel=2) + message = ( + 'The -t command line option is deprecated in favor of ' + 'specifying the output format within the output URI. See the ' + 'documentation of the -o and -O options for more information.', + ) + warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} else: - raise UsageError('The -t command line option cannot be used if multiple' - ' output files are specified with the -o option') + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result = {} for element in output: @@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None): feed_uri = 'stdout:' check_valid_format(feed_format) result[feed_uri] = {'format': feed_format} + if overwrite: + result[feed_uri]['overwrite'] = True - # FEEDS setting should take precedence over the -o and -t CLI options + # FEEDS setting should take precedence over the matching CLI options result.update(settings.getdict('FEEDS')) return result diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index f07bdd748..19d56d6ec 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( *, path, file, host, port, - username, password, use_active_mode=False): + username, password, use_active_mode=False, overwrite=True): """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ @@ -32,4 +32,6 @@ def ftp_store_file( file.seek(0) dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) - ftp.storbinary('STOR %s' % filename, file) + command = 'STOR' if overwrite else 'APPE' + ftp.storbinary('%s %s' % (command, filename), file) + file.close() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 59f1b8371..1f2333264 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -198,7 +198,8 @@ def _getargspec_py23(func): def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = _getargspec_py23(func) + spec = inspect.getfullargspec(func) + func_args = spec.args + spec.kwonlyargs elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): diff --git a/tests/ftpserver.py b/tests/ftpserver.py new file mode 100644 index 000000000..6f0289e08 --- /dev/null +++ b/tests/ftpserver.py @@ -0,0 +1,24 @@ +from argparse import ArgumentParser + +from pyftpdlib.authorizers import DummyAuthorizer +from pyftpdlib.handlers import FTPHandler +from pyftpdlib.servers import FTPServer + + +def main(): + parser = ArgumentParser() + parser.add_argument('-d', '--directory') + args = parser.parse_args() + + authorizer = DummyAuthorizer() + full_permissions = 'elradfmwMT' + authorizer.add_anonymous(args.directory, perm=full_permissions) + handler = FTPHandler + handler.authorizer = authorizer + address = ('127.0.0.1', 2121) + server = FTPServer(address, handler) + server.serve_forever() + + +if __name__ == '__main__': + main() diff --git a/tests/mockserver.py b/tests/mockserver.py index 1f40473ba..48d7b8d37 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -3,7 +3,10 @@ import json import os import random import sys +from pathlib import Path +from shutil import rmtree from subprocess import Popen, PIPE +from tempfile import mkdtemp from urllib.parse import urlencode from OpenSSL import SSL @@ -256,6 +259,29 @@ class MockDNSServer: self.proc.communicate() +class MockFTPServer: + """Creates an FTP server on port 2121 with a default passwordless user + (anonymous) and a temporary root path that you can read from the + :attr:`path` attribute.""" + + def __enter__(self): + self.path = Path(mkdtemp()) + self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)], + stderr=PIPE, env=get_testenv()) + for line in self.proc.stderr: + if b'starting FTP server' in line: + break + return self + + def __exit__(self, exc_type, exc_value, traceback): + rmtree(str(self.path)) + self.proc.kill() + self.proc.communicate() + + def url(self, path): + return 'ftp://127.0.0.1:2121/' + path + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index b51177abb..fe1cbc997 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,6 +4,7 @@ dataclasses; python_version == '3.6' mitmproxy; python_version >= '3.7' mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' mitmproxy < 4; python_version < '3.6' +pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-azurepipelines diff --git a/tests/test_commands.py b/tests/test_commands.py index 109f006a8..f76f851e7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -74,6 +74,7 @@ class ProjectTest(unittest.TestCase): def kill_proc(): p.kill() + p.communicate() assert False, 'Command took too much time to complete' timer = Timer(15, kill_proc) @@ -569,6 +570,55 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + class BenchCommandTest(CommandTest): @@ -577,3 +627,79 @@ class BenchCommandTest(CommandTest): '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) + + +class CrawlCommandTest(CommandTest): + + def crawl(self, code, args=()): + fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) + with open(fname, 'w') as f: + f.write(code) + return self.proc('crawl', 'myspider', *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('It works!') + return [] +""" + log = self.get_log(spider_code) + self.assertIn("[myspider] DEBUG: It works!", log) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log) + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return [] +""" + args = ['-O', 'example.json'] + log = self.get_log(spider_code, args=args) + self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + return [] +""" + args = ['-o', 'example1.json', '-O', 'example2.json'] + log = self.get_log(spider_code, args=args) + self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2afc25a7a..850485b5e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import random import shutil import string import tempfile +import warnings from abc import ABC, abstractmethod from collections import defaultdict from io import BytesIO @@ -25,7 +26,7 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, @@ -46,7 +47,7 @@ from scrapy.utils.test import ( mock_google_cloud_storage, ) -from tests.mockserver import MockServer +from tests.mockserver import MockFTPServer, MockServer class FileFeedStorageTest(unittest.TestCase): @@ -75,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase): st = FileFeedStorage(path) verifyObject(IFeedStorage, st) + def _store(self, feed_options=None): + path = os.path.abspath(self.mktemp()) + storage = FileFeedStorage(path, feed_options=feed_options) + spider = scrapy.Spider("default") + file = storage.open(spider) + file.write(b"content") + storage.store(file) + return path + + def test_append(self): + path = self._store() + return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") + + def test_overwrite(self): + path = self._store({"overwrite": True}) + return self._assert_stores( + FileFeedStorage(path, feed_options={"overwrite": True}), + path + ) + @defer.inlineCallbacks - def _assert_stores(self, storage, path): + def _assert_stores(self, storage, path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") @@ -84,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase): self.assertTrue(os.path.exists(path)) try: with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") + self.assertEqual(fp.read(), expected_content) finally: os.unlink(path) @@ -99,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase): spider = TestSpider.from_crawler(crawler) return spider - def test_store(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - st = FTPFeedStorage(uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _store(self, uri, content, feed_options=None, settings=None): + crawler = get_crawler(settings_dict=settings or {}) + storage = FTPFeedStorage.from_crawler( + crawler, + uri, + feed_options=feed_options, + ) + verifyObject(IFeedStorage, storage) + spider = self.get_test_spider() + file = storage.open(spider) + file.write(content) + return storage.store(file) - def test_store_active_mode(self): - uri = os.environ.get('FEEDTEST_FTP_URI') - path = os.environ.get('FEEDTEST_FTP_PATH') - if not (uri and path): - raise unittest.SkipTest("No FTP server available for testing") - use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True} - crawler = get_crawler(settings_dict=use_active_mode) - st = FTPFeedStorage.from_crawler(crawler, uri) - verifyObject(IFeedStorage, st) - return self._assert_stores(st, path) + def _assert_stored(self, path, content): + self.assertTrue(path.exists()) + try: + with path.open('rb') as fp: + self.assertEqual(fp.read(), content) + finally: + os.unlink(str(path)) + + @defer.inlineCallbacks + def test_append(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options) + yield self._store(url, b"bar", feed_options=feed_options) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite(self): + with MockFTPServer() as ftp_server: + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo") + yield self._store(url, b"bar") + self._assert_stored(ftp_server.path / filename, b"bar") + + @defer.inlineCallbacks + def test_append_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + feed_options = {'overwrite': False} + yield self._store(url, b"foo", feed_options=feed_options, settings=settings) + yield self._store(url, b"bar", feed_options=feed_options, settings=settings) + self._assert_stored(ftp_server.path / filename, b"foobar") + + @defer.inlineCallbacks + def test_overwrite_active_mode(self): + with MockFTPServer() as ftp_server: + settings = {'FEED_STORAGE_FTP_ACTIVE': True} + filename = 'file' + url = ftp_server.url(filename) + yield self._store(url, b"foo", settings=settings) + yield self._store(url, b"bar", settings=settings) + self._assert_stored(ftp_server.path / filename, b"bar") def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted) + st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted, + {}) self.assertEqual(st.password, string.punctuation) - @defer.inlineCallbacks - def _assert_stores(self, storage, path): - spider = self.get_test_spider() - file = storage.open(spider) - file.write(b"content") - yield storage.store(file) - self.assertTrue(os.path.exists(path)) - try: - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") - # again, to check s3 objects are overwritten - yield storage.store(BytesIO(b"new content")) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"new content") - finally: - os.unlink(path) - class BlockingFeedStorageTest(unittest.TestCase): @@ -190,8 +236,10 @@ class S3FeedStorageTest(unittest.TestCase): 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler - storage = S3FeedStorage.from_crawler(crawler, - 's3://mybucket/export.csv') + storage = S3FeedStorage.from_crawler( + crawler, + 's3://mybucket/export.csv', + ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') # Instantiate directly @@ -254,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -269,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase): crawler = get_crawler(settings_dict=settings) storage = S3FeedStorage.from_crawler( crawler, - 's3://mybucket/export.csv' + 's3://mybucket/export.csv', ) self.assertEqual(storage.access_key, 'access_key') self.assertEqual(storage.secret_key, 'secret_key') @@ -370,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase): key.set_contents_from_file.call_args ) + def test_overwrite_default(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl' + ) + self.assertNotIn('S3 does not support appending to files', str(log)) + + def test_overwrite_false(self): + with LogCapture() as log: + S3FeedStorage( + 's3://mybucket/export.csv', + 'access_key', + 'secret_key', + 'custom-acl', + feed_options={'overwrite': False}, + ) + self.assertIn('S3 does not support appending to files', str(log)) + class GCSFeedStorageTest(unittest.TestCase): @@ -439,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase): yield storage.store(file) self.assertEqual(out.getvalue(), b"content") + def test_overwrite_default(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:') + self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log)) + + def test_overwrite_true(self): + with LogCapture() as log: + StdoutFeedStorage('stdout:', feed_options={'overwrite': True}) + self.assertIn('Standard output (stdout) storage does not support overwriting', str(log)) + class FromCrawlerMixin: init_with_crawler = False @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): cls.init_with_crawler = True return cls(*args, **kwargs) @@ -454,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin): class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): - pass + + @classmethod + def from_crawler(cls, crawler, *args, feed_options=None, **kwargs): + cls.init_with_crawler = True + return cls(*args, feed_options=feed_options, **kwargs) class DummyBlockingFeedStorage(BlockingFeedStorage): @@ -588,8 +671,8 @@ class FeedExportTest(FeedExportTestBase): FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - printf_escape(path_to_url(file_path)): feed - for file_path, feed in FEEDS.items() + printf_escape(path_to_url(file_path)): feed_options + for file_path, feed_options in FEEDS.items() } content = {} @@ -599,12 +682,12 @@ class FeedExportTest(FeedExportTestBase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - for file_path, feed in FEEDS.items(): + for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): continue with open(str(file_path), 'rb') as f: - content[feed['format']] = f.read() + content[feed_options['format']] = f.read() finally: for file_path in FEEDS.keys(): @@ -1542,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase): content = json.loads(content.decode('utf-8')) expected_batch, items = items[:batch_size], items[batch_size:] self.assertEqual(expected_batch, content) + + +class FeedExportInitTest(unittest.TestCase): + + def test_unsupported_storage(self): + settings = { + 'FEEDS': { + 'unsupported://uri': {}, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + def test_unsupported_format(self): + settings = { + 'FEEDS': { + 'file://path': { + 'format': 'unsupported_format', + }, + }, + } + crawler = get_crawler(settings_dict=settings) + with self.assertRaises(NotConfigured): + FeedExporter.from_crawler(crawler) + + +class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FileFeedStorageWithoutFeedOptions(FileFeedStorage): + + def __init__(self, uri): + super().__init__(uri) + + +class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class S3FeedStorageWithoutFeedOptions(S3FeedStorage): + + def __init__(self, uri, access_key, secret_key, acl): + super().__init__(uri, access_key, secret_key, acl) + + +class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + +class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): + + def __init__(self, uri, use_active_mode=False): + super().__init__(uri) + + +class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage): + + @classmethod + def from_crawler(cls, crawler, uri): + return super().from_crawler(crawler, uri) + + +class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): + """Make sure that any feed exporter created by users before the + introduction of the ``feed_options`` parameter continues to work as + expected, and simply issues a warning.""" + + maxDiff = None + + def test_init(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) + + def test_from_crawler(self): + settings_dict = { + 'FEED_URI': 'file:///tmp/foobar', + 'FEED_STORAGES': { + 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler' + }, + } + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple(str(item.message) for item in w + if item.category is ScrapyDeprecationWarning) + self.assertEqual( + messages, + ( + ( + "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " + "does not support the 'feed_options' keyword argument. Add a " + "'feed_options' parameter to its signature to remove " + "this warning. This parameter will become mandatory " + "in a future version of Scrapy." + ), + ) + ) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f3ef36127..ccc65c4fd 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli(settings, ['-:pickle']) ) + def test_feed_export_config_overwrite(self): + settings = Settings() + self.assertEqual( + {'output.json': {'format': 'json', 'overwrite': True}}, + feed_process_params_from_cli(settings, [], None, ['output.json']) + ) + + def test_output_and_overwrite_output(self): + with self.assertRaises(UsageError): + feed_process_params_from_cli( + Settings(), + ['output1.json'], + None, + ['output2.json'], + ) + def test_feed_complete_default_values_from_settings_empty(self): feed = {} settings = Settings({ diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3f93f509e..c298d0bd2 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass + def f3(a, b=None, *, c=None): + pass + class A: def __init__(self, a, b, c): pass @@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) + self.assertEqual(get_func_args(f3), ['a', 'b', 'c']) self.assertEqual(get_func_args(A), ['a', 'b', 'c']) self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) self.assertEqual(get_func_args(partial_f1), ['b', 'c']) From d9e69bfb51d5ed5a08d57878b29a6e7db8ef9d15 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Aug 2020 19:46:24 +0500 Subject: [PATCH 088/119] Re-enable TLS 1.2 in cipher tests. --- tests/mockserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 48d7b8d37..6f0c274b9 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -289,8 +289,8 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c ) if cipher_string: ctx = factory.getContext() - # disabling TLS1.2+ because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3) + # disabling TLS1.3 because it unconditionally enables some strong ciphers + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory From 42383cc267393c3c5fb89c9108759125939ab3e5 Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Wed, 19 Aug 2020 12:48:14 +0530 Subject: [PATCH 089/119] Add a setting to customize the asyncio event loop (#4414) --- docs/topics/asyncio.rst | 12 +++++++++++ docs/topics/settings.rst | 20 ++++++++++++++++++ scrapy/crawler.py | 2 +- scrapy/settings/default_settings.py | 2 ++ scrapy/utils/log.py | 7 +++++++ scrapy/utils/reactor.py | 12 ++++++++--- tests/CrawlerProcess/asyncio_custom_loop.py | 17 +++++++++++++++ tests/requirements-py3.txt | 1 + tests/test_commands.py | 23 +++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 10 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_custom_loop.py diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 038a459fd..bfb430d52 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -26,3 +26,15 @@ reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`:: install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to +use it instead of the default asyncio event loop. + + + diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 722ae4593..618b9989e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -216,6 +216,26 @@ Default: ``None`` The name of the region associated with the AWS client. +.. setting:: ASYNCIO_EVENT_LOOP + +ASYNCIO_EVENT_LOOP +------------------ + +Default: ``None`` + +Import path of a given asyncio event loop class. + +If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the +asyncio event loop to be used with it. Set the setting to the import path of the +desired asyncio event loop class. If the setting is set to ``None`` the default asyncio +event loop will be used. + +If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor` +function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop +class to be used. + +Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`. + .. setting:: BOT_NAME BOT_NAME diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d028bea4d..4c6b0e496 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -340,5 +340,5 @@ class CrawlerProcess(CrawlerRunner): def _handle_twisted_reactor(self): if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"]) + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) super()._handle_twisted_reactor() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 896afa995..a0251394b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,6 +19,8 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False +ASYNCIO_EVENT_LOOP = None + AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 1d6a2c39d..e41315738 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -150,6 +150,13 @@ def log_scrapy_info(settings): logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) + from twisted.internet import asyncioreactor + if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): + logger.debug( + "Using asyncio event loop: %s.%s", + reactor._asyncioEventloop.__module__, + reactor._asyncioEventloop.__class__.__name__, + ) class StreamLogger: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 3c705f69b..879d27907 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -50,13 +50,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def install_reactor(reactor_path): +def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified - import path.""" + import path. Also installs the asyncio event loop with the specified import + path if the asyncio reactor is enabled""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + else: + event_loop = asyncio.new_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") installer_path = module + ["install"] diff --git a/tests/CrawlerProcess/asyncio_custom_loop.py b/tests/CrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..1e4ada722 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop" +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index fe1cbc997..44ddcded8 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -13,6 +13,7 @@ pytest-twisted >= 1.11 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures +uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython diff --git a/tests/test_commands.py b/tests/test_commands.py index f76f851e7..ee8a92604 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -16,6 +16,7 @@ from tempfile import mkdtemp from threading import Timer from unittest import skipIf +from pytest import mark from twisted.trial import unittest import scrapy @@ -570,6 +571,28 @@ class BadSpider(scrapy.Spider): log = self.get_log(self.debug_log_spider, args=[]) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', + 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor', + '-s', + 'ASYNCIO_EVENT_LOOP=uvloop.Loop', + ]) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + import asyncio + loop = asyncio.new_event_loop() + self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log) + def test_output(self): spider_code = """ import scrapy diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 1a4cfe813..7c2e251a9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + def test_custom_loop_asyncio(self): + log = self.run_script("asyncio_custom_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') From a57db9e3024fe1426021b8b692a48f4c8db82a77 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 19 Aug 2020 18:45:24 +0300 Subject: [PATCH 090/119] Bitbucket no longer supports Mercurial repositories (#4738) --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index db720b918..33a920bb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ install: - | if [[ ! -z "$PYPY_VERSION" ]]; then export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2" + wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" diff --git a/setup.py b/setup.py index d0880051f..52a27c368 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ if has_environment_marker_platform_impl_support(): ] extras_require[':platform_python_implementation == "PyPy"'] = [ # Earlier lxml versions are affected by - # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no, + # https://foss.heptapod.net/pypy/pypy/-/issues/2498, # which was fixed in Cython 0.26, released on 2017-06-19, and used to # generate the C headers of lxml release tarballs published since then, the # first of which was: From d68aab992e435aa1c88061e83d03e57e7d31533d Mon Sep 17 00:00:00 2001 From: Grisha Temchenko Date: Thu, 20 Aug 2020 09:22:07 -0400 Subject: [PATCH 091/119] Smarter generator check for combined return/yield statements (#4721) --- scrapy/utils/misc.py | 19 ++++++++++++++++++- ...t_return_with_argument_inside_generator.py | 19 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d6966be8e..bd400bd30 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -5,6 +5,7 @@ import os import re import hashlib import warnings +from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -184,6 +185,22 @@ def set_environ(**kwargs): os.environ[k] = v +def walk_callable(node): + """Similar to ``ast.walk``, but walks only function body and skips nested + functions defined within the node. + """ + todo = deque([node]) + walked_func_def = False + while todo: + node = todo.popleft() + if isinstance(node, ast.FunctionDef): + if walked_func_def: + continue + walked_func_def = True + todo.extend(ast.iter_child_nodes(node)) + yield node + + _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) @@ -201,7 +218,7 @@ def is_generator_with_return_value(callable): if inspect.isgeneratorfunction(callable): tree = ast.parse(dedent(inspect.getsource(callable))) - for node in ast.walk(tree): + for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True return _generator_callbacks_cache[callable] diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index bdbec1beb..2be38620c 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase): yield 1 yield from g() + def m(): + yield 1 + + def helper(): + return 0 + + yield helper() + + def n(): + yield 1 + + def helper(): + return 0 + + yield helper() + return 2 + assert is_generator_with_return_value(f) assert is_generator_with_return_value(g) assert not is_generator_with_return_value(h) assert not is_generator_with_return_value(i) assert not is_generator_with_return_value(j) assert not is_generator_with_return_value(k) # not recursive + assert not is_generator_with_return_value(m) + assert is_generator_with_return_value(n) From 2fbfe2c21411480c35a99ff5497ee59f2e5e0dbb Mon Sep 17 00:00:00 2001 From: yogendra0sharma Date: Fri, 21 Aug 2020 12:18:15 +0530 Subject: [PATCH 092/119] Removed appveyor.xml no longer needed --- appveyor.yml | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 7fd636864..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,25 +0,0 @@ -platform: x86 -version: '{branch}-{build}' -environment: - matrix: - - PYTHON: "C:\\Python36" - TOX_ENV: py36 - -branches: - only: - - master - - /d+\.\d+\.\d+[\w\-]*$/ - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" - - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - - "pip install -U tox" - -build: false -skip_tags: true -test_script: - - "tox -e %TOX_ENV%" - -cache: - - '%LOCALAPPDATA%\pip\cache' From 7c076122ebb16a6db9b85b68c85c354c2a49fd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 21 Aug 2020 17:06:54 +0200 Subject: [PATCH 093/119] Skip checks introduced in Pylint 2.6.0 --- pylintrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylintrc b/pylintrc index 129c7bf7d..5b6b9fab0 100644 --- a/pylintrc +++ b/pylintrc @@ -68,6 +68,7 @@ disable=abstract-method, pointless-statement, pointless-string-statement, protected-access, + raise-missing-from, redefined-argument-from-local, redefined-builtin, redefined-outer-name, @@ -75,6 +76,7 @@ disable=abstract-method, signature-differs, singleton-comparison, super-init-not-called, + super-with-arguments, superfluous-parens, too-few-public-methods, too-many-ancestors, From f1250177dc1c486517b9ca8ed136162533014da7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Sat, 22 Aug 2020 04:33:35 -0300 Subject: [PATCH 094/119] Remove Python 3.5 from CI (#4743) --- .travis.yml | 12 +++--------- azure-pipelines.yml | 4 +--- tests/test_utils_python.py | 18 +++++++++--------- tox.ini | 4 ++-- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 33a920bb6..b883c5b78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,16 +19,10 @@ matrix: python: 3.8 - env: TOXENV=pinned - python: 3.5.2 + python: 3.6.1 - env: TOXENV=asyncio-pinned - python: 3.5.2 # We use additional code to support 3.5.3 and earlier - - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 - - - env: TOXENV=py - python: 3.5 - - env: TOXENV=asyncio - python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 - - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + python: 3.6.1 + - env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0 - env: TOXENV=py python: 3.6 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 710e42090..c03e258c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,11 +4,9 @@ pool: vmImage: 'windows-latest' strategy: matrix: - Python35: - python.version: '3.5' - TOXENV: windows-pinned Python36: python.version: '3.6' + TOXENV: windows-pinned Python37: python.version: '3.7' Python38: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c298d0bd2..3115cc92f 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,8 +3,8 @@ import gc import operator import platform import unittest +from datetime import datetime from itertools import count -from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( @@ -216,15 +216,15 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) - else: - self.assertEqual( - get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual( - get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - if version_info < (3, 6): - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) - else: + elif platform.python_implementation() == 'PyPy': + self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj']) + + build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y') + if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1 self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) + else: + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tox.ini b/tox.ini index 4f5531aea..dec0d75e8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ deps = # Extras boto3>=1.13.0 botocore>=1.4.87 - Pillow>=3.4.2 + Pillow>=4.0.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -78,7 +78,7 @@ deps = # Extras botocore==1.4.87 google-cloud-storage==1.29.0 - Pillow==3.4.2 + Pillow==4.0.0 [testenv:pinned] deps = From 2d8ec9d44fa2201986da98c3f5f7c3c7f1ecfb56 Mon Sep 17 00:00:00 2001 From: WinterComes Date: Wed, 17 Jul 2019 22:50:34 +0300 Subject: [PATCH 095/119] Change DOWNLOAD_MAXSIZE logger level from Error to Warning --- scrapy/core/downloader/handlers/http11.py | 21 +++++++++++---------- tests/test_downloader_handlers.py | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index fb04d1fb7..a78b19318 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -394,13 +394,14 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - error_msg = ("Cancelling download of %(url)s: expected response " - "size (%(size)s) larger than download max size (%(maxsize)s).") - error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Expected response size (%(size)s) larger than " + "download max size (%(maxsize)s) in request %(request)s.") + warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize} + + logger.warning(warning_msg, warning_args) - logger.error(error_msg, error_args) txresponse._transport._producer.loseConnection() - raise defer.CancelledError(error_msg % error_args) + raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " @@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: - logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", - {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + logger.warning("Received (%(bytes)s) bytes larger than download " + "max size (%(maxsize)s) in request %(request)s.", + {'bytes': self._bytes_received, + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 13063d106..7059f0892 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('largechunkedfile')) def check(logger): - logger.error.assert_called_once_with(mock.ANY, mock.ANY) + logger.warning.assert_called_once_with(mock.ANY, mock.ANY) d = self.download_request(request, Spider('foo', download_maxsize=1500)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) From 0b3881d65e12e7e0ac5f70c0f33e45dbab546c5d Mon Sep 17 00:00:00 2001 From: drs-11 Date: Mon, 24 Aug 2020 20:26:06 +0530 Subject: [PATCH 096/119] Reverted maxsize warning log message --- scrapy/core/downloader/handlers/http11.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a78b19318..25e800984 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -394,9 +394,9 @@ class ScrapyAgent: fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: - warning_msg = ("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s) in request %(request)s.") - warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize} + warning_msg = ("Cancelling download of %(url)s: expected response " + "size (%(size)s) larger than download max size (%(maxsize)s).") + warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize} logger.warning(warning_msg, warning_args) From 3e726b9df721f2288f4ae9a685de9bdff0762c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 11:22:05 +0100 Subject: [PATCH 097/119] Support for delegated methods as callbacks It can be useful to structure the spiders code around some helper classes. --- scrapy/utils/reqser.py | 29 ++++++++++++++--------------- tests/test_utils_reqser.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 5ea2aafb8..35a4fc72c 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -73,23 +73,22 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): if obj: try: - func_self = func.__self__ - except AttributeError: # func has no __self__ + func.__func__ + except AttributeError: # func is not a instance method. Not supported. pass else: - if func_self is obj: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError("Function %s is not a method of: %s" % (func, obj)) + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) def _get_method(obj, name): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index de94ec960..c8d1db138 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -102,6 +102,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_delegated_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.delegated_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -131,6 +137,9 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass +class TestSpiderDelegation: + def delegated_callback(self, response): + pass def parse_item(response): pass @@ -155,6 +164,9 @@ class TestSpider(Spider, TestSpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error + def __init__(self): + self.delegated_callback = TestSpiderDelegation().delegated_callback + def parse_item(self, response): pass From a2d6fa5adc17035cebb8dad4a3bd2020236b4f71 Mon Sep 17 00:00:00 2001 From: maranqz Date: Tue, 25 Aug 2020 13:34:43 +0300 Subject: [PATCH 098/119] Add errors parameter for CsvItemExporter with tests --- scrapy/exporters.py | 5 +++-- tests/test_exporters.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..8cd2077b6 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): - def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs): + def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = 'utf-8' @@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + errors=errors, ) self.csv_writer = csv.writer(self.stream, **self._kwargs) self._headers_not_written = True diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6c25a0064..ebc477e74 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest): expected='22,False,3.14,2015-01-01 01:01:01\r\n' ) + def test_errors_default(self): + with self.assertRaises(UnicodeEncodeError): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + expected=None, + encoding='windows-1251', + ) + + def test_errors_xmlcharrefreplace(self): + self.assertExportResult( + item=dict(text=u'W\u0275\u200Brd'), + include_headers_line=False, + expected='Wɵ​rd\r\n', + encoding='windows-1251', + errors='xmlcharrefreplace', + ) + class CsvItemExporterDataclassTest(CsvItemExporterTest): item_class = TestDataClass From 39affea93c5b60cde89e000486c39c3c0ce87dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 13:57:48 +0200 Subject: [PATCH 099/119] Fix style issues --- tests/test_utils_reqser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index c8d1db138..ee68cf6b1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -137,10 +137,12 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass + class TestSpiderDelegation: def delegated_callback(self, response): pass + def parse_item(response): pass From 0524df866936506ae9438a5565fc4733fa5ba5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 14:36:38 +0100 Subject: [PATCH 100/119] Code simplification. Thanks @victor-torres for the suggestion --- scrapy/utils/reqser.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 35a4fc72c..503d7b133 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,23 +71,19 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): - if obj: - try: - func.__func__ - except AttributeError: # func is not a instance method. Not supported. - pass - else: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) From 58ca8bbf6d1589bd0c8cc1ebda52299346f55e8a Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 22 Aug 2020 22:32:03 +0200 Subject: [PATCH 101/119] Use f-strings (#4307) --- docs/conf.py | 2 +- docs/intro/tutorial.rst | 6 +- docs/topics/developer-tools.rst | 138 +++++++++--------- docs/topics/exporters.rst | 6 +- docs/topics/item-pipeline.rst | 6 +- docs/topics/leaks.rst | 2 +- docs/topics/selectors.rst | 7 +- docs/topics/settings.rst | 2 +- docs/topics/spiders.rst | 4 +- extras/qps-bench-server.py | 2 +- extras/qpsclient.py | 4 +- scrapy/cmdline.py | 16 +- scrapy/commands/__init__.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 10 +- scrapy/commands/edit.py | 4 +- scrapy/commands/genspider.py | 27 ++-- scrapy/commands/parse.py | 4 +- scrapy/commands/runspider.py | 8 +- scrapy/commands/startproject.py | 12 +- scrapy/commands/version.py | 4 +- scrapy/contracts/__init__.py | 8 +- scrapy/contracts/default.py | 12 +- scrapy/core/downloader/__init__.py | 16 +- scrapy/core/downloader/handlers/__init__.py | 3 +- scrapy/core/downloader/handlers/http11.py | 17 ++- scrapy/core/downloader/handlers/s3.py | 6 +- scrapy/core/downloader/middleware.py | 15 +- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/engine.py | 12 +- scrapy/core/scraper.py | 4 +- scrapy/core/spidermw.py | 20 +-- scrapy/downloadermiddlewares/cookies.py | 12 +- scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 6 +- scrapy/downloadermiddlewares/stats.py | 6 +- scrapy/exporters.py | 2 +- scrapy/extensions/corestats.py | 2 +- scrapy/extensions/debug.py | 2 +- scrapy/extensions/httpcache.py | 10 +- scrapy/extensions/memdebug.py | 2 +- scrapy/extensions/memusage.py | 14 +- scrapy/extensions/statsmailer.py | 10 +- scrapy/http/common.py | 2 +- scrapy/http/headers.py | 2 +- scrapy/http/request/__init__.py | 12 +- scrapy/http/request/form.py | 19 ++- scrapy/http/response/__init__.py | 6 +- scrapy/http/response/text.py | 15 +- scrapy/item.py | 6 +- scrapy/link.py | 6 +- scrapy/logformatter.py | 4 +- scrapy/pipelines/files.py | 23 ++- scrapy/pipelines/images.py | 9 +- scrapy/pipelines/media.py | 7 +- scrapy/pqueues.py | 9 +- scrapy/responsetypes.py | 2 +- scrapy/selector/unified.py | 4 +- scrapy/settings/__init__.py | 2 +- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 4 +- scrapy/spiderloader.py | 13 +- scrapy/spidermiddlewares/depth.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/offsite.py | 6 +- scrapy/spidermiddlewares/referer.py | 2 +- scrapy/spiders/__init__.py | 11 +- scrapy/spiders/feed.py | 4 +- scrapy/utils/benchserver.py | 6 +- scrapy/utils/conf.py | 22 +-- scrapy/utils/curl.py | 4 +- scrapy/utils/decorators.py | 4 +- scrapy/utils/deprecate.py | 11 +- scrapy/utils/engine.py | 4 +- scrapy/utils/ftp.py | 2 +- scrapy/utils/iterators.py | 13 +- scrapy/utils/log.py | 4 +- scrapy/utils/misc.py | 15 +- scrapy/utils/project.py | 4 +- scrapy/utils/python.py | 12 +- scrapy/utils/reactor.py | 8 +- scrapy/utils/reqser.py | 4 +- scrapy/utils/response.py | 10 +- scrapy/utils/serialize.py | 6 +- scrapy/utils/ssl.py | 4 +- scrapy/utils/test.py | 2 +- scrapy/utils/testproc.py | 6 +- scrapy/utils/testsite.py | 4 +- scrapy/utils/trackref.py | 4 +- scrapy/utils/url.py | 4 +- sep/sep-002.rst | 2 +- sep/sep-004.rst | 4 +- sep/sep-014.rst | 29 ++-- sep/sep-018.rst | 22 +-- tests/CrawlerRunner/ip_address.py | 2 +- tests/mockserver.py | 10 +- tests/py36/_test_crawl.py | 2 +- tests/spiders.py | 8 +- tests/test_cmdline/extensions.py | 2 +- tests/test_command_check.py | 10 +- tests/test_command_parse.py | 16 +- tests/test_command_shell.py | 16 +- tests/test_command_version.py | 2 +- tests/test_commands.py | 32 ++-- tests/test_contracts.py | 2 +- tests/test_crawl.py | 6 +- tests/test_downloader_handlers.py | 22 +-- tests/test_downloadermiddleware.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 10 +- tests/test_downloadermiddleware_redirect.py | 10 +- tests/test_downloadermiddleware_retry.py | 2 +- tests/test_engine.py | 23 +-- tests/test_feedexport.py | 13 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_crawl.py | 4 +- tests/test_pipeline_files.py | 4 +- tests/test_pipeline_images.py | 4 +- tests/test_proxy_connect.py | 6 +- tests/test_request_attribute_binding.py | 4 +- tests/test_responsetypes.py | 12 +- tests/test_selector.py | 2 +- tests/test_signals.py | 2 +- tests/test_spidermiddleware_output_chain.py | 24 +-- tests/test_utils_curl.py | 2 +- tests/test_utils_datatypes.py | 2 +- tests/test_utils_defer.py | 8 +- tests/test_utils_iterators.py | 2 +- tests/test_utils_url.py | 6 +- tests/test_webclient.py | 8 +- 133 files changed, 561 insertions(+), 568 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 427c79481..27d2b5dff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -49,7 +49,7 @@ master_doc = 'index' # General information about the project. project = 'Scrapy' -copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) +copyright = f'2008–{datetime.now().year}, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f96c78887..914b91022 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -101,10 +101,10 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) - self.log('Saved file %s' % filename) + self.log(f'Saved file {filename}') As you can see, our Spider subclasses :class:`scrapy.Spider ` @@ -190,7 +190,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] - filename = 'quotes-%s.html' % page + filename = f'quotes-{page}.html' with open(filename, 'wb') as f: f.write(response.body) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 101aa159c..c83b1a9d9 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -5,9 +5,9 @@ Using your browser's Developer Tools for scraping ================================================= Here is a general guide on how to use your browser's Developer Tools -to ease the scraping process. Today almost all browsers come with +to ease the scraping process. Today almost all browsers come with built in `Developer Tools`_ and although we will use Firefox in this -guide, the concepts are applicable to any other browser. +guide, the concepts are applicable to any other browser. In this guide we'll introduce the basic tools to use from a browser's Developer Tools by scraping `quotes.toscrape.com`_. @@ -41,16 +41,16 @@ Therefore, you should keep in mind the following things: Inspecting a website ==================== -By far the most handy feature of the Developer Tools is the `Inspector` -feature, which allows you to inspect the underlying HTML code of -any webpage. To demonstrate the Inspector, let's look at the +By far the most handy feature of the Developer Tools is the `Inspector` +feature, which allows you to inspect the underlying HTML code of +any webpage. To demonstrate the Inspector, let's look at the `quotes.toscrape.com`_-site. On the site we have a total of ten quotes from various authors with specific -tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes -on this page, without any meta-information about authors, tags, etc. +tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes +on this page, without any meta-information about authors, tags, etc. -Instead of viewing the whole source code for the page, we can simply right click +Instead of viewing the whole source code for the page, we can simply right click on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`. In it you should see something like this: @@ -97,16 +97,16 @@ Then, back to your web browser, right-click on the ``span`` tag, select >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] -Adding ``text()`` at the end we are able to extract the first quote with this +Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is -go down a desired path in the source code starting from ``html``. So let's -see if we can refine our XPath a bit: +go down a desired path in the source code starting from ``html``. So let's +see if we can refine our XPath a bit: -If we check the `Inspector` again we'll see that directly beneath our -expanded ``div`` tag we have nine identical ``div`` tags, each with the -same attributes as our first. If we expand any of them, we'll see the same +If we check the `Inspector` again we'll see that directly beneath our +expanded ``div`` tag we have nine identical ``div`` tags, each with the +same attributes as our first. If we expand any of them, we'll see the same structure as with our first quote: Two ``span`` tags and one ``div`` tag. We can -expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and +expand each ``span`` tag with the ``class="text"`` inside our ``div`` tags and see each quote: .. code-block:: html @@ -121,7 +121,7 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, -we'll simply select all ``span`` tags with the ``class="text"`` by using +we'll simply select all ``span`` tags with the ``class="text"`` by using the `has-class-extension`_: >>> response.xpath('//span[has-class("text")]/text()').getall() @@ -130,45 +130,45 @@ the `has-class-extension`_: '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', ...] -And with one simple, cleverer XPath we are able to extract all quotes from -the page. We could have constructed a loop over our first XPath to increase -the number of the last ``div``, but this would have been unnecessarily +And with one simple, cleverer XPath we are able to extract all quotes from +the page. We could have constructed a loop over our first XPath to increase +the number of the last ``div``, but this would have been unnecessarily complex and by simply constructing an XPath with ``has-class("text")`` -we were able to extract all quotes in one line. +we were able to extract all quotes in one line. -The `Inspector` has a lot of other helpful features, such as searching in the +The `Inspector` has a lot of other helpful features, such as searching in the source code or directly scrolling to an element you selected. Let's demonstrate -a use case: +a use case: -Say you want to find the ``Next`` button on the page. Type ``Next`` into the -search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="next"``, the second the text +Say you want to find the ``Next`` button on the page. Type ``Next`` into the +search bar on the top right of the `Inspector`. You should get two results. +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here -we could easily create a :ref:`Link Extractor ` to -follow the pagination. On a simple site such as this, there may not be +we could easily create a :ref:`Link Extractor ` to +follow the pagination. On a simple site such as this, there may not be the need to find an element visually but the ``Scroll into View`` function -can be quite useful on complex sites. +can be quite useful on complex sites. Note that the search bar can also be used to search for and test CSS -selectors. For example, you could search for ``span.text`` to find -all quote texts. Instead of a full text search, this searches for -exactly the ``span`` tag with the ``class="text"`` in the page. +selectors. For example, you could search for ``span.text`` to find +all quote texts. Instead of a full text search, this searches for +exactly the ``span`` tag with the ``class="text"`` in the page. .. _topics-network-tool: The Network-tool ================ While scraping you may come across dynamic webpages where some parts -of the page are loaded dynamically through multiple requests. While -this can be quite tricky, the `Network`-tool in the Developer Tools +of the page are loaded dynamically through multiple requests. While +this can be quite tricky, the `Network`-tool in the Developer Tools greatly facilitates this task. To demonstrate the Network-tool, let's -take a look at the page `quotes.toscrape.com/scroll`_. +take a look at the page `quotes.toscrape.com/scroll`_. -The page is quite similar to the basic `quotes.toscrape.com`_-page, -but instead of the above-mentioned ``Next`` button, the page -automatically loads new quotes when you scroll to the bottom. We -could go ahead and try out different XPaths directly, but instead +The page is quite similar to the basic `quotes.toscrape.com`_-page, +but instead of the above-mentioned ``Next`` button, the page +automatically loads new quotes when you scroll to the bottom. We +could go ahead and try out different XPaths directly, but instead we'll check another quite useful command from the Scrapy shell: .. skip: next @@ -179,9 +179,9 @@ we'll check another quite useful command from the Scrapy shell: (...) >>> view(response) -A browser window should open with the webpage but with one -crucial difference: Instead of the quotes we just see a greenish -bar with the word ``Loading...``. +A browser window should open with the webpage but with one +crucial difference: Instead of the quotes we just see a greenish +bar with the word ``Loading...``. .. image:: _images/network_01.png :width: 777 @@ -189,21 +189,21 @@ bar with the word ``Loading...``. :alt: Response from quotes.toscrape.com/scroll The ``view(response)`` command let's us view the response our -shell or later our spider receives from the server. Here we see -that some basic template is loaded which includes the title, +shell or later our spider receives from the server. Here we see +that some basic template is loaded which includes the title, the login-button and the footer, but the quotes are missing. This tells us that the quotes are being loaded from a different request -than ``quotes.toscrape/scroll``. +than ``quotes.toscrape/scroll``. -If you click on the ``Network`` tab, you will probably only see -two entries. The first thing we do is enable persistent logs by -clicking on ``Persist Logs``. If this option is disabled, the +If you click on the ``Network`` tab, you will probably only see +two entries. The first thing we do is enable persistent logs by +clicking on ``Persist Logs``. If this option is disabled, the log is automatically cleared each time you navigate to a different -page. Enabling this option is a good default, since it gives us -control on when to clear the logs. +page. Enabling this option is a good default, since it gives us +control on when to clear the logs. If we reload the page now, you'll see the log get populated with six -new requests. +new requests. .. image:: _images/network_02.png :width: 777 @@ -212,31 +212,31 @@ new requests. Here we see every request that has been made when reloading the page and can inspect each request and its response. So let's find out -where our quotes are coming from: +where our quotes are coming from: -First click on the request with the name ``scroll``. On the right +First click on the request with the name ``scroll``. On the right you can now inspect the request. In ``Headers`` you'll find details about the request headers, such as the URL, the method, the IP-address, and so on. We'll ignore the other tabs and click directly on ``Response``. -What you should see in the ``Preview`` pane is the rendered HTML-code, -that is exactly what we saw when we called ``view(response)`` in the -shell. Accordingly the ``type`` of the request in the log is ``html``. -The other requests have types like ``css`` or ``js``, but what -interests us is the one request called ``quotes?page=1`` with the -type ``json``. +What you should see in the ``Preview`` pane is the rendered HTML-code, +that is exactly what we saw when we called ``view(response)`` in the +shell. Accordingly the ``type`` of the request in the log is ``html``. +The other requests have types like ``css`` or ``js``, but what +interests us is the one request called ``quotes?page=1`` with the +type ``json``. -If we click on this request, we see that the request URL is +If we click on this request, we see that the request URL is ``http://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click -on the request and open ``Open in new tab`` to get a better overview. +on the request and open ``Open in new tab`` to get a better overview. .. image:: _images/network_03.png :width: 777 :height: 375 :alt: JSON-object returned from the quotes.toscrape API -With this response we can now easily parse the JSON-object and +With this response we can now easily parse the JSON-object and also request each page to get every quote on the site:: import scrapy @@ -255,17 +255,17 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page) + url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) -This spider starts at the first page of the quotes-API. With each -response, we parse the ``response.text`` and assign it to ``data``. -This lets us operate on the JSON-object like on a Python dictionary. +This spider starts at the first page of the quotes-API. With each +response, we parse the ``response.text`` and assign it to ``data``. +This lets us operate on the JSON-object like on a Python dictionary. We iterate through the ``quotes`` and print out the ``quote["text"]``. -If the handy ``has_next`` element is ``true`` (try loading +If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a -page-number greater than 10), we increment the ``page`` attribute -and ``yield`` a new request, inserting the incremented page-number +page-number greater than 10), we increment the ``page`` attribute +and ``yield`` a new request, inserting the incremented page-number into our ``url``. .. _requests-from-curl: @@ -298,7 +298,7 @@ Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. As you can see, with a few inspections in the `Network`-tool we -were able to easily replicate the dynamic requests of the scrolling +were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..793799a9a 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -57,7 +57,7 @@ value of one of their fields:: adapter = ItemAdapter(item) year = adapter['year'] if year not in self.year_to_exporter: - f = open('{}.xml'.format(year), 'wb') + f = open(f'{year}.xml', 'wb') exporter = XmlItemExporter(f) exporter.start_exporting() self.year_to_exporter[year] = exporter @@ -98,7 +98,7 @@ Example:: import scrapy def serialize_price(value): - return '$ %s' % str(value) + return f'$ {str(value)}' class Product(scrapy.Item): name = scrapy.Field() @@ -122,7 +122,7 @@ Example:: def serialize_field(self, field, name, value): if field == 'price': - return '$ %s' % str(value) + return f'$ {str(value)}' return super(Product, self).serialize_field(field, name, value) .. _topics-exporters-reference: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index cd6a6d47e..6287ee0ad 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -96,7 +96,7 @@ contain a price:: adapter['price'] = adapter['price'] * self.vat_factor return item else: - raise DropItem("Missing price in %s" % item) + raise DropItem(f"Missing price in {item}") Write items to a JSON file @@ -211,7 +211,7 @@ item. # Save screenshot to file, filename will be hash of url. url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() - filename = "{}.png".format(url_hash) + filename = f"{url_hash}.png" with open(filename, "wb") as f: f.write(response.body) @@ -240,7 +240,7 @@ returns multiples items with the same id:: def process_item(self, item, spider): adapter = ItemAdapter(item) if adapter['id'] in self.ids_seen: - raise DropItem("Duplicate item found: %r" % item) + raise DropItem(f"Duplicate item found: {item!r}") else: self.ids_seen.add(adapter['id']) return item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index d2f7edf0a..b895b95cb 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -102,7 +102,7 @@ A real example Let's see a concrete example of a hypothetical case of memory leaks. Suppose we have some spider with a line similar to this one:: - return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id, + return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}", callback=self.parse, cb_kwargs={'referer': response}) That line is passing a response reference inside a request which effectively diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9e2c6ba42..b576fde91 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -328,8 +328,9 @@ too. Here's an example: 'Name: My image 5
    '] >>> for index, link in enumerate(links): -... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) -... print('Link number %d points to url %r and image %r' % args) +... href_xpath = link.xpath('@href').get() +... img_xpath = link.xpath('img/@src').get() +... print(f'Link number {index} points to url {href_xpath!r} and image {img_xpath!r}') Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' @@ -822,7 +823,7 @@ with groups of itemscopes and corresponding itemprops:: ... props = scope.xpath(''' ... set:difference(./descendant::*/@itemprop, ... .//*[@itemscope]/*/@itemprop)''') - ... print(" properties: %s" % (props.getall())) + ... print(f" properties: {props.getall()}") ... print("") current scope: ['http://schema.org/Product'] diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 618b9989e..22d60f87c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -110,7 +110,7 @@ In a spider, the settings are available through ``self.settings``:: start_urls = ['http://example.com'] def parse(self, response): - print("Existing settings: %s" % self.settings.attributes.keys()) + print(f"Existing settings: {self.settings.attributes.keys()}") .. note:: The ``settings`` attribute is set in the base Spider class after the spider diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e50e4aa0a..2056664c7 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -279,7 +279,7 @@ Spiders can access arguments in their `__init__` methods:: def __init__(self, category=None, *args, **kwargs): super(MySpider, self).__init__(*args, **kwargs) - self.start_urls = ['http://www.example.com/categories/%s' % category] + self.start_urls = [f'http://www.example.com/categories/{category}'] # ... The default `__init__` method will take any spider arguments @@ -292,7 +292,7 @@ The above example can also be written as follows:: name = 'myspider' def start_requests(self): - yield scrapy.Request('http://www.example.com/categories/%s' % self.category) + yield scrapy.Request(f'http://www.example.com/categories/{self.category}') Keep in mind that spider arguments are only strings. The spider will not do any parsing on its own. diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index da7a0022b..a6472b1ba 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -37,7 +37,7 @@ class Root(Resource): if now - self.lastmark >= 3: self.lastmark = now qps = len(self.tail) / sum(self.tail) - print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps)) + print(f'samplesize={len(self.tail)} concurrent={self.concurrent} qps={qps:0.2f}') if 'latency' in request.args: latency = float(request.args['latency'][0]) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index fe1f96cbb..f9fb70342 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -37,11 +37,11 @@ class QPSSpider(Spider): def start_requests(self): url = self.benchurl if self.latency is not None: - url += '?latency={0}'.format(self.latency) + url += f'?latency={self.latency}' slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] + urls = [url.replace('localhost', f'127.0.0.{x + 1}') for x in range(slots)] else: urls = [url] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 3e88536e4..91482ce01 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -44,7 +44,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'): if inspect.isclass(obj): cmds[entry_point.name] = obj() else: - raise Exception("Invalid entry point %s" % entry_point.name) + raise Exception(f"Invalid entry point {entry_point.name}") return cmds @@ -67,11 +67,11 @@ def _pop_command_name(argv): def _print_header(settings, inproject): + version = scrapy.__version__ if inproject: - print("Scrapy %s - project: %s\n" % (scrapy.__version__, - settings['BOT_NAME'])) + print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") else: - print("Scrapy %s - no active project\n" % scrapy.__version__) + print(f"Scrapy {version} - no active project\n") def _print_commands(settings, inproject): @@ -81,7 +81,7 @@ def _print_commands(settings, inproject): print("Available commands:") cmds = _get_commands_dict(settings, inproject) for cmdname, cmdclass in sorted(cmds.items()): - print(" %-13s %s" % (cmdname, cmdclass.short_desc())) + print(f" {cmdname:<13} {cmdclass.short_desc()}") if not inproject: print() print(" [ more ] More commands available when run from project directory") @@ -91,7 +91,7 @@ def _print_commands(settings, inproject): def _print_unknown_command(settings, cmdname, inproject): _print_header(settings, inproject) - print("Unknown command: %s\n" % cmdname) + print(f"Unknown command: {cmdname}\n") print('Use "scrapy" to see available commands') @@ -133,7 +133,7 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax()) + parser.usage = f"scrapy {cmdname} {cmd.syntax()}" parser.description = cmd.long_desc() settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings @@ -155,7 +155,7 @@ def _run_command(cmd, args, opts): def _run_command_profiled(cmd, args, opts): if opts.profile: - sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) + sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n") loc = locals() p = cProfile.Profile() p.runctx('cmd.run(args, opts)', globals(), loc) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index cfd940fe7..23ccffcd9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -61,7 +61,7 @@ class ScrapyCommand: group.add_option("--logfile", metavar="FILE", help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help=f"log level (default: {self.settings['LOG_LEVEL']})") group.add_option("--nolog", action="store_true", help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index c9f3b38e0..999c987ea 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -50,7 +50,7 @@ class _BenchSpider(scrapy.Spider): def start_requests(self): qargs = {'total': self.total, 'show': self.show} - url = '{}?{}'.format(self.baseurl, urlencode(qargs, doseq=1)) + url = f'{self.baseurl}?{urlencode(qargs, doseq=1)}' return [scrapy.Request(url, dont_filter=True)] def parse(self, response): diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 09a76ca7a..7e848dc97 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start)) + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}") writeln() infos = [] @@ -25,14 +25,14 @@ class TextTestResult(_TextTestResult): write("FAILED") failed, errored = map(len, (self.failures, self.errors)) if failed: - infos.append("failures=%d" % failed) + infos.append(f"failures={failed}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"errors={errored}") else: write("OK") if infos: - writeln(" (%s)" % (", ".join(infos),)) + writeln(f" ({', '.join(infos)})") else: write("\n") @@ -85,7 +85,7 @@ class Command(ScrapyCommand): continue print(spider) for method in sorted(methods): - print(' * %s' % method) + print(f' * {method}') else: start = time.time() self.crawler_process.start() diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 25d843a53..177b20143 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -32,8 +32,8 @@ class Command(ScrapyCommand): try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: - return self._err("Spider not found: %s" % args[0]) + return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace('.pyc', '.py') - self.exitcode = os.system('%s "%s"' % (editor, sfile)) + self.exitcode = os.system(f'{editor} "{sfile}"') diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 74a077d1b..72248bded 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -73,17 +73,18 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, domain, opts.template, template_file) if opts.edit: - self.exitcode = os.system('scrapy edit "%s"' % name) + self.exitcode = os.system(f'scrapy edit "{name}"') def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" + capitalized_module = ''.join(s.capitalize() for s in module.split('_')) tvars = { 'project_name': self.settings.get('BOT_NAME'), 'ProjectName': string_camelcase(self.settings.get('BOT_NAME')), 'module': module, 'name': name, 'domain': domain, - 'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_')) + 'classname': f'{capitalized_module}Spider' } if self.settings.get('NEWSPIDER_MODULE'): spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) @@ -91,32 +92,32 @@ class Command(ScrapyCommand): else: spiders_module = None spiders_dir = "." - spider_file = "%s.py" % join(spiders_dir, module) + spider_file = f"{join(spiders_dir, module)}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print("Created spider %r using template %r " - % (name, template_name), end=('' if spiders_module else '\n')) + print(f"Created spider {name!r} using template {template_name!r} ", + end=('' if spiders_module else '\n')) if spiders_module: - print("in module:\n %s.%s" % (spiders_module.__name__, module)) + print("in module:\n {spiders_module.__name__}.{module}") def _find_template(self, template): - template_file = join(self.templates_dir, '%s.tmpl' % template) + template_file = join(self.templates_dir, f'{template}.tmpl') if exists(template_file): return template_file - print("Unable to find template: %s\n" % template) + print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') def _list_templates(self): print("Available templates:") for filename in sorted(os.listdir(self.templates_dir)): if filename.endswith('.tmpl'): - print(" %s" % splitext(filename)[0]) + print(f" {splitext(filename)[0]}") def _spider_exists(self, name): if not self.settings.get('NEWSPIDER_MODULE'): # if run as a standalone command and file with same filename already exists if exists(name + ".py"): - print("%s already exists" % (abspath(name + ".py"))) + print(f"{abspath(name + '.py')} already exists") return True return False @@ -126,8 +127,8 @@ class Command(ScrapyCommand): pass else: # if spider with same name exists - print("Spider %r already exists in module:" % name) - print(" %s" % spidercls.__module__) + print(f"Spider {name!r} already exists in module:") + print(f" {spidercls.__module__}") return True # a file with the same name exists in the target directory @@ -135,7 +136,7 @@ class Command(ScrapyCommand): spiders_dir = dirname(spiders_module.__file__) spiders_dir_abs = abspath(spiders_dir) if exists(join(spiders_dir_abs, name + ".py")): - print("%s already exists" % (join(spiders_dir_abs, (name + ".py")))) + print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") return True return False diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index abc8ba9ff..83ee074da 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -96,13 +96,13 @@ class Command(BaseRunSpiderCommand): if opts.verbose: for level in range(1, self.max_level + 1): - print('\n>>> DEPTH LEVEL: %s <<<' % level) + print(f'\n>>> DEPTH LEVEL: {level} <<<') if not opts.noitems: self.print_items(level, colour) if not opts.nolinks: self.print_requests(level, colour) else: - print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level) + print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<') if not opts.noitems: self.print_items(colour=colour) if not opts.nolinks: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index befee021b..aedd8c2ce 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,7 +12,7 @@ def _import_file(filepath): dirname, file = os.path.split(abspath) fname, fext = os.path.splitext(file) if fext != '.py': - raise ValueError("Not a Python source file: %s" % abspath) + raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path try: @@ -42,14 +42,14 @@ class Command(BaseRunSpiderCommand): raise UsageError() filename = args[0] if not os.path.exists(filename): - raise UsageError("File not found: %s\n" % filename) + raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError("Unable to load %r: %s\n" % (filename, e)) + raise UsageError(f"Unable to load {filename!r}: {e}\n") spclasses = list(iter_spider_classes(module)) if not spclasses: - raise UsageError("No spider found in file: %s\n" % filename) + raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() self.crawler_process.crawl(spidercls, **opts.spargs) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e5158d993..1d73fa0cb 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -52,7 +52,7 @@ class Command(ScrapyCommand): print('Error: Project names must begin with a letter and contain' ' only\nletters, numbers and underscores') elif _module_exists(project_name): - print('Error: Module %r already exists' % project_name) + print(f'Error: Module {project_name!r} already exists') else: return True return False @@ -100,7 +100,7 @@ class Command(ScrapyCommand): if exists(join(project_dir, 'scrapy.cfg')): self.exitcode = 1 - print('Error: scrapy.cfg already exists in %s' % abspath(project_dir)) + print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') return if not self._is_valid_name(project_name): @@ -113,11 +113,11 @@ class Command(ScrapyCommand): path = join(*paths) tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) - print("New Scrapy project '%s', using template directory '%s', " - "created in:" % (project_name, self.templates_dir)) - print(" %s\n" % abspath(project_dir)) + print(f"New Scrapy project '{project_name}', using template directory " + f"'{self.templates_dir}', created in:") + print(f" {abspath(project_dir)}\n") print("You can start your first spider with:") - print(" cd %s" % project_dir) + print(f" cd {project_dir}") print(" scrapy genspider example example.com") @property diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index d0ea72a67..dc8087043 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,8 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = "%-{}s : %s".format(width) + patt = f"%-{width}s : %s" for name, version in versions: print(patt % (name, version)) else: - print("Scrapy %s" % scrapy.__version__) + print(f"Scrapy {scrapy.__version__}") diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 5af3831a2..db0a56e56 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -112,8 +112,8 @@ class Contract: request_cls = None def __init__(self, method, *args): - self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) - self.testcase_post = _create_testcase(method, '@%s post-hook' % self.name) + self.testcase_pre = _create_testcase(method, f'@{self.name} pre-hook') + self.testcase_post = _create_testcase(method, f'@{self.name} post-hook') self.args = args def add_pre_hook(self, request, results): @@ -172,8 +172,8 @@ def _create_testcase(method, desc): class ContractTestCase(TestCase): def __str__(_self): - return "[%s] %s (%s)" % (spider, method.__name__, desc) + return f"[{spider}] {method.__name__} ({desc})" - name = '%s_%s' % (spider, method.__name__) + name = f'{spider}_{method.__name__}' setattr(ContractTestCase, name, lambda x: x) return ContractTestCase(name) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index cfdcc7c25..9704f5253 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -60,8 +60,7 @@ class ReturnsContract(Contract): if len(self.args) not in [1, 2, 3]: raise ValueError( - "Incorrect argument quantity: expected 1, 2 or 3, got %i" - % len(self.args) + f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) self.obj_name = self.args[0] or None self.obj_type_verifier = self.object_type_verifiers[self.obj_name] @@ -88,10 +87,9 @@ class ReturnsContract(Contract): if self.min_bound == self.max_bound: expected = self.min_bound else: - expected = '%s..%s' % (self.min_bound, self.max_bound) + expected = f'{self.min_bound}..{self.max_bound}' - raise ContractFail("Returned %s %s, expected %s" % - (occurrences, self.obj_name, expected)) + raise ContractFail(f"Returned {occurrences} {self.obj_name}, expected {expected}") class ScrapesContract(Contract): @@ -106,5 +104,5 @@ class ScrapesContract(Contract): if is_item(x): missing = [arg for arg in self.args if arg not in ItemAdapter(x)] if missing: - missing_str = ", ".join(missing) - raise ContractFail("Missing fields: %s" % missing_str) + missing_fields = ", ".join(missing) + raise ContractFail(f"Missing fields: {missing_fields}") diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index dc5cf1ab8..12a9db6dd 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -41,17 +41,17 @@ class Slot: def __repr__(self): cls_name = self.__class__.__name__ - return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( - cls_name, self.concurrency, self.delay, self.randomize_delay) + return (f"{cls_name}(concurrency={self.concurrency!r}, " + f"delay={self.delay:.2f}, " + f"randomize_delay={self.randomize_delay!r}") def __str__(self): return ( - "" % ( - self.concurrency, self.delay, self.randomize_delay, - len(self.active), len(self.queue), len(self.transferring), - datetime.fromtimestamp(self.lastseen).isoformat() - ) + f"" ) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index e86680978..73aeb2352 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -71,8 +71,7 @@ class DownloadHandlers: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: - raise NotSupported("Unsupported URL scheme '%s': %s" % - (scheme, self._notconfigured[scheme])) + raise NotSupported(f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}") return handler.download_request(request, spider) @defer.inlineCallbacks diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 25e800984..1b041c8a8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -60,11 +60,11 @@ class HTTP11DownloadHandler: settings=settings, crawler=crawler, ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + msg = f""" + '{settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]}' does not accept `method` \ + argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD) and/or \ + `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') @@ -169,8 +169,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): else: extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( - self._host, self._port, extra))) + TunnelError('Could not open CONNECT tunnel with proxy ' + f'{self._host}:{self._port} [{extra!r}]') + ) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -371,7 +372,7 @@ class ScrapyAgent: if self._txresponse: self._txresponse._transport.stopProducing() - raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") def _cb_latency(self, result, request, start_time): request.meta['download_latency'] = time() - start_time diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 8f63ad974..0ef977893 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -56,7 +56,7 @@ class S3DownloadHandler: import botocore.credentials kw.pop('anon', None) if kw: - raise TypeError('Unexpected keyword arguments: %s' % kw) + raise TypeError(f'Unexpected keyword arguments: {kw}') if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( @@ -85,14 +85,14 @@ class S3DownloadHandler: scheme = 'https' if request.meta.get('is_secure') else 'http' bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path - url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) + url = f'{scheme}://{bucket}.s3.amazonaws.com{path}' if self.anon: request = request.replace(url=url) elif self._signer is not None: import botocore.awsrequest awsrequest = botocore.awsrequest.AWSRequest( method=request.method, - url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path), + url=f'{scheme}://s3.amazonaws.com/{bucket}{path}', headers=request.headers.to_unicode_dict(), data=request.body) self._signer.add_auth(awsrequest) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 4c2eea522..b0e612e43 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -36,8 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_request must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, response.__class__.__name__) + f"Middleware {method.__self__.__class__.__name__}" + ".process_request must return None, Response or " + f"Request, got {response.__class__.__name__}" ) if response: return response @@ -54,8 +55,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_response must return Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_response must return Response or Request, " + f"got {type(response)}" ) if isinstance(response, Request): return response @@ -68,8 +70,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( - "Middleware %s.process_exception must return None, Response or Request, got %s" - % (method.__self__.__class__.__name__, type(response)) + f"Middleware {method.__self__.__class__.__name__}" + ".process_exception must return None, Response or " + f"Request, got {type(response)}" ) if response: return response diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index b2b96f1ea..c13683393 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." - % (self.factory.url, self.factory.timeout))) + defer.TimeoutError(f"Getting {self.factory.url} took longer " + f"than {self.factory.timeout} seconds.")) # This class used to inherit from Twisted’s @@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): self.headers['Content-Length'] = 0 def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, self.url) + return f"<{self.__class__.__name__}: {self.url}>" def _cancelTimeout(self, result, timeoutCall): if timeoutCall.active(): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 5e0dfe37c..93bcdb49a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -171,8 +171,8 @@ class ExecutionEngine: def _handle_downloader_output(self, response, request, spider): if not isinstance(response, (Request, Response, Failure)): raise TypeError( - "Incorrect type: expected Request, Response or Failure, got %s: %r" - % (type(response), response) + "Incorrect type: expected Request, Response or Failure, got " + f"{type(response)}: {response!r}" ) # downloader middleware can return requests (for example, redirects) if isinstance(response, Request): @@ -214,7 +214,7 @@ class ExecutionEngine: def crawl(self, request, spider): if spider not in self.open_spiders: - raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request)) + raise RuntimeError(f"Spider {spider.name!r} not opened when crawling: {request}") self.schedule(request, spider) self.slot.nextcall.schedule() @@ -239,8 +239,8 @@ class ExecutionEngine: def _on_success(response): if not isinstance(response, (Response, Request)): raise TypeError( - "Incorrect type: expected Response or Request, got %s: %r" - % (type(response), response) + "Incorrect type: expected Response or Request, got " + f"{type(response)}: {response!r}" ) if isinstance(response, Response): if response.request is None: @@ -268,7 +268,7 @@ class ExecutionEngine: @defer.inlineCallbacks def open_spider(self, spider, start_requests=(), close_if_idle=True): if not self.has_capacity(): - raise RuntimeError("No free spider slot when opening %r" % spider.name) + raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={'spider': spider}) nextcall = CallLaterOnce(self._next_request, spider) scheduler = self.scheduler_cls.from_crawler(self.crawler) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 20bdb22a1..0d3e3450f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -125,7 +125,7 @@ class Scraper: Handle the downloaded response or failure through the spider callback/errback """ if not isinstance(result, (Response, Failure)): - raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result)) + raise TypeError(f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}") dfd = self._scrape2(result, request, spider) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, result, spider) dfd.addCallback(self.handle_spider_output, request, result, spider) @@ -173,7 +173,7 @@ class Scraper: spider=spider ) self.crawler.stats.inc_value( - "spider_exceptions/%s" % _failure.value.__class__.__name__, + f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 5a99b96be..763e0cdf6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,10 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "{}.{}".format( - f.__self__.__class__.__name__, - f.__func__.__name__ - ) + return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" class SpiderMiddlewareManager(MiddlewareManager): @@ -51,8 +48,9 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = "Middleware {} must return None or raise an exception, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or raise an exception, got {type(result)}") + raise _InvalidOutput(msg) except _InvalidOutput: raise except Exception: @@ -86,8 +84,9 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = "Middleware {} must return None or an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return None " + f"or an iterable, got {type(result)}") + raise _InvalidOutput(msg) return _failure def process_spider_output(result, start_index=0): @@ -110,8 +109,9 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = _evaluate_iterable(result, method_index + 1, recovered) else: - msg = "Middleware {} must return an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = (f"Middleware {_fname(method)} must return an " + f"iterable, got {type(result)}") + raise _InvalidOutput(msg) return MutableChain(result, recovered) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 77048f389..e2b7dd901 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -54,8 +54,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: - cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) - msg = "Sending cookies to: {}\n{}".format(request, cookies) + cookies = "\n".join(f"Cookie: {c}\n" for c in cl) + msg = f"Sending cookies to: {request}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _debug_set_cookie(self, response, spider): @@ -63,8 +63,8 @@ class CookiesMiddleware: cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: - cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) - msg = "Received cookies from: {}\n{}".format(response, cookies) + cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl) + msg = f"Received cookies from: {response}\n{cookies}" logger.debug(msg, extra={'spider': spider}) def _format_cookie(self, cookie, request): @@ -90,9 +90,9 @@ class CookiesMiddleware: request, cookie) decoded[key] = cookie[key].decode("latin1", errors="replace") - cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) + cookie_str = f"{decoded.pop('name')}={decoded.pop('value')}" for key, value in decoded.items(): # path, domain - cookie_str += "; {}={}".format(key.capitalize(), value) + cookie_str += f"; {key.capitalize()}={value}" return cookie_str def _get_request_cookies(self, jar, request): diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index da89d3e9b..04da11311 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -24,7 +24,7 @@ class HttpProxyMiddleware: def _basic_auth_header(self, username, password): user_pass = to_bytes( - '%s:%s' % (unquote(username), unquote(password)), + f'{unquote(username)}:{unquote(password)}', encoding=self.auth_encoding) return base64.b64encode(user_pass) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 67be8c282..51fe59254 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -88,7 +88,7 @@ class RetryMiddleware: reason = global_object_name(reason.__class__) stats.inc_value('retry/count') - stats.inc_value('retry/reason_count/%s' % reason) + stats.inc_value(f'retry/reason_count/{reason}') return retryreq else: stats.inc_value('retry/max_reached') diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f18b2bf2..d6da55535 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -61,7 +61,7 @@ class RobotsTxtMiddleware: if netloc not in self._parsers: self._parsers[netloc] = Deferred() - robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) + robotsurl = f"{url.scheme}://{url.netloc}/robots.txt" robotsreq = Request( robotsurl, priority=self.DOWNLOAD_PRIORITY, @@ -94,7 +94,7 @@ class RobotsTxtMiddleware: def _parse_robots(self, response, netloc, spider): self.crawler.stats.inc_value('robotstxt/response_count') - self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status)) + self.crawler.stats.inc_value(f'robotstxt/response_status_count/{response.status}') rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp @@ -102,7 +102,7 @@ class RobotsTxtMiddleware: def _robots_error(self, failure, netloc): if failure.type is not IgnoreRequest: - key = 'robotstxt/exception_count/{}'.format(failure.type) + key = f'robotstxt/exception_count/{failure.type}' self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 46a2ad397..5479cd0e2 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -17,13 +17,13 @@ class DownloaderStats: def process_request(self, request, spider): self.stats.inc_value('downloader/request_count', spider=spider) - self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) + self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider) reqlen = len(request_httprepr(request)) self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) - self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) + self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) reslen = len(response_httprepr(response)) self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response @@ -31,4 +31,4 @@ class DownloaderStats: def process_exception(self, request, exception, spider): ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) - self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) + self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 95518b3ac..54cf5c0b1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -39,7 +39,7 @@ class BaseItemExporter: self.export_empty_fields = options.pop('export_empty_fields', False) self.indent = options.pop('indent', None) if not dont_fail and options: - raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) + raise TypeError(f"Unexpected options: {', '.join(options.keys())}") def export_item(self, item): raise NotImplementedError diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 389cb65bc..675f8276f 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -43,4 +43,4 @@ class CoreStats: def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ self.stats.inc_value('item_dropped_count', spider=spider) - self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) + self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 586399784..fd2a02d8d 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -48,7 +48,7 @@ class StackTraceDump: for id_, frame in sys._current_frames().items(): name = id2name.get(id_, '') dump = ''.join(traceback.format_stack(frame)) - dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump) + dumps += f"# Thread: {name}({id_})\n{dump}\n" return dumps diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 6294a9b52..e0c04b2de 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -223,7 +223,7 @@ class DbmCacheStorage: self.db = None def open_spider(self, spider): - dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) + dbpath = os.path.join(self.cachedir, f'{spider.name}.db') self.db = self.dbmodule.open(dbpath, 'c') logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) @@ -251,13 +251,13 @@ class DbmCacheStorage: 'headers': dict(response.headers), 'body': response.body, } - self.db['%s_data' % key] = pickle.dumps(data, protocol=4) - self.db['%s_time' % key] = str(time()) + self.db[f'{key}_data'] = pickle.dumps(data, protocol=4) + self.db[f'{key}_time'] = str(time()) def _read_data(self, spider, request): key = self._request_key(request) db = self.db - tkey = '%s_time' % key + tkey = f'{key}_time' if tkey not in db: return # not found @@ -265,7 +265,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db['%s_data' % key]) + return pickle.loads(db[f'{key}_data']) def _request_key(self, request): return request_fingerprint(request) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index dc8cdbb1d..cee44ea62 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -30,4 +30,4 @@ class MemoryDebugger: for cls, wdict in live_refs.items(): if not wdict: continue - self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) + self.stats.set_value(f'memdebug/live_refs/{cls.__name__}', len(wdict), spider=spider) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ab2e43e8c..274cbdbfe 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -82,8 +82,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s terminated: memory usage exceeded %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} terminated: " + f"memory usage exceeded {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -105,8 +105,8 @@ class MemoryUsage: {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( - "%s warning: memory usage reached %dM at %s" - % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + f"{self.crawler.settings['BOT_NAME']} warning: " + f"memory usage reached {mem}M at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) @@ -115,9 +115,9 @@ class MemoryUsage: def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" stats = self.crawler.stats - s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) - s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) - s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) + s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n" + s += f"Maximum memory usage : {stats.get_value('memusage/max')/1024/1024}M\r\n" + s += f"Current memory usage : {self.get_virtual_size()/1024/1024}M\r\n" s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 320f13b29..997e74fc9 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -24,11 +24,11 @@ class StatsMailer: o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - + def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.name - body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - return self.mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) + body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats()) + body += f"\n\n{spider.name} stats\n\n" + body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats) + return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) diff --git a/scrapy/http/common.py b/scrapy/http/common.py index ba6ab277c..98699d7fd 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,6 +1,6 @@ def obsolete_setter(setter, attrname): def newsetter(self, value): c = self.__class__.__name__ - msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) + msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) return newsetter diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 6bf9e5346..1a2b99b0a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -33,7 +33,7 @@ class Headers(CaselessDict): elif isinstance(x, int): return str(x).encode(self.encoding) else: - raise TypeError('Unsupported value type: {}'.format(type(x))) + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a98ba9960..ef58deacc 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -25,13 +25,13 @@ class Request(object_ref): self._set_url(url) self._set_body(body) if not isinstance(priority, int): - raise TypeError("Request priority not an integer: %r" % priority) + raise TypeError(f"Request priority not an integer: {priority!r}") self.priority = priority if callback is not None and not callable(callback): - raise TypeError('callback must be a callable, got %s' % type(callback).__name__) + raise TypeError(f'callback must be a callable, got {type(callback).__name__}') if errback is not None and not callable(errback): - raise TypeError('errback must be a callable, got %s' % type(errback).__name__) + raise TypeError(f'errback must be a callable, got {type(errback).__name__}') self.callback = callback self.errback = errback @@ -60,13 +60,13 @@ class Request(object_ref): def _set_url(self, url): if not isinstance(url, str): - raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) + raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) if ('://' not in self._url) and (not self._url.startswith('data:')): - raise ValueError('Missing scheme in request url: %s' % self._url) + raise ValueError(f'Missing scheme in request url: {self._url}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -86,7 +86,7 @@ class Request(object_ref): return self._encoding def __str__(self): - return "<%s %s>" % (self.method, self.url) + return f"<{self.method} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 59af81321..c90d68fa1 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -80,15 +80,15 @@ def _get_form(response, formname, formid, formnumber, formxpath): base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: - raise ValueError("No
    element found in %s" % response) + raise ValueError(f"No element found in {response}") if formname is not None: - f = root.xpath('//form[@name="%s"]' % formname) + f = root.xpath(f'//form[@name="{formname}"]') if f: return f[0] if formid is not None: - f = root.xpath('//form[@id="%s"]' % formid) + f = root.xpath(f'//form[@id="{formid}"]') if f: return f[0] @@ -103,7 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - raise ValueError('No element found with %s' % formxpath) + raise ValueError(f'No element found with {formxpath}') # If we get here, it means that either formname was None # or invalid @@ -111,8 +111,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): try: form = forms[formnumber] except IndexError: - raise IndexError("Form number %d not found in %s" % - (formnumber, response)) + raise IndexError(f"Form number {formnumber} not found in {response}") else: return form @@ -205,12 +204,12 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') elif len(el) > 1: - raise ValueError("Multiple elements found (%r) matching the criteria " - "in clickdata: %r" % (el, clickdata)) + raise ValueError(f"Multiple elements found ({el!r}) matching the " + f"criteria in clickdata: {clickdata!r}") else: - raise ValueError('No clickable element matching clickdata: %r' % (clickdata,)) + raise ValueError(f'No clickable element matching clickdata: {clickdata!r}') diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index c2c37dd1d..c635fde69 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -55,8 +55,8 @@ class Response(object_ref): if isinstance(url, str): self._url = url else: - raise TypeError('%s url must be str, got %s:' % - (type(self).__name__, type(url).__name__)) + raise TypeError(f'{type(self).__name__} url must be str, ' + f'got {type(url).__name__}') url = property(_get_url, obsolete_setter(_set_url, 'url')) @@ -77,7 +77,7 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) def __str__(self): - return "<%d %s>" % (self.status, self.url) + return f"<{self.status} {self.url}>" __repr__ = __str__ diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a7bb34d48..e36e14880 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -47,8 +47,8 @@ class TextResponse(Response): self._body = b'' # used by encoding detection if isinstance(body, str): if self._encoding is None: - raise TypeError('Cannot convert unicode body - %s has no encoding' % - type(self).__name__) + raise TypeError('Cannot convert unicode body - ' + f'{type(self).__name__} has no encoding') self._body = body.encode(self._encoding) else: super()._set_body(body) @@ -92,7 +92,7 @@ class TextResponse(Response): # _body_inferred_encoding is called benc = self.encoding if self._cached_ubody is None: - charset = 'charset=%s' % benc + charset = f'charset={benc}' self._cached_ubody = html_to_unicode(charset, self.body)[1] return self._cached_ubody @@ -255,12 +255,11 @@ def _url_from_selector(sel): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): - raise _InvalidSelector("Unsupported selector: %s" % sel) + raise _InvalidSelector(f"Unsupported selector: {sel}") if sel.root.tag not in ('a', 'link'): - raise _InvalidSelector("Only and elements are supported; got <%s>" % - sel.root.tag) + raise _InvalidSelector("Only and elements are supported; " + f"got <{sel.root.tag}>") href = sel.root.get('href') if href is None: - raise _InvalidSelector("<%s> element has no href attribute: %s" % - (sel.root.tag, sel)) + raise _InvalidSelector(f"<{sel.root.tag}> element has no href attribute: {sel}") return strip_html5_whitespace(href) diff --git a/scrapy/item.py b/scrapy/item.py index c262a153c..af3849302 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -96,19 +96,19 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) + raise KeyError(f"{self.__class__.__name__} does not support field: {key}") def __delitem__(self, key): del self._values[key] def __getattr__(self, name): if name in self.fields: - raise AttributeError("Use item[%r] to get field value" % name) + raise AttributeError(f"Use item[{name!r}] to get field value") raise AttributeError(name) def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) + raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value") super().__setattr__(name, value) def __len__(self): diff --git a/scrapy/link.py b/scrapy/link.py index 1ef50b113..684735f6e 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -14,7 +14,7 @@ class Link: def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): got = url.__class__.__name__ - raise TypeError("Link urls must be str objects, got %s" % got) + raise TypeError(f"Link urls must be str objects, got {got}") self.url = url self.text = text self.fragment = fragment @@ -33,6 +33,6 @@ class Link: def __repr__(self): return ( - 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' - % (self.url, self.text, self.fragment, self.nofollow) + f'Link(url={self.url!r}, text={self.text!r}, ' + f'fragment={self.fragment!r}, nofollow={self.nofollow!r})' ) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 0f9e6f1cb..87568b2d1 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -54,8 +54,8 @@ class LogFormatter: def crawled(self, request, response, spider): """Logs a message when the crawler finds a webpage.""" - request_flags = ' %s' % str(request.flags) if request.flags else '' - response_flags = ' %s' % str(response.flags) if response.flags else '' + request_flags = f' {str(request.flags)}' if request.flags else '' + response_flags = f' {str(response.flags)}' if response.flags else '' return { 'level': logging.DEBUG, 'msg': CRAWLEDMSG, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 5a2184681..99a72aa70 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -108,7 +108,7 @@ class S3FilesStore: from boto.s3.connection import S3Connection self.S3Connection = S3Connection if not uri.startswith("s3://"): - raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split('/', 1) def stat_file(self, path, info): @@ -133,7 +133,7 @@ class S3FilesStore: return c.get_bucket(self.bucket, validate=False) def _get_boto_key(self, path): - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' if self.is_botocore: return threads.deferToThread( self.s3_client.head_object, @@ -145,7 +145,7 @@ class S3FilesStore: def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" - key_name = '%s%s' % (self.prefix, path) + key_name = f'{self.prefix}{path}' buf.seek(0) if self.is_botocore: extra = self._headers_to_botocore_kwargs(self.HEADERS) @@ -208,8 +208,7 @@ class S3FilesStore: try: kwarg = mapping[key] except KeyError: - raise TypeError( - 'Header "%s" is not supported by botocore' % key) + raise TypeError(f'Header "{key}" is not supported by botocore') else: extra[kwarg] = value return extra @@ -283,7 +282,7 @@ class FTPFilesStore: def __init__(self, uri): if not uri.startswith("ftp://"): - raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri) + raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'") u = urlparse(uri) self.port = u.port self.host = u.hostname @@ -293,7 +292,7 @@ class FTPFilesStore: self.basedir = u.path.rstrip('/') def persist_file(self, path, buf, info, meta=None, headers=None): - path = '%s/%s' % (self.basedir, path) + path = f'{self.basedir}/{path}' return threads.deferToThread( ftp_store_file, path=path, file=buf, host=self.host, port=self.port, username=self.username, @@ -308,10 +307,10 @@ class FTPFilesStore: ftp.login(self.username, self.password) if self.USE_ACTIVE_MODE: ftp.set_pasv(False) - file_path = "%s/%s" % (self.basedir, path) - last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip()) + file_path = f"{self.basedir}/{path}" + last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) m = hashlib.md5() - ftp.retrbinary('RETR %s' % file_path, m.update) + ftp.retrbinary(f'RETR {file_path}', m.update) return {'last_modified': last_modified, 'checksum': m.hexdigest()} # The file doesn't exist except Exception: @@ -515,7 +514,7 @@ class FilesPipeline(MediaPipeline): def inc_stats(self, spider, status): spider.crawler.stats.inc_value('file_count', spider=spider) - spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + spider.crawler.stats.inc_value(f'file_status_count/{status}', spider=spider) # Overridable Interface def get_media_requests(self, item, info): @@ -545,4 +544,4 @@ class FilesPipeline(MediaPipeline): media_type = mimetypes.guess_type(request.url)[0] if media_type: media_ext = mimetypes.guess_extension(media_type) - return 'full/%s%s' % (media_guid, media_ext) + return f'full/{media_guid}{media_ext}' diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 0a67a0b1d..aafd1d8b2 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -125,8 +125,9 @@ class ImagesPipeline(FilesPipeline): width, height = orig_image.size if width < self.min_width or height < self.min_height: - raise ImageException("Image too small (%dx%d < %dx%d)" % - (width, height, self.min_width, self.min_height)) + raise ImageException("Image too small " + f"({width}x{height} < " + f"{self.min_width}x{self.min_height})") image, buf = self.convert_image(orig_image) yield path, image, buf @@ -168,8 +169,8 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'full/%s.jpg' % (image_guid) + return f'full/{image_guid}.jpg' def thumb_path(self, request, thumb_id, response=None, info=None): thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbs/{thumb_id}/{thumb_guid}.jpg' diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 2439de9a5..0a12f3e2c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -61,7 +61,7 @@ class MediaPipeline: 'MYPIPE_IMAGES' """ class_name = self.__class__.__name__ - formatted_key = "{}_{}".format(class_name.upper(), key) + formatted_key = f"{class_name.upper()}_{key}" if ( not base_class_name or class_name == base_class_name @@ -151,9 +151,8 @@ class MediaPipeline: if 'item' not in sig.parameters: old_params = str(sig)[1:-1] new_params = old_params + ", *, item=None" - warn('%s(self, %s) is deprecated, ' - 'please use %s(self, %s)' - % (func.__name__, old_params, func.__name__, new_params), + warn(f'{func.__name__}(self, {old_params}) is deprecated, ' + f'please use {func.__name__}(self, {new_params})', ScrapyDeprecationWarning, stacklevel=2) self._expects_item[func.__name__] = False diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index e13d389ee..a9aa6c649 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -141,17 +141,16 @@ class DownloaderAwarePriorityQueue: def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: - raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' - % (self.__class__,)) + raise ValueError(f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP') if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError("DownloaderAwarePriorityQueue accepts " - "``slot_startprios`` as a dict; %r instance " + "``slot_startprios`` as a dict; " + f"{slot_startprios.__class__!r} instance " "is passed. Most likely, it means the state is" "created by an incompatible priority queue. " "Only a crawl started with the same priority " - "queue class can be resumed." % - slot_startprios.__class__) + "queue class can be resumed.") self._downloader_interface = DownloaderInterface(crawler) self.downstream_queue_cls = downstream_queue_cls diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index d207088e6..6ed9f8b8f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -45,7 +45,7 @@ class ResponseTypes: elif mimetype in self.classes: return self.classes[mimetype] else: - basetype = "%s/*" % mimetype.split('/')[0] + basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index f12c61081..a25871433 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -66,8 +66,8 @@ class Selector(_ParselSelector, object_ref): def __init__(self, response=None, text=None, type=None, root=None, **kwargs): if response is not None and text is not None: - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + raise ValueError(f'{self.__class__.__name__}.__init__() received ' + 'both response and text') st = _st(response, type or self._default_type) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 951fc65e2..1fe1e6fd1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -52,7 +52,7 @@ class SettingsAttribute: self.priority = priority def __str__(self): - return "".format(self=self) + return f"" __repr__ = __str__ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a0251394b..4ef330dd2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -287,7 +287,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = f'Scrapy/{import_module("scrapy").__version__} (+https://scrapy.org)' TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/shell.py b/scrapy/shell.py index 10de119ce..c370ccaff 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -140,7 +140,7 @@ class Shell: b.append(" scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)") for k, v in sorted(self.vars.items()): if self._is_relevant(v): - b.append(" %-10s %s" % (k, v)) + b.append(f" {k:<10} {v}") b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " @@ -150,7 +150,7 @@ class Shell: b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % line for line in b) + return "\n".join(f"[s] {line}" for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) or is_item(value) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index db4193430..04fda311f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader: dupes = [] for name, locations in self._found.items(): dupes.extend([ - " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + f" {cls} named {name!r} (in {mod})" for mod, cls in locations if len(locations) > 1 ]) @@ -36,7 +36,7 @@ class SpiderLoader: dupes_string = "\n\n".join(dupes) warnings.warn( "There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string), + f"{dupes_string}\n\n This can cause unexpected behavior.", category=UserWarning, ) @@ -53,10 +53,9 @@ class SpiderLoader: except ImportError: if self.warn_only: warnings.warn( - "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc() - ), + f"\n{traceback.format_exc()}Could not load spiders " + f"from module '{name}'. " + "See above traceback for details.", category=RuntimeWarning, ) else: @@ -75,7 +74,7 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError("Spider not found: {}".format(spider_name)) + raise KeyError(f"Spider not found: {spider_name}") def find_by_request(self, request): """ diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index fa7f5bef9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -43,7 +43,7 @@ class DepthMiddleware: return False else: if self.verbose_stats: - self.stats.inc_value('request_depth_count/%s' % depth, + self.stats.inc_value(f'request_depth_count/{depth}', spider=spider) self.stats.max_value('request_depth_max', depth, spider=spider) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index db9d0f2ae..ae5c258df 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -48,7 +48,7 @@ class HttpErrorMiddleware: if isinstance(exception, HttpError): spider.crawler.stats.inc_value('httperror/response_ignored_count') spider.crawler.stats.inc_value( - 'httperror/response_ignored_status_count/%s' % response.status + f'httperror/response_ignored_status_count/{response.status}' ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a006f3177..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -61,15 +61,15 @@ class OffsiteMiddleware: continue elif url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " - "Ignoring URL entry %s in allowed_domains." % domain) + f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) elif port_pattern.search(domain): message = ("allowed_domains accepts only domains without ports. " - "Ignoring entry %s in allowed_domains." % domain) + f"Ignoring entry {domain} in allowed_domains.") warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 434067b00..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -278,7 +278,7 @@ def _load_policy_class(policy, warning_only=False): try: return _policy_classes[policy.lower()] except KeyError: - msg = "Could not load referrer policy %r" % policy + msg = f"Could not load referrer policy {policy!r}" if not warning_only: raise RuntimeError(msg) else: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 12b4fba09..3da0a11db 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -25,7 +25,7 @@ class Spider(object_ref): if name is not None: self.name = name elif not getattr(self, 'name', None): - raise ValueError("%s must have a name" % type(self).__name__) + raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, 'start_urls'): self.start_urls = [] @@ -66,9 +66,8 @@ class Spider(object_ref): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " "won't be called in future Scrapy releases. Please " - "override Spider.start_requests method instead (see %s.%s)." % ( - cls.__module__, cls.__name__ - ), + "override Spider.start_requests method instead " + f"(see {cls.__module__}.{cls.__name__}).", ) for url in self.start_urls: yield self.make_requests_from_url(url) @@ -90,7 +89,7 @@ class Spider(object_ref): return self.parse(response, **kwargs) def parse(self, response, **kwargs): - raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) + raise NotImplementedError(f'{self.__class__.__name__}.parse callback is not defined') @classmethod def update_settings(cls, settings): @@ -107,7 +106,7 @@ class Spider(object_ref): return closed(reason) def __str__(self): - return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) + return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" __repr__ = __str__ diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index cf658aec4..6ed17e4dd 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -71,11 +71,11 @@ class XMLFeedSpider(Spider): elif self.iterator == 'xml': selector = Selector(response, type='xml') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') elif self.iterator == 'html': selector = Selector(response, type='html') self._register_namespaces(selector) - nodes = selector.xpath('//%s' % self.itertag) + nodes = selector.xpath(f'//{self.itertag}') else: raise NotSupported('Unsupported node iterator') diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index f595a1acb..86238c4cd 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -21,8 +21,8 @@ class Root(Resource): for nl in nlist: args['n'] = nl argstr = urlencode(args, doseq=True) - request.write("follow {1}
    " - .format(argstr, nl).encode('utf8')) + request.write(f"follow {nl}
    " + .encode('utf8')) request.write(b"") return b'' @@ -39,6 +39,6 @@ if __name__ == '__main__': def _print_listening(): httpHost = httpPort.getHost() - print("Bench server at http://{}:{}".format(httpHost.host, httpHost.port)) + print(f"Bench server at http://{httpHost.host}:{httpHost.port}") reactor.callWhenRunning(_print_listening) reactor.run() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 90a52b25b..05cd5f25c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,8 +17,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {!r} convert to the same object, ' - 'please update your settings'.format(complist)) + raise ValueError('Some paths in {complist!r} convert to the same object, ' + 'please update your settings') def _map_keys(compdict): if isinstance(compdict, BaseSettings): @@ -26,9 +26,10 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: - raise ValueError('Some paths in {!r} convert to the same ' + raise ValueError(f'Some paths in {list(compdict.keys())!r} ' + 'convert to the same ' 'object, please update your settings' - ''.format(list(compdict.keys()))) + ) else: compbs.set(convert(k), v, priority=prio) return compbs @@ -40,8 +41,9 @@ def build_component_list(compdict, custom=None, convert=update_classpath): """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError('Invalid value {} for component {}, please provide ' - 'a real number or None instead'.format(value, name)) + raise ValueError(f'Invalid value {value} for component {name}, ' + 'please provide a real number or None instead' + ) # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): @@ -141,12 +143,10 @@ def feed_process_params_from_cli(settings, output, output_format=None, def check_valid_format(output_format): if output_format not in valid_output_formats: raise UsageError( - "Unrecognized output format '%s'. Set a supported one (%s) " + f"Unrecognized output format '{output_format}'. " + f"Set a supported one ({tuple(valid_output_formats)}) " "after a colon at the end of the output URI (i.e. -o/-O " - ":) or as a file extension." % ( - output_format, - tuple(valid_output_formats), - ) + ":) or as a file extension." ) overwrite = False diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c0efcec4..6660b9dc0 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = 'There was an error parsing the curl command: {}'.format(message) + error_msg = f'There was an error parsing the curl command: {message}' raise ValueError(error_msg) @@ -52,7 +52,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) if argv: - msg = 'Unrecognized options: {}'.format(', '.join(argv)) + msg = f'Unrecognized options: {", ".join(argv)}' if ignore_unknown_options: warnings.warn(msg) else: diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 2e2c7adc1..fef3882cb 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -14,9 +14,9 @@ def deprecated(use_instead=None): def deco(func): @wraps(func) def wrapped(*args, **kwargs): - message = "Call to deprecated function %s." % func.__name__ + message = f"Call to deprecated function {func.__name__}." if use_instead: - message += " Use %s instead." % use_instead + message += f" Use {use_instead} instead." warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) return func(*args, **kwargs) return wrapped diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3c8e3c8b5..fb7e69889 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -8,9 +8,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning def attribute(obj, oldattr, newattr, version='0.12'): cname = obj.__class__.__name__ warnings.warn( - "%s.%s attribute is deprecated and will be no longer supported " - "in Scrapy %s, use %s.%s attribute instead" - % (cname, oldattr, version, cname, newattr), + f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " + f"in Scrapy {version}, use {cname}.{newattr} attribute instead", ScrapyDeprecationWarning, stacklevel=3) @@ -116,7 +115,7 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn("Error detecting parent module: %r" % e) + warnings.warn(f"Error detecting parent module: {e!r}") return deprecated_cls @@ -124,7 +123,7 @@ def create_deprecated_class( def _clspath(cls, forced=None): if forced is not None: return forced - return '{}.{}'.format(cls.__module__, cls.__name__) + return f'{cls.__module__}.{cls.__name__}' DEPRECATION_RULES = [ @@ -137,7 +136,7 @@ def update_classpath(path): for prefix, replacement in DEPRECATION_RULES: if path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) - warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), + warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning) return new_path return path diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 267c7ecd1..0c1cee1a0 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -29,7 +29,7 @@ def get_engine_status(engine): try: checks += [(test, eval(test))] except Exception as e: - checks += [(test, "%s (exception)" % type(e).__name__)] + checks += [(test, f"{type(e).__name__} (exception)")] return checks @@ -38,7 +38,7 @@ def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: - s += "%-47s : %s\n" % (test, result) + s += f"{test:<47} : {result}\n" s += "\n" return s diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 19d56d6ec..6cace4f07 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -33,5 +33,5 @@ def ftp_store_file( dirname, filename = posixpath.split(path) ftp_makedirs_cwd(ftp, dirname) command = 'STOR' if overwrite else 'APPE' - ftp.storbinary('%s %s' % (command, filename), file) + ftp.storbinary(f'{command} {filename}', file) file.close() diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 5e15bf0c8..789da1392 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -22,8 +22,8 @@ def xmliter(obj, nodename): """ nodename_patt = re.escape(nodename) - HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename_patt, re.S) - HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename_patt, re.S) + HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S) + HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S) text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) @@ -31,7 +31,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) + r = re.compile(fr'<{nodename_patt}[\s>].*?', re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -40,9 +40,9 @@ def xmliter(obj, nodename): def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): from lxml import etree reader = _StreamReader(obj) - tag = '{%s}%s' % (namespace, nodename) if namespace else nodename + tag = f'{{{namespace}}}{nodename}'if namespace else nodename iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) - selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) + selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding='unicode') node.clear() @@ -131,8 +131,7 @@ def _body_or_str(obj, unicode=True): if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) raise TypeError( - "Object %r must be %s, not %s" - % (obj, expected_types_str, type(obj).__name__) + f"Object {obj!r} must be {expected_types_str}, not {type(obj).__name__}" ) if isinstance(obj, Response): if not unicode: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index e41315738..62df7a6ab 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ - "%s %s" % (name, version) + f"{name} {version}" for name, version in scrapy_components_versions() if name != "Scrapy" ] @@ -187,7 +187,7 @@ class LogCounterHandler(logging.Handler): self.crawler = crawler def emit(self, record): - sname = 'log_count/{}'.format(record.levelname) + sname = f'log_count/{record.levelname}' self.crawler.stats.inc_value(sname) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index bd400bd30..9107f30ef 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -46,7 +46,7 @@ def load_object(path): try: dot = path.rindex('.') except ValueError: - raise ValueError("Error loading object '%s': not a full path" % path) + raise ValueError(f"Error loading object '{path}': not a full path") module, name = path[:dot], path[dot + 1:] mod = import_module(module) @@ -54,7 +54,7 @@ def load_object(path): try: obj = getattr(mod, name) except AttributeError: - raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name)) + raise NameError(f"Module '{module}' doesn't define any object named '{name}'") return obj @@ -163,7 +163,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): instance = objcls(*args, **kwargs) method_name = '__new__' if instance is None: - raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) + raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") return instance @@ -234,9 +234,10 @@ def warn_on_generator_with_return_value(spider, callable): """ if is_generator_with_return_value(callable): warnings.warn( - 'The "{}.{}" method is a generator and includes a "return" statement with a ' - 'value different than None. This could lead to unexpected behaviour. Please see ' + f'The "{spider.__class__.__name__}.{callable.__name__}" method is ' + 'a generator and includes a "return" statement with a value ' + 'different than None. This could lead to unexpected behaviour. Please see ' 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' - 'for details about the semantics of the "return" statement within generators' - .format(spider.__class__.__name__, callable.__name__), stacklevel=2, + 'for details about the semantics of the "return" statement within generators', + stacklevel=2, ) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index b8d3ebf9d..fd13d85e3 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -20,7 +20,7 @@ def inside_project(): try: import_module(scrapy_module) except ImportError as exc: - warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc)) + warnings.warn(f"Cannot import scrapy settings module {scrapy_module}: {exc}") else: return True return bool(closest_scrapy_cfg()) @@ -90,7 +90,7 @@ def get_project_settings(): warnings.warn( 'Use of environment variables prefixed with SCRAPY_ to override ' 'settings is deprecated. The following environment variables are ' - 'currently defined: {}'.format(setting_envvar_list), + f'currently defined: {setting_envvar_list}', ScrapyDeprecationWarning ) settings.setdict(scrapy_envvars, priority='project') diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 1f2333264..5703fd4c3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -91,7 +91,7 @@ def to_unicode(text, encoding=None, errors='strict'): return text if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.decode(encoding, errors) @@ -104,7 +104,7 @@ def to_bytes(text, encoding=None, errors='strict'): return text if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' - 'object, got %s' % type(text).__name__) + f'object, got {type(text).__name__}') if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) @@ -174,7 +174,7 @@ def binary_is_text(data): does not contain unprintable control characters. """ if not isinstance(data, bytes): - raise TypeError("data must be bytes, got '%s'" % type(data).__name__) + raise TypeError(f"data must be bytes, got '{type(data).__name__}'") return all(c not in _BINARYCHARS for c in data) @@ -217,7 +217,7 @@ def get_func_args(func, stripself=False): else: return get_func_args(func.__call__, True) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') if stripself: func_args.pop(0) return func_args @@ -250,7 +250,7 @@ def get_spec(func): elif hasattr(func, '__call__'): spec = _getargspec_py23(func.__call__) else: - raise TypeError('%s is not callable' % type(func)) + raise TypeError(f'{type(func)} is not callable') defaults = spec.defaults or [] @@ -322,7 +322,7 @@ def global_object_name(obj): >>> global_object_name(Request) 'scrapy.http.request.Request' """ - return "%s.%s" % (obj.__module__, obj.__name__) + return f"{obj.__module__}.{obj.__name__}" if hasattr(sys, "pypy_version_info"): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 879d27907..831d29462 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,7 +10,7 @@ def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor if len(portrange) > 2: - raise ValueError("invalid portrange: %s" % portrange) + raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) if not hasattr(portrange, '__iter__'): @@ -78,9 +78,9 @@ def verify_installed_reactor(reactor_path): from twisted.internet import reactor reactor_class = load_object(reactor_path) if not isinstance(reactor, reactor_class): - msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( - reactor.__module__, reactor.__class__.__name__, reactor_path - ) + msg = ("The installed reactor " + f"({reactor.__module__}.{reactor.__class__.__name__}) does not " + f"match the requested one ({reactor_path})") raise Exception(msg) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 503d7b133..d38b1bc4d 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -84,7 +84,7 @@ def _find_method(obj, func): # https://docs.python.org/3/reference/datamodel.html if obj_func.__func__ is func.__func__: return name - raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) + raise ValueError(f"Function {func} is not an instance method in: {obj}") def _get_method(obj, name): @@ -92,4 +92,4 @@ def _get_method(obj, name): try: return getattr(obj, name) except AttributeError: - raise ValueError("Method %r not found in: %s" % (name, obj)) + raise ValueError(f"Method {name!r} not found in: {obj}") diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c29b619ce..99b089b6f 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -39,7 +39,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_unicode(message)) + return f'{status} {to_unicode(message)}' def response_httprepr(response): @@ -69,15 +69,15 @@ def open_in_browser(response, _openfunc=webbrowser.open): body = response.body if isinstance(response, HtmlResponse): if b'' body = body.replace(b'', to_bytes(repl)) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' else: - raise TypeError("Unsupported response type: %s" % - response.__class__.__name__) + raise TypeError("Unsupported response type: " + f"{response.__class__.__name__}") fd, fname = tempfile.mkstemp(ext) os.write(fd, body) os.close(fd) - return _openfunc("file://%s" % fname) + return _openfunc(f"file://{fname}") diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index cc3263602..a73cf03c5 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -17,7 +17,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): if isinstance(o, set): return list(o) elif isinstance(o, datetime.datetime): - return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) + return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") elif isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) elif isinstance(o, datetime.time): @@ -29,9 +29,9 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif is_item(o): return ItemAdapter(o).asdict() elif isinstance(o, Request): - return "<%s %s %s>" % (type(o).__name__, o.method, o.url) + return f"<{type(o).__name__} {o.method} {o.url}>" elif isinstance(o, Response): - return "<%s %s %s>" % (type(o).__name__, o.status, o.url) + return f"<{type(o).__name__} {o.status} {o.url}>" else: return super().default(o) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index c3c5e329b..ea4dde882 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -50,7 +50,7 @@ def get_temp_key_info(ssl_object): key_info.append(ffi_buf_to_string(cname)) else: key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) - key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) + key_info.append(f'{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits') return ', '.join(key_info) @@ -58,4 +58,4 @@ def get_openssl_version(): system_openssl = OpenSSL.SSL.SSLeay_version( OpenSSL.SSL.SSLEAY_VERSION ).decode('ascii', errors='replace') - return '{} ({})'.format(OpenSSL.version.__version__, system_openssl) + return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 7442a2f33..f54942ffb 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -79,7 +79,7 @@ def get_ftp_content_and_delete( def buffer_data(data): ftp_data.append(data) - ftp.retrbinary('RETR %s' % path, buffer_data) + ftp.retrbinary(f'RETR {path}', buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index a63c9a942..a54c7db95 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -23,10 +23,10 @@ class ProcessTest: def _process_finished(self, pp, cmd, check_code): if pp.exitcode and check_code: - msg = "process %s exit with code %d" % (cmd, pp.exitcode) - msg += "\n>>> stdout <<<\n%s" % pp.out + msg = f"process {cmd} exit with code {pp.exitcode}" + msg += f"\n>>> stdout <<<\n{pp.out}" msg += "\n" - msg += "\n>>> stderr <<<\n%s" % pp.err + msg += f"\n>>> stderr <<<\n{pp.err}" raise RuntimeError(msg) return pp.exitcode, pp.out, pp.err diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 397e54703..fce77be32 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -9,7 +9,7 @@ class SiteTest: from twisted.internet import reactor super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - self.baseurl = "http://localhost:%d/" % self.site.getHost().port + self.baseurl = f"http://localhost:{self.site.getHost().port}/" def tearDown(self): super().tearDown() @@ -40,5 +40,5 @@ def test_site(): if __name__ == '__main__': from twisted.internet import reactor port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") - print("http://localhost:%d/" % port.getHost().port) + print(f"http://localhost:{port.getHost().port}/") reactor.run() diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index baed5c536..3e40acd69 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -41,9 +41,7 @@ def format_live_refs(ignore=NoneType): if issubclass(cls, ignore): continue oldest = min(wdict.values()) - s += "%-30s %6d oldest: %ds ago\n" % ( - cls.__name__, len(wdict), now - oldest - ) + s += f"{cls.__name__:<30} {len(wdict):6} oldest: {int(now - oldest)}s ago\n" return s diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b23ddb459..a6a2a9e8b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains): if not host: return False domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith('.%s' % d)) for d in domains) + return any((host == d) or (host.endswith(f'.{d}')) for d in domains) def url_is_from_spider(url, spider): @@ -153,7 +153,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443), ('ftp', 21)): - netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + netloc = netloc.replace(f':{parsed_url.port}', '') return urlunparse(( parsed_url.scheme, netloc, diff --git a/sep/sep-002.rst b/sep/sep-002.rst index c467cb402..2e8a28340 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError("Expected iterable, got %s" % type(value).__name__) + raise TypeError(f"Expected iterable, got {type(value).__name__}") def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index 05b0eb99c..b9f5e556f 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print "%d items scraped" % len(scraped_items) + print(f"{len(scraped_items)} items scraped") # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 8ca81824d..4e3340521 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given %r in %r' \ - % (rule.matcher, rule)) + raise ValueError('Not valid matcher given ' + f'{rule.matcher!r} in {rule!r}') # prepare callback if callable(rule.callback): @@ -553,8 +553,7 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError('Invalid callback %r can not be resolved' \ - % callback) + raise AttributeError(f'Invalid callback {callback!r} can not be resolved') else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index fe707923a..d0169b81e 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) + robotsurl = f"{url.scheme}://{netloc}/robots.txt" meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = r'^(.*\.)?(%s)$' % '|'.join(domains) + regex = fr'^(.*\.)?({"|".join(domains)})$' return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 3f9738798..f545de39f 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -38,7 +38,7 @@ class LocalhostSpider(Spider): if __name__ == "__main__": with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: port = urlparse(mock_http_server.http_address).port - url = "http://not.a.real.domain:{port}/echo".format(port=port) + url = f"http://not.a.real.domain:{port}/echo" servers = [(mock_dns_server.host, mock_dns_server.port)] reactor.installResolver(createResolver(servers=servers)) diff --git a/tests/mockserver.py b/tests/mockserver.py index 6f0c274b9..ab9aec6a6 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -73,7 +73,7 @@ class Follow(LeafResource): for nl in nlist: args[b"n"] = [to_bytes(str(nl))] argstr = urlencode(args, doseq=True) - s += "follow %d
    " % (argstr, nl) + s += f"follow {nl}
    " s += """""" request.write(to_bytes(s)) request.finish() @@ -91,7 +91,7 @@ class Delay(LeafResource): return NOT_DONE_YET def _delayedRender(self, request, n): - request.write(to_bytes("Response delayed for %0.3f seconds\n" % n)) + request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n")) request.finish() @@ -310,8 +310,8 @@ if __name__ == "__main__": def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) - httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + httpAddress = f'http://{httpHost.host}:{httpHost.port}' + httpsAddress = f'https://{httpsHost.host}:{httpsHost.port}' print(httpAddress) print(httpsAddress) @@ -323,7 +323,7 @@ if __name__ == "__main__": def print_listening(): host = listener.getHost() - print("%s:%s" % (host.host, host.port)) + print(f"{host.host}:{host.port}") reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py index 162a53760..e49340284 100644 --- a/tests/py36/_test_crawl.py +++ b/tests/py36/_test_crawl.py @@ -33,7 +33,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), + return Request(self.mockserver.url(f"/status?n=200&request={index}"), meta={'index': index}, dont_filter=True, callback=cb) diff --git a/tests/spiders.py b/tests/spiders.py index 63bd726fb..3e525e62f 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -45,7 +45,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") self.start_urls = [url] def parse(self, response): @@ -67,7 +67,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -192,7 +192,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) + url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=1)}") yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -239,7 +239,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = self.mockserver.url("/echo?headers=1&body=test%d" % i) + url = self.mockserver.url(f"/echo?headers=1&body=test{i}") yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 6504b4d2c..005e45214 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -4,7 +4,7 @@ class TestExtension: def __init__(self, settings): - settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) + settings.set('TEST1', f"{settings['TEST1']} + started") @classmethod def from_crawler(cls, crawler): diff --git a/tests/test_command_check.py b/tests/test_command_check.py index f27f526a3..34f5e59dd 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(""" + file.write(f""" import scrapy class CheckSpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://example.com - {1} + {contracts} \"\"\" - {2} - """.format(self.spider_name, contracts, parse_def)) + {parse_def} + """) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index e115f420f..ed3848d88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -21,14 +21,14 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: - f.write(""" + f.write(f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule class MySpider(scrapy.Spider): - name = '{0}' + name = '{self.spider_name}' def parse(self, response): if getattr(self, 'test_arg', None): @@ -58,7 +58,7 @@ class MySpider(scrapy.Spider): self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): - name = 'goodcrawl{0}' + name = 'goodcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -74,7 +74,7 @@ class MyGoodCrawlSpider(CrawlSpider): class MyBadCrawlSpider(CrawlSpider): '''Spider which doesn't define a parse_item callback while using it in a rule.''' - name = 'badcrawl{0}' + name = 'badcrawl{self.spider_name}' rules = ( Rule(LinkExtractor(allow=r'/html'), callback='parse_item', follow=True), @@ -82,7 +82,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""".format(self.spider_name)) +""") fname = abspath(join(self.proj_mod_path, 'pipelines.py')) with open(fname, 'w') as f: @@ -99,9 +99,9 @@ class MyPipeline: fname = abspath(join(self.proj_mod_path, 'settings.py')) with open(fname, 'a') as f: - f.write(""" -ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} -""" % self.project_name) + f.write(f""" +ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} +""") @defer.inlineCallbacks def test_spider_arguments(self): diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 66c293c00..16c9559b5 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -65,8 +65,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}')" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}')" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Redirecting (302)' in errout assert b'Crawled (200)' in errout @@ -75,23 +75,23 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url('/redirect-no-meta-refresh') - code = "fetch('{0}', redirect=False)" - errcode, out, errout = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) assert b'Crawled (302)' in errout @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') - code = "fetch('{0}') or fetch(response.request.replace(method='POST'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks def test_scrapy_import(self): url = self.url('/text') - code = "fetch(scrapy.Request('{0}'))" - errcode, out, _ = yield self.execute(['-c', code.format(url)]) + code = f"fetch(scrapy.Request('{url}'))" + errcode, out, _ = yield self.execute(['-c', code]) self.assertEqual(errcode, 0, out) @defer.inlineCallbacks diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 99c01c2b7..00d998388 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -16,7 +16,7 @@ class VersionTest(ProcessTest, unittest.TestCase): _, out, _ = yield self.execute([]) self.assertEqual( out.strip().decode(encoding), - "Scrapy %s" % scrapy.__version__, + f"Scrapy {scrapy.__version__}", ) @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index ee8a92604..5faaca738 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -42,7 +42,7 @@ class CommandSettings(unittest.TestCase): def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py']) + opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) @@ -163,10 +163,10 @@ class StartprojectTemplatesTest(ProjectTest): pass assert exists(join(self.tmpl_proj, 'root_template')) - args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl] + args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] p, out, err = self.proc('startproject', self.project_name, *args) - self.assertIn("New Scrapy project '%s', using template directory" - % self.project_name, out) + self.assertIn(f"New Scrapy project '{self.project_name}', " + "using template directory", out) self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) @@ -247,7 +247,7 @@ class StartprojectTemplatesTest(ProjectTest): 'startproject', project_name, '--set', - 'TEMPLATES_DIR={}'.format(read_only_templates_dir), + f'TEMPLATES_DIR={read_only_templates_dir}', ), cwd=destination, env=self.env, @@ -320,7 +320,7 @@ class CommandTest(ProjectTest): super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) - self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name + self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' class GenspiderCommandTest(CommandTest): @@ -334,14 +334,14 @@ class GenspiderCommandTest(CommandTest): assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) def test_template(self, tplname='crawl'): - args = ['--template=%s' % tplname] if tplname else [] + args = [f'--template={tplname}'] if tplname else [] spname = 'test_spider' p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out) + self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out) self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) p, out, err = self.proc('genspider', spname, 'test.com', *args) - self.assertIn("Spider %r already exists in module" % spname, out) + self.assertIn(f"Spider {spname!r} already exists in module", out) modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) self.assertEqual(modify_time_after, modify_time_before) @@ -363,11 +363,11 @@ class GenspiderCommandTest(CommandTest): def test_same_name_as_project(self): self.assertEqual(2, self.call('genspider', self.project_name)) - assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name)) + assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) def test_same_filename_as_existing_spider(self, force=False): file_name = 'example' - file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name) + file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') self.assertEqual(0, self.call('genspider', file_name, 'example.com')) assert exists(file_path) @@ -383,14 +383,14 @@ class GenspiderCommandTest(CommandTest): if force: p, out, err = self.proc('genspider', '--force', file_name, 'example.com') - self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) modify_time_after = getmtime(file_path) self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("%s already exists" % (file_path), out) + self.assertIn(f"{file_path} already exists", out) modify_time_after = getmtime(file_path) self.assertEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() @@ -410,7 +410,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): file_name = 'example' file_path = join(self.temp_path, file_name + '.py') p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("Created spider %r using template \'basic\' " % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) assert exists(file_path) modify_time_before = getmtime(file_path) file_contents_before = open(file_path, 'r').read() @@ -418,14 +418,14 @@ class GenspiderStandaloneCommandTest(ProjectTest): if force: # use different template to ensure contents were changed p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') - self.assertIn("Created spider %r using template \'crawl\' " % file_name, out) + self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) modify_time_after = getmtime(file_path) self.assertNotEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out) + self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) modify_time_after = getmtime(file_path) self.assertEqual(modify_time_after, modify_time_before) file_contents_after = open(file_path, 'r').read() diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 2e7e3ccc4..d0f4a68c2 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -393,7 +393,7 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + contract_doc = f'@url {mockserver.url("/status?n=200")}' TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..e703f45de 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): total_time = times[-1] - times[0] average = total_time / (len(times) - 1) self.assertTrue(average > delay * tolerance, - "download delay too small: %s" % average) + f"download delay too small: {average}") # Ensure that the same test parameters would cause a failure if no # download delay is set. Otherwise, it means we are using a combination @@ -204,7 +204,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) + yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks @@ -465,7 +465,7 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) for req_id in range(3): - self.assertIn("Got response 200, req_id %d" % req_id, str(log)) + self.assertIn(f"Got response 200, req_id {req_id}", str(log)) @defer.inlineCallbacks def test_response_ssl_certificate_none(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7059f0892..0d3c42797 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -121,7 +121,7 @@ class FileTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_non_existent(self): - request = Request('file://%s' % self.mktemp()) + request = Request(f'file://{self.mktemp()}') d = self.download_request(request, Spider('foo')) return self.assertFailure(d, IOError) @@ -249,7 +249,7 @@ class HttpTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -300,7 +300,7 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): self.assertEqual( - response.body, to_bytes('%s:%d' % (self.host, self.portno))) + response.body, to_bytes(f'{self.host}:{self.portno}')) self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) @@ -583,7 +583,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + return f"{self.scheme}://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL('file')) @@ -678,7 +678,7 @@ class HttpProxyTestCase(unittest.TestCase): yield self.download_handler.close() def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def test_download_with_proxy(self): def _test(response): @@ -696,7 +696,7 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEqual(response.url, request.url) self.assertEqual(response.body, b'https://example.com') - http_proxy = '%s?noconnect' % self.getURL('') + http_proxy = f'{self.getURL("")}?noconnect' request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): @@ -977,7 +977,7 @@ class BaseFTPTestCase(unittest.TestCase): return deferred def test_ftp_download_success(self): - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -989,7 +989,7 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_download_path_with_spaces(self): request = Request( - url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, + url=f"ftp://127.0.0.1:{self.portNum}/file with spaces.txt", meta=self.req_meta ) d = self.download_handler.download_request(request, None) @@ -1001,7 +1001,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): - request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/notexist.txt", meta=self.req_meta) d = self.download_handler.download_request(request, None) @@ -1015,7 +1015,7 @@ class BaseFTPTestCase(unittest.TestCase): os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) @@ -1037,7 +1037,7 @@ class FTPTestCase(BaseFTPTestCase): meta = dict(self.req_meta) meta.update({"ftp_password": 'invalid'}) - request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index a9190c62b..79f24c8a1 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -84,7 +84,7 @@ class DefaultsTest(ManagerTestCase): }) ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), - "Not redirected: {0!r}".format(ret)) + f"Not redirected: {ret!r}") self.assertEqual(to_bytes(ret.url), resp.headers['Location'], "Not redirected to location header") diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index dbae4d3ae..b2b5ce77d 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -28,7 +28,7 @@ class DecompressionMiddlewareTest(TestCase): for fmt in self.test_formats: rsp = self.test_responses[fmt] new = self.mw.process_response(None, rsp, self.spider) - error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__) + error_msg = f'Failed {fmt}, response type {type(new).__name__}' assert isinstance(new, XmlResponse), error_msg assert_samelines(self, new.body, self.uncompressed_body, fmt) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 299fb0eb8..0c6dcf2aa 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -324,7 +324,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (shouldcache, status, headers) in enumerate(responses): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -343,7 +343,7 @@ class RFC2616PolicyTest(DefaultStorageTest): with self._middleware(HTTPCACHE_ALWAYS_STORE=True) as mw: for idx, (_, status, headers) in enumerate(responses): shouldcache = 'no-store' not in headers.get('Cache-Control', '') and status != 304 - req0 = Request('http://example2-%d.com' % idx) + req0 = Request(f'http://example2-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) res1 = self._process_requestresponse(mw, req0, res0) res304 = res0.replace(status=304) @@ -386,7 +386,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) @@ -423,7 +423,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware() as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0a = Response(req0.url, status=status, headers=headers) # cache expired response res1 = self._process_requestresponse(mw, req0, res0a) @@ -490,7 +490,7 @@ class RFC2616PolicyTest(DefaultStorageTest): ] with self._middleware(HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS=['no-cache', 'no-store']) as mw: for idx, (status, headers) in enumerate(sampledata): - req0 = Request('http://example-%d.com' % idx) + req0 = Request(f'http://example-{idx}.com') res0 = Response(req0.url, status=status, headers=headers) # cache fresh response res1 = self._process_requestresponse(mw, req0, res0) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 131332131..816ac1440 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,7 +22,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_redirect_3xx_permanent(self): def _test(method, status=301): - url = 'http://www.example.com/{}'.format(status) + url = f'http://www.example.com/{status}' url2 = 'http://www.example.com/redirected' req = Request(url, method=method) rsp = Response(url, headers={'Location': url2}, status=status) @@ -79,7 +79,7 @@ class RedirectMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" # response without Location header but with status code is 3XX should be ignored del rsp.headers['Location'] @@ -207,8 +207,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = """""" - return html.format(interval, url).encode('utf-8') + html = f"""""" + return html.encode('utf-8') def test_priority_adjust(self): req = Request('http://a.com') @@ -243,7 +243,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.method, 'GET') assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" - assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body + assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" def test_max_redirect_times(self): self.mw.max_redirect_times = 1 diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 29357ba94..364ce0c89 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -94,7 +94,7 @@ class RetryTest(unittest.TestCase): ] for exc in exceptions: - req = Request('http://www.scrapytest.org/%s' % exc.__name__) + req = Request(f'http://www.scrapytest.org/{exc.__name__}') self._test_retry_exception(req, exc('foo')) stats = self.crawler.stats diff --git a/tests/test_engine.py b/tests/test_engine.py index 1b848ac72..3629aa1aa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -127,8 +127,8 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." - % port.getHost().port) + print(f"Test server running at http://localhost:{port.getHost().port}/ " + "- hit Ctrl-C to finish.") return port @@ -185,7 +185,7 @@ class CrawlerRun: self.deferred.callback(None) def geturl(self, path): - return "http://localhost:%s%s" % (self.portno, path) + return f"http://localhost:{self.portno}{path}" def getpath(self, url): u = urlparse(url) @@ -265,7 +265,7 @@ class EngineTest(unittest.TestCase): "/item1.html", "/item2.html", "/item999.html"] urls_visited = {rp[0].url for rp in self.run.respplug} urls_expected = {self.run.geturl(p) for p in must_be_visited} - assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) + assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" def _assert_scheduled_requests(self, urls_to_visit=None): self.assertEqual(urls_to_visit, len(self.run.reqplug)) @@ -413,16 +413,19 @@ class StopDownloadEngineTest(EngineTest): yield self.run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - "Download stopped for from signal handler" - " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + f"Download stopped for " + "from signal handler" + " StopDownloadCrawlerRun.bytes_received")) self._assert_visited_urls() self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 850485b5e..94568581a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -184,8 +184,7 @@ class FTPFeedStorageTest(unittest.TestCase): def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe='') - st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted, - {}) + st = FTPFeedStorage(f'ftp://foo:{pw_quoted}@example.com/some_path', {}) self.assertEqual(st.password, string.punctuation) @@ -1230,7 +1229,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Stored %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Stored {fmt} feed (2 items)', str(log)) @defer.inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): @@ -1251,7 +1250,7 @@ class FeedExportTest(FeedExportTestBase): print(log) for fmt in ['json', 'xml', 'csv']: - self.assertIn('Error storing %s feed (2 items)' % fmt, str(log)) + self.assertIn(f'Error storing {fmt} feed (2 items)', str(log)) class BatchDeliveriesTest(FeedExportTestBase): @@ -1582,10 +1581,8 @@ class BatchDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) - prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( - bucket_name=s3_test_bucket_name, prefix=prefix - ) + prefix = f'tmp/{filename}' + s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json' storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) settings = Settings({ 'FEEDS': { diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 624dd9ab8..41afa2896 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -657,7 +657,7 @@ class SelectJmesTestCase(unittest.TestCase): self.assertEqual( test, expected, - msg='test "{}" got {} expected {}'.format(tl, test, expected) + msg=f'test "{tl}" got {test} expected {expected}' ) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 41ff3651d..dc5be398f 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -20,7 +20,7 @@ class CustomItem(Item): name = Field() def __str__(self): - return "name: %s" % self['name'] + return f"name: {self['name']}" class LogFormatterTestCase(unittest.TestCase): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b2b75ef20..e3e46db07 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [f'tests.test_middleware.{x}' for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): super()._add_middleware(mw) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9af5affec..55fcfa7ba 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -123,10 +123,10 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) - self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3) # check that logs do show the failure on the file downloads - file_dl_failure = 'File (code: %d): Error downloading file from' % code + file_dl_failure = f'File (code: {code}): Error downloading file from' self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index b19b4ff2a..1dd7031fe 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -167,7 +167,7 @@ class FilesPipelineTestCase(unittest.TestCase): """ class CustomFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, item=None): - return 'full/%s' % item.get('path') + return f'full/{item.get("path")}' file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path item = dict(path='path-to-store-file') @@ -495,7 +495,7 @@ class TestFTPFileStore(unittest.TestCase): self.assertIn('last_modified', stat) self.assertIn('checksum', stat) self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6') - path = '%s/%s' % (store.basedir, path) + path = f'{store.basedir}/{path}' content = get_ftp_content_and_delete( path, store.host, store.port, store.username, store.password, store.USE_ACTIVE_MODE) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 082e9ee21..ad138a2dc 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -128,11 +128,11 @@ class DeprecatedImagesPipeline(ImagesPipeline): def image_key(self, url): image_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'empty/%s.jpg' % (image_guid) + return f'empty/{image_guid}.jpg' def thumb_key(self, url, thumb_id): thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() - return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) + return f'thumbsup/{thumb_id}/{thumb_guid}.jpg' class ImagesPipelineTestCaseFieldsMixin: diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..d3f58634e 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -37,14 +37,14 @@ sys.exit(mitmdump()) '-c', script, '--listen-host', '127.0.0.1', '--listen-port', '0', - '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--proxyauth', f'{self.auth_user}:{self.auth_pass}', '--certs', cert_path, '--ssl-insecure', ], stdout=PIPE, env=get_testenv()) line = self.proc.stdout.readline().decode('utf-8') host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) - address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}' return address def stop(self): @@ -118,7 +118,7 @@ class ProxyConnectTestCase(TestCase): def _assert_got_response_code(self, code, log): print(log) - self.assertEqual(str(log).count('Crawled (%d)' % code), 1) + self.assertEqual(str(log).count(f'Crawled ({code})'), 1) def _assert_got_tunnel_error(self, log): print(log) diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index b60b7c579..907117468 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_response_error(self): for status in ("404", "500"): - url = self.mockserver.url("/status?n={}".format(status)) + url = self.mockserver.url(f"/status?n={status}") crawler = CrawlerRunner().create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] @@ -135,7 +135,7 @@ class CrawlTestCase(TestCase): self.assertEqual(signal_params["request"].url, OVERRIDEN_URL) log.check_present( - ("scrapy.core.engine", "DEBUG", "Crawled (200) (referer: None)".format(OVERRIDEN_URL)), + ("scrapy.core.engine", "DEBUG", f"Crawled (200) (referer: None)"), ) @defer.inlineCallbacks diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index a175f88ca..c07d3a99c 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -17,7 +17,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_filename(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_disposition(self): mappings = [ @@ -32,7 +32,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_content_type(self): mappings = [ @@ -47,7 +47,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_body(self): mappings = [ @@ -58,7 +58,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_body(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_headers(self): mappings = [ @@ -70,7 +70,7 @@ class ResponseTypesTest(unittest.TestCase): for source, cls in mappings: source = Headers(source) retcls = responsetypes.from_headers(source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_from_args(self): # TODO: add more tests that check precedence between the different arguments @@ -86,7 +86,7 @@ class ResponseTypesTest(unittest.TestCase): ] for source, cls in mappings: retcls = responsetypes.from_args(**source) - assert retcls is cls, "%s ==> %s != %s" % (source, retcls, cls) + assert retcls is cls, f"{source} ==> {retcls} != {cls}" def test_custom_mime_types_loaded(self): # check that mime.types files shipped with scrapy are loaded diff --git a/tests/test_selector.py b/tests/test_selector.py index 62036ad8c..cff8d0393 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -88,7 +88,7 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__ + assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__" def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): diff --git a/tests/test_signals.py b/tests/test_signals.py index d6ae526be..a43f00b27 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -13,7 +13,7 @@ class ItemSpider(Spider): def start_requests(self): for index in range(10): - yield Request(self.mockserver.url('/status?n=200&id=%d' % index), + yield Request(self.mockserver.url(f'/status?n=200&id={index}'), meta={'index': index}) def parse(self, response): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 79eda35b3..2f454addc 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -163,11 +163,11 @@ class GeneratorOutputChainSpider(Spider): class _GeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -175,12 +175,12 @@ class _GeneratorDoNothingMiddleware: class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r raise LookupError() def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -192,11 +192,11 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') yield r def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -229,12 +229,12 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -243,13 +243,13 @@ class NotGeneratorFailMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) raise ReferenceError() return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] @@ -262,12 +262,12 @@ class NotGeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): out = [] for r in result: - r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + r['processed'].append(f'{self.__class__.__name__}.process_spider_output') out.append(r) return out def process_spider_exception(self, response, exception, spider): - method = '{}.process_spider_exception'.format(self.__class__.__name__) + method = f'{self.__class__.__name__}.process_spider_exception' spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 6b05c8771..f5d684d3f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -16,7 +16,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): try: Request(**result) except TypeError as e: - self.fail("Request kwargs are not correct {}".format(e)) + self.fail(f"Request kwargs are not correct {e}") def test_get(self): curl_command = "curl http://example.org/" diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index aa18ef1f3..e4bccf30e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -299,7 +299,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): cache = LocalWeakReferencedCache() refs = [] for x in range(max): - refs.append(Request('https://example.org/{}'.format(x))) + refs.append(Request(f'https://example.org/{x}')) cache[refs[-1]] = x self.assertEqual(len(cache), max) for i, r in enumerate(refs): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 8c84331b9..e60242a3b 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -40,15 +40,15 @@ class MustbeDeferredTest(unittest.TestCase): def cb1(value, arg1, arg2): - return "(cb1 %s %s %s)" % (value, arg1, arg2) + return f"(cb1 {value} {arg1} {arg2})" def cb2(value, arg1, arg2): - return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + return defer.succeed(f"(cb2 {value} {arg1} {arg2})") def cb3(value, arg1, arg2): - return "(cb3 %s %s %s)" % (value, arg1, arg2) + return f"(cb3 {value} {arg1} {arg2})" def cb_fail(value, arg1, arg2): @@ -56,7 +56,7 @@ def cb_fail(value, arg1, arg2): def eb1(failure, arg1, arg2): - return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) + return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" class DeferUtilsTest(unittest.TestCase): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 50190d4d1..79f5a2bbe 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -409,7 +409,7 @@ class TestHelper(unittest.TestCase): def _assert_type_and_value(self, a, b, obj): self.assertTrue(type(a) is type(b), - 'Got {}, expected {} for {!r}'.format(type(a), type(b), obj)) + f'Got {type(a)}, expected {type(b)} for { obj!r}') self.assertEqual(a, b) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 2f885a0e8..144c7bd76 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -213,7 +213,7 @@ def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1]) + f'Wrong scheme guessed: for `{args[0]}` got `{url}`, expected `{args[1]}...`' return do_expected @@ -254,7 +254,7 @@ for k, args in enumerate( start=1, ): t_method = create_guess_scheme_t(args) - t_method.__name__ = 'test_uri_%03d' % k + t_method.__name__ = f'test_uri_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation @@ -269,7 +269,7 @@ for k, args in enumerate( start=1, ): t_method = create_skipped_scheme_t(args) - t_method.__name__ = 'test_uri_skipped_%03d' % k + t_method.__name__ = f'test_uri_skipped_{k:03}' setattr(GuessSchemeTest, t_method.__name__, t_method) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index ee64d455c..a60181a3a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -253,7 +253,7 @@ class WebClientTestCase(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return "http://127.0.0.1:%d/%s" % (self.portno, path) + return f"http://127.0.0.1:{self.portno}/{path}" def testPayload(self): s = "0123456789" * 10 @@ -265,7 +265,7 @@ class WebClientTestCase(unittest.TestCase): # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( self.assertEqual, to_bytes("www.example.com"))]) @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}")) return d def test_timeoutTriggering(self): @@ -376,7 +376,7 @@ class WebClientSSLTestCase(unittest.TestCase): interface="127.0.0.1") def getURL(self, path): - return "https://127.0.0.1:%d/%s" % (self.portno, path) + return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): self.tmpname = self.mktemp() From 560c335c0782ec9a834a83abfa9dfbaa8fd67fed Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:00:51 +0300 Subject: [PATCH 102/119] Add errors parameter in documentation. --- docs/topics/exporters.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 11ef5b2a6..3f8906326 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -292,7 +292,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs) Exports items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -311,6 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str + :param errors: The optional string that specifies how encoding and decoding errors are to be handled. + For more information see `documentation `_. + :type errors: str + The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the :func:`csv.writer` function, so you can use any :func:`csv.writer` function From cf50561b8696b645940ae33ba9b168ef97580477 Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 26 Aug 2020 11:08:14 +0000 Subject: [PATCH 103/119] Allow passing classes directly in Settings (#3873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 26 +++++++++++++++++++++ scrapy/utils/deprecate.py | 2 +- scrapy/utils/misc.py | 14 +++++++++-- tests/test_crawler.py | 2 +- tests/test_downloader_handlers.py | 6 ++--- tests/test_feedexport.py | 22 +++++++++--------- tests/test_middleware.py | 2 +- tests/test_settings/__init__.py | 32 ++++++++++++++++++++++++++ tests/test_spidermiddleware_referer.py | 2 +- tests/test_utils_misc/__init__.py | 15 ++++++++++-- 10 files changed, 101 insertions(+), 22 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 618b9989e..2924c0566 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,32 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. + +Import paths and classes +======================== + +.. versionadded:: VERSION + +When a setting references a callable object to be imported by Scrapy, such as a +class or a function, there are two different ways you can specify that object: + +- As a string containing the import path of that object + +- As the object itself + +For example:: + + from mybot.pipelines.validate import ValidateMyItem + ITEM_PIPELINES = { + # passing the classname... + ValidateMyItem: 300, + # ...equals passing the class path + 'mybot.pipelines.validate.ValidateMyItem': 300, + } + +.. note:: Passing non-callable objects is not supported. + + How to access settings ====================== diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3c8e3c8b5..8277a3c8f 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -135,7 +135,7 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: - if path.startswith(prefix): + if isinstance(path, str) and path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path), ScrapyDeprecationWarning) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index bd400bd30..d5d1f301c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -39,10 +39,20 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be the import path of a class, function, variable or an - instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' + The object can be the import path of a class, function, variable or an + instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'. + + If ``path`` is not a string, but is a callable object, such as a class or + a function, then return it as is. """ + if not isinstance(path, str): + if callable(path): + return path + else: + raise TypeError("Unexpected argument type, expected string " + "or object, got: %s" % type(path)) + try: dot = path.rindex('.') except ValueError: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7c2e251a9..85035a220 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -142,7 +142,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): def test_spider_manager_verify_interface(self): settings = Settings({ - 'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface' + 'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface, }) with warnings.catch_warnings(record=True) as w: self.assertRaises(AttributeError, CrawlerRunner, settings) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7059f0892..e50bdb391 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -61,7 +61,7 @@ class OffDH: class LoadTestCase(unittest.TestCase): def test_enabled_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'} + handlers = {'scheme': DummyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -69,7 +69,7 @@ class LoadTestCase(unittest.TestCase): self.assertNotIn('scheme', dh._notconfigured) def test_not_configured_handler(self): - handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'} + handlers = {'scheme': OffDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) @@ -87,7 +87,7 @@ class LoadTestCase(unittest.TestCase): self.assertIn('scheme', dh._notconfigured) def test_lazy_handlers(self): - handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + handlers = {'scheme': DummyLazyDH} crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 850485b5e..689d25fef 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -845,7 +845,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'}, + 'FEED_STORAGES': {'file': LogOnStoreFileStorage}, 'FEED_STORE_EMPTY': False } @@ -1189,8 +1189,8 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { - 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'}, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'}, + 'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter}, + 'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage}, 'FEEDS': { self._random_temp_filename(): {'format': 'csv'}, }, @@ -1219,7 +1219,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': DummyBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1240,7 +1240,7 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {'format': 'xml'}, self._random_temp_filename(): {'format': 'csv'}, }, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'}, + 'FEED_STORAGES': {'file': FailingBlockingFeedStorage}, } items = [ {'foo': 'bar1', 'baz': ''}, @@ -1667,7 +1667,7 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions' + 'file': StdoutFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1708,7 +1708,7 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions' + 'file': FileFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1756,7 +1756,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions' + 'file': S3FeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1784,7 +1784,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler' + 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1833,7 +1833,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions' + 'file': FTPFeedStorageWithoutFeedOptions }, } crawler = get_crawler(settings_dict=settings_dict) @@ -1861,7 +1861,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): settings_dict = { 'FEED_URI': 'file:///tmp/foobar', 'FEED_STORAGES': { - 'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler' + 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler }, } crawler = get_crawler(settings_dict=settings_dict) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b2b75ef20..8651431b5 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] + return [M1, MOff, M3] def _add_middleware(self, mw): super()._add_middleware(mw) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 6e56a28f5..916fe012a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -385,6 +385,38 @@ class SettingsTest(unittest.TestCase): self.assertIn('key', mydict) self.assertEqual(mydict['key'], 'val') + def test_passing_objects_as_values(self): + from scrapy.core.downloader.handlers.file import FileDownloadHandler + from scrapy.utils.misc import create_instance + from scrapy.utils.test import get_crawler + + class TestPipeline(): + def process_item(self, i, s): + return i + + settings = Settings({ + 'ITEM_PIPELINES': { + TestPipeline: 800, + }, + 'DOWNLOAD_HANDLERS': { + 'ftp': FileDownloadHandler, + }, + }) + + self.assertIn('ITEM_PIPELINES', settings.attributes) + + mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem() + self.assertEqual(priority, 800) + self.assertEqual(mypipeline, TestPipeline) + self.assertIsInstance(mypipeline(), TestPipeline) + self.assertEqual(mypipeline().process_item('item', None), 'item') + + myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp') + self.assertEqual(myhandler, FileDownloadHandler) + myhandler_instance = create_instance(myhandler, None, get_crawler()) + self.assertIsInstance(myhandler_instance, FileDownloadHandler) + self.assertTrue(hasattr(myhandler_instance, 'download_request')) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 5141f47af..9456b01d4 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -385,7 +385,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': CustomPythonOrgPolicy} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 9bb996d27..e95a3a316 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -12,11 +12,22 @@ __doctests__ = ['scrapy.utils.misc'] class UtilsMiscTestCase(unittest.TestCase): - def test_load_object(self): + def test_load_object_class(self): + obj = load_object(Field) + self.assertIs(obj, Field) + obj = load_object('scrapy.item.Field') + self.assertIs(obj, Field) + + def test_load_object_function(self): + obj = load_object(load_object) + self.assertIs(obj, load_object) obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + self.assertRaises(TypeError, load_object, dict()) def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') From 29725e4b58bfe417388b5ce2c2b4f1c587a465da Mon Sep 17 00:00:00 2001 From: maranqz Date: Wed, 26 Aug 2020 14:38:37 +0300 Subject: [PATCH 104/119] Fix tabulation of rst and change documentation link. --- docs/topics/exporters.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 3f8906326..0203def74 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -311,9 +311,10 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str - :param errors: The optional string that specifies how encoding and decoding errors are to be handled. - For more information see `documentation `_. - :type errors: str + :param errors: The optional string that specifies how encoding and decoding + errors are to be handled. For more information see + :class:`io.TextIOWrapper`. + :type errors: str The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the From c77450990d9f78e68380d724b4a8c15e7b99c995 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:41:24 +0000 Subject: [PATCH 105/119] Update scrapy/commands/version.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/commands/version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index dc8087043..1237610cb 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,7 @@ class Command(ScrapyCommand): if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) - patt = f"%-{width}s : %s" for name, version in versions: - print(patt % (name, version)) + print(f"{name:<{width}} : {version}") else: print(f"Scrapy {scrapy.__version__}") From 92dfa7176dbadcb92e4aae9d3f2c2b02c6e52a4a Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:43:40 +0000 Subject: [PATCH 106/119] Update scrapy/extensions/statsmailer.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/extensions/statsmailer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 997e74fc9..bcdbaff24 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -28,7 +28,7 @@ class StatsMailer: def spider_closed(self, spider): spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" - body += "\n".join(f"{i:<50} : {self.stats.get_stats()[i]}" for i in self.stats.get_stats()) + body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) body += f"\n\n{spider.name} stats\n\n" - body += "\n".join(f"{i:<50} : {spider_stats[i]}" for i in spider_stats) + body += "\n".join(f"{k:<50} : {v}" for k, v in spider_stats.items()) return self.mail.send(self.recipients, f"Scrapy stats for: {spider.name}", body) From ea03e4254fbaff71ce584a461344eb0eb9aa7e09 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:43:52 +0000 Subject: [PATCH 107/119] Update scrapy/http/request/form.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index c90d68fa1..2815303a2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -204,7 +204,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = './/*' + ''.join(f'[@{key}="{clickdata[key]}"]' for key in clickdata) + xpath = './/*' + ''.join(f'[@{k}="{v}"]' for k, v in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') From 9aaddcde0af5910e33fbba253777149e04f021f8 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 11:44:20 +0000 Subject: [PATCH 108/119] Update scrapy/utils/conf.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 05cd5f25c..afd8f5374 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -42,8 +42,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): raise ValueError(f'Invalid value {value} for component {name}, ' - 'please provide a real number or None instead' - ) + 'please provide a real number or None instead') # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): From 2ca8dfb4b08ad416716dc18d03983cc159cc5634 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 13:49:39 +0200 Subject: [PATCH 109/119] revert f-string changes for files under sep/ Issue #4324 --- sep/sep-002.rst | 2 +- sep/sep-004.rst | 4 ++-- sep/sep-014.rst | 29 +++++++++++++++-------------- sep/sep-018.rst | 22 +++++++++++----------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/sep/sep-002.rst b/sep/sep-002.rst index 2e8a28340..c467cb402 100644 --- a/sep/sep-002.rst +++ b/sep/sep-002.rst @@ -30,7 +30,7 @@ Proposed Implementation if hasattr(value, '__iter__'): # str/unicode not allowed return [self._field.to_python(v) for v in value] else: - raise TypeError(f"Expected iterable, got {type(value).__name__}") + raise TypeError("Expected iterable, got %s" % type(value).__name__) def get_default(self): # must return a new copy to avoid unexpected behaviors with mutable defaults diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b9f5e556f..05b0eb99c 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ @@ -49,7 +49,7 @@ Here's a simple proof-of-concept code of such script: cr = Crawler(start_urls, callback=parse_start_page) cr.run() # blocking call - this populates scraped_items - print(f"{len(scraped_items)} items scraped") + print "%d items scraped" % len(scraped_items) # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 4e3340521..8ca81824d 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ @@ -142,7 +142,7 @@ Custom Processor and External Callback # Callback defined out of spider def my_external_callback(response): - # process item + # process item pass class SampleSpider(CrawlSpider): @@ -233,7 +233,7 @@ Request/Response Matchers def matches_request(self, request): """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) + return self.matches_url(request.url) def matches_response(self, response): """REturns True if Response's url matches initial url""" @@ -305,14 +305,14 @@ Request Extractor for req in self.requests: req.meta.setdefault('link_text', '') req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) + encoding) def reset(self): """Reset state""" FixedSGMLParser.reset(self) self.requests = [] self.base_url = None - + def unknown_starttag(self, tag, attrs): """Process unknown start tag""" if 'base' tag: @@ -376,7 +376,7 @@ Request Processor #!python # - # Request Processors + # Request Processors # Processors receive list of requests and return list of requests # """Request Processors""" @@ -390,7 +390,7 @@ Request Processor # replace in-place req.url = canonicalize_url(req.url) yield req - + class Unique(object): """Filter duplicate Requests""" @@ -455,9 +455,9 @@ Request Processor """Initialize allow/deny attributes""" _re_type = type(re.compile('', 0)) - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] def __call__(self, requests): @@ -524,7 +524,7 @@ Rules Manager # # Handles rules matcher/callbacks # Resolve rule for given response - # + # class RulesManager(object): """Rules Manager""" def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): @@ -542,8 +542,8 @@ Rules Manager # instance default matcher matcher = default_matcher(rule.matcher) else: - raise ValueError('Not valid matcher given ' - f'{rule.matcher!r} in {rule!r}') + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) # prepare callback if callable(rule.callback): @@ -553,7 +553,8 @@ Rules Manager callback = getattr(spider, rule.callback) if not callable(callback): - raise AttributeError(f'Invalid callback {callback!r} can not be resolved') + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) else: callback = None diff --git a/sep/sep-018.rst b/sep/sep-018.rst index d0169b81e..fe707923a 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -171,7 +171,7 @@ the same spider: #!python class MySpider(BaseSpider): - middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), + middlewares = [RegexLinkExtractor(), CallbackRules(), CanonicalizeUrl(), ItemIdSetter(), OffsiteMiddleware()] allowed_domains = ['example.com', 'sub.example.com'] @@ -196,7 +196,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -324,7 +324,7 @@ Another example could be for building URL canonicalizers: class CanonializeUrl(object): def process_request(self, request, response, spider): - curl = canonicalize_url(request.url, + curl = canonicalize_url(request.url, rules=spider.canonicalization_rules) return request.replace(url=curl) @@ -332,7 +332,7 @@ Another example could be for building URL canonicalizers: class MySpider(BaseSpider): middlewares = [CanonicalizeUrl()] - canonicalization_rules = ['sort-query-args', + canonicalization_rules = ['sort-query-args', 'normalize-percent-encoding', ...] # ... @@ -414,7 +414,7 @@ A spider middleware to avoid visiting pages forbidden by robots.txt: if netloc in info.pending: res = None else: - robotsurl = f"{url.scheme}://{netloc}/robots.txt" + robotsurl = "%s://%s/robots.txt" % (url.scheme, netloc) meta = {'spider': spider, {'handle_httpstatus_list': [403, 404, 500]} res = Request(robotsurl, callback=self.parse_robots, meta=meta, priority=self.REQUEST_PRIORITY) @@ -474,7 +474,7 @@ This is a port of the Offsite middleware to the new spider middleware API: if host and host not in info.hosts_seen: spider.log("Filtered offsite request to %r: %s" % (host, request)) info.hosts_seen.add(host) - + def should_follow(self, request, spider): info = self.spiders[spider] # hostname can be None for wrong urls (like javascript links) @@ -484,7 +484,7 @@ This is a port of the Offsite middleware to the new spider middleware API: def get_host_regex(self, spider): """Override this method to implement a different offsite policy""" domains = [d.replace('.', r'\.') for d in spider.allowed_domains] - regex = fr'^(.*\.)?({"|".join(domains)})$' + regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) def spider_opened(self, spider): @@ -570,7 +570,7 @@ A middleware to filter out requests already seen: self.dupefilter = load_object(clspath)() dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - + def enqueue_request(self, spider, request): seen = self.dupefilter.request_seen(spider, request) if not seen or request.dont_filter: @@ -601,8 +601,8 @@ A middleware to Scrape data using Parsley as described in UsingParsley for name in parslet.keys(): self.fields[name] = Field() super(ParsleyItem, self).__init__(*a, **kw) - self.item_class = ParsleyItem - self.parsley = PyParsley(parslet, output='python') + self.item_class = ParsleyItem + self.parsley = PyParsley(parslet, output='python') def process_response(self, response, request, spider): return self.item_class(self.parsly.parse(string=response.body)) @@ -627,7 +627,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed From a8114d3731cfd1ef8fdb808b1563a7288b8a2e90 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 26 Aug 2020 09:00:36 -0300 Subject: [PATCH 110/119] Typing: annotate a few Spider attributes --- scrapy/spiders/__init__.py | 5 ++-- scrapy/spiders/crawl.py | 3 ++- setup.cfg | 51 -------------------------------------- tests/spiders.py | 2 +- 4 files changed, 6 insertions(+), 55 deletions(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 12b4fba09..a66d65846 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spiders.rst """ import logging import warnings +from typing import Optional from scrapy import signals from scrapy.http import Request @@ -18,8 +19,8 @@ class Spider(object_ref): class. """ - name = None - custom_settings = None + name: Optional[str] = None + custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): if name is not None: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index c9fbce08d..bc4551a54 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -7,6 +7,7 @@ See documentation in docs/topics/spiders.rst import copy import warnings +from typing import Sequence from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse @@ -72,7 +73,7 @@ class Rule: class CrawlSpider(Spider): - rules = () + rules: Sequence[Rule] = () def __init__(self, *a, **kw): super().__init__(*a, **kw) diff --git a/setup.cfg b/setup.cfg index 3a624ec94..8101443e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,9 +16,6 @@ ignore_errors = True [mypy-scrapy.commands] ignore_errors = True -[mypy-scrapy.commands.bench] -ignore_errors = True - [mypy-scrapy.commands.parse] ignore_errors = True @@ -28,9 +25,6 @@ ignore_errors = True [mypy-scrapy.contracts] ignore_errors = True -[mypy-scrapy.core.spidermw] -ignore_errors = True - [mypy-scrapy.interfaces] ignore_errors = True @@ -70,15 +64,6 @@ ignore_errors = True [mypy-tests.mocks.dummydbm] ignore_errors = True -[mypy-tests.spiders] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception] -ignore_errors = True - -[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal] -ignore_errors = True - [mypy-tests.test_command_fetch] ignore_errors = True @@ -94,9 +79,6 @@ ignore_errors = True [mypy-tests.test_contracts] ignore_errors = True -[mypy-tests.test_crawler] -ignore_errors = True - [mypy-tests.test_downloader_handlers] ignore_errors = True @@ -127,53 +109,20 @@ ignore_errors = True [mypy-tests.test_pipeline_images] ignore_errors = True -[mypy-tests.test_pipelines] -ignore_errors = True - -[mypy-tests.test_request_attribute_binding] -ignore_errors = True - [mypy-tests.test_request_cb_kwargs] ignore_errors = True -[mypy-tests.test_request_left] -ignore_errors = True - [mypy-tests.test_scheduler] ignore_errors = True -[mypy-tests.test_signals] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.nested.spider4] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider1] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider2] -ignore_errors = True - -[mypy-tests.test_spiderloader.test_spiders.spider3] -ignore_errors = True - [mypy-tests.test_spidermiddleware_httperror] ignore_errors = True -[mypy-tests.test_spidermiddleware_output_chain] -ignore_errors = True - [mypy-tests.test_spidermiddleware_referer] ignore_errors = True -[mypy-tests.test_utils_reqser] -ignore_errors = True - [mypy-tests.test_utils_serialize] ignore_errors = True -[mypy-tests.test_utils_spider] -ignore_errors = True - [mypy-tests.test_utils_url] ignore_errors = True diff --git a/tests/spiders.py b/tests/spiders.py index 63bd726fb..8a0ee44b7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -255,7 +255,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): A CrawlSpider which overrides the 'parse' method """ name = 'crawl_spider_with_parse_method' - custom_settings = { + custom_settings: dict = { 'RETRY_HTTP_CODES': [], # no need to retry } rules = ( From 5ab1a318e8a8245cad694edc297d94ac80a90760 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Wed, 26 Aug 2020 15:11:46 +0200 Subject: [PATCH 111/119] test: list appears in ValueError Exception message Issue #4324 --- scrapy/utils/conf.py | 2 +- tests/test_utils_conf.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index afd8f5374..4e7a9967e 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -17,7 +17,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _check_components(complist): if len({convert(c) for c in complist}) != len(complist): - raise ValueError('Some paths in {complist!r} convert to the same object, ' + raise ValueError(f'Some paths in {complist!r} convert to the same object, ' 'please update your settings') def _map_keys(compdict): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index ccc65c4fd..061bc8c7c 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -50,8 +50,9 @@ class BuildComponentListTest(unittest.TestCase): def test_duplicate_components_in_list(self): duplicate_list = ['a', 'b', 'a'] - self.assertRaises(ValueError, build_component_list, None, - duplicate_list, convert=lambda x: x) + with self.assertRaises(ValueError) as cm: + build_component_list(None, duplicate_list, convert=lambda x: x) + self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): # Higher priority takes precedence From 195f738bbaa7c090df795161390056c3c6c7f1f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 12:43:43 +0200 Subject: [PATCH 112/119] Update Python version references after dropping support for 3.5 (#4742) * Update Python version references after dropping support for 3.5 * Remove outdated test * Undo change affecting collect_asyncgen * Undo change to be handled by #4743 * Remove unused import * Remove unused import * Update tests/requirements-py3.txt Co-authored-by: Mikhail Korobov Co-authored-by: Mikhail Korobov --- README.rst | 2 +- docs/intro/install.rst | 4 ++-- docs/intro/tutorial.rst | 2 -- docs/topics/coroutines.rst | 15 ++++----------- scrapy/__init__.py | 4 ++-- scrapy/utils/defer.py | 11 ++--------- scrapy/utils/gz.py | 7 +++---- setup.py | 3 +-- tests/requirements-py3.txt | 3 +-- tests/test_crawl.py | 4 ---- tests/test_item.py | 12 ------------ tests/test_proxy_connect.py | 11 ----------- tox.ini | 2 +- 13 files changed, 17 insertions(+), 63 deletions(-) diff --git a/README.rst b/README.rst index 0e3939e9b..a8f2ba52b 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ including a list of features. Requirements ============ -* Python 3.5.2+ +* Python 3.6+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6d65ae2ee..8b4240bf6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,8 +9,8 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.5.2+, either the CPython implementation (default) or -the PyPy 5.9+ implementation (see :ref:`python:implementations`). +Scrapy requires Python 3.6+, either the CPython implementation (default) or +the PyPy 7.2.0+ implementation (see :ref:`python:implementations`). Installing Scrapy diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index f96c78887..b3b8b4706 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -405,8 +405,6 @@ to get all of them: from sys import version_info -.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") - Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a0952d323..3b1549bd3 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - The following are known caveats of the current implementation that we aim - to address in future versions of Scrapy: - - - The callback output is not processed until the whole callback finishes. + .. note:: The callback output is not processed until the whole callback + finishes. As a side effect, if the callback raises an exception, none of its output is processed. - - Because `asynchronous generators were introduced in Python 3.6`_, you - can only use ``yield`` if you are using Python 3.6 or later. - - If you need to output multiple items or requests and you are using - Python 3.5, return an iterable (e.g. a list) instead. + This is a known caveat of the current implementation that we aim to + address in a future version of Scrapy. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/ - Usage ===== diff --git a/scrapy/__init__.py b/scrapy/__init__.py index f0259a9b7..4326ca4aa 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 5, 2): - print("Scrapy %s requires Python 3.5.2" % __version__) +if sys.version_info < (3, 6): + print("Scrapy %s requires Python 3.6+" % __version__) sys.exit(1) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index a3950db75..21ba02a0b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -124,18 +124,11 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) -def _isfuture(o): - # workaround for Python before 3.5.3 not having asyncio.isfuture - if hasattr(asyncio, 'isfuture'): - return asyncio.isfuture(o) - return isinstance(o, asyncio.Future) - - def deferred_from_coro(o): """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): return o - if _isfuture(o) or inspect.isawaitable(o): + if asyncio.isfuture(o) or inspect.isawaitable(o): if not is_asyncio_reactor_installed(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" @@ -167,7 +160,7 @@ def maybeDeferred_coro(f, *args, **kw): if isinstance(result, defer.Deferred): return result - elif _isfuture(result) or inspect.isawaitable(result): + elif asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) elif isinstance(result, failure.Failure): return defer.fail(result) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index fbd7bd18f..11d433cf5 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -6,11 +6,10 @@ import struct from scrapy.utils.decorators import deprecated -# - Python>=3.5 GzipFile's read() has issues returning leftover -# uncompressed data when input is corrupted -# (regression or bug-fix compared to Python 3.4) +# - GzipFile's read() has issues returning leftover uncompressed data when +# input is corrupted # - read1(), which fetches data before raising EOFError on next call -# works here but is only available from Python>=3.3 +# works here @deprecated('GzipFile.read1') def read1(gzf, size=-1): return gzf.read1(size) diff --git a/setup.py b/setup.py index 52a27c368..0c2281400 100644 --- a/setup.py +++ b/setup.py @@ -82,7 +82,6 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', @@ -92,7 +91,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.5.2', + python_requires='>=3.6', install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 44ddcded8..2247ed917 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,8 +2,7 @@ attrs dataclasses; python_version == '3.6' mitmproxy; python_version >= '3.7' -mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7' -mitmproxy < 4; python_version < '3.6' +mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' pyftpdlib # https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 642c24651..c1b918baf 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import sys from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -405,7 +404,6 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("Got response 200", str(log)) self.assertIn({"foo": 42}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): @@ -417,7 +415,6 @@ class CrawlSpiderTestCase(TestCase): itemcount = crawler.stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): @@ -437,7 +434,6 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({'foo': i}, items) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): diff --git a/tests/test_item.py b/tests/test_item.py index 66fa761f0..78d204e34 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,4 +1,3 @@ -import sys import unittest from unittest import mock from warnings import catch_warnings @@ -7,9 +6,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta -PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) - - class ItemTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): @@ -280,14 +276,6 @@ class ItemMetaTest(unittest.TestCase): with mock.patch.object(base, '__new__', new_mock): class MyItem(Item): - if not PY36_PLUS: - # This attribute is an internal attribute in Python 3.6+ - # and must be propagated properly. See - # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object - # In <3.6, we add a dummy attribute just to ensure the - # __new__ method propagates it correctly. - __classcell__ = object() - def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index a56e3c39a..6f70c4267 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -7,7 +7,6 @@ from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit from unittest import skipIf -import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -58,8 +57,6 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) -@skipIf(sys.version_info < (3, 5, 4), - "requires mitmproxy < 3.0.0, which these tests do not support") @skipIf("pypy" in sys.executable, "mitmproxy does not support PyPy") @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), @@ -88,14 +85,6 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, log) - @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6)) - @defer.inlineCallbacks - def test_https_connect_tunnel_error(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(log) - @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) diff --git a/tox.ini b/tox.ini index dec0d75e8..12e40295c 100644 --- a/tox.ini +++ b/tox.ini @@ -89,7 +89,7 @@ deps = basepython = python3 deps = {[pinned]deps} - # First lxml version that includes a Windows wheel for Python 3.5, so we do + # First lxml version that includes a Windows wheel for Python 3.6, so we do # not need to build lxml from sources in a CI Windows job: lxml==3.8.0 From 3f0a677c0496da3d4d4294a182aa5e5b297a7cb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Aug 2020 20:56:58 +0200 Subject: [PATCH 113/119] Cover version directive usage in the documentation policy (#4310) * Cover version directives in the documentation policy * Remove version directives in preparation for Scrapy 2.0 * Update the policy based on the deprecation policy * Only remove version directives after 3 years --- docs/contributing.rst | 11 ++++++++ docs/topics/api.rst | 2 -- docs/topics/autothrottle.rst | 2 -- docs/topics/benchmarking.rst | 2 -- docs/topics/commands.rst | 4 --- docs/topics/contracts.rst | 2 -- docs/topics/downloader-middleware.rst | 36 --------------------------- docs/topics/extensions.rst | 4 --- docs/topics/feed-exports.rst | 2 -- docs/topics/request-response.rst | 12 --------- docs/topics/settings.rst | 4 --- docs/topics/spider-middleware.rst | 6 ----- 12 files changed, 11 insertions(+), 76 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 525ad3497..675f55c38 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -199,6 +199,17 @@ In any case, if something is covered in a docstring, use the documentation instead of duplicating the docstring in files within the ``docs/`` directory. +Documentation updates that cover new or modified features must use Sphinx’s +:rst:dir:`versionadded` and :rst:dir:`versionchanged` directives. Use +``VERSION`` as version, we will replace it with the actual version right before +the corresponding release. When we release a new major or minor version of +Scrapy, we remove these directives if they are older than 3 years. + +Documentation about deprecated features must be removed as those features are +deprecated, so that new readers do not run into it. New deprecations and +deprecation removals are documented in the :ref:`release notes `. + + Tests ===== diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 52509ffdf..445b2979f 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -4,8 +4,6 @@ Core API ======== -.. versionadded:: 0.15 - This section documents the Scrapy core API, and it's intended for developers of extensions and middlewares. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 4317019fc..8e6aae65c 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -128,8 +128,6 @@ The maximum download delay (in seconds) to be set in case of high latencies. AUTOTHROTTLE_TARGET_CONCURRENCY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 1.1 - Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index 99469ebf1..b01a66188 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -4,8 +4,6 @@ Benchmarking ============ -.. versionadded:: 0.17 - Scrapy comes with a simple benchmarking suite that spawns a local HTTP server and crawls it at the maximum possible speed. The goal of this benchmarking is to get an idea of how Scrapy performs in your hardware, in order to have a diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 9638a2322..7de5e8121 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -6,8 +6,6 @@ Command line tool ================= -.. versionadded:: 0.10 - Scrapy is controlled through the ``scrapy`` command-line tool, to be referred here as the "Scrapy tool" to differentiate it from the sub-commands, which we just call "commands" or "Scrapy commands". @@ -566,8 +564,6 @@ and Platform info, which is useful for bug reports. bench ----- -.. versionadded:: 0.17 - * Syntax: ``scrapy bench`` * Requires project: *no* diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 430720fe3..e61421bf1 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -4,8 +4,6 @@ Spiders Contracts ================= -.. versionadded:: 0.15 - Testing spiders can get particularly annoying and while nothing prevents you from writing unit tests the task gets cumbersome quickly. Scrapy offers an integrated way of testing your spiders by the means of contracts. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 323e553e5..06e614941 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -217,8 +217,6 @@ The following settings can be used to configure the cookie middleware: Multiple cookie sessions per spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: 0.15 - There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. @@ -475,8 +473,6 @@ DBM storage backend .. class:: DbmCacheStorage - .. versionadded:: 0.13 - A DBM_ storage backend is also available for the HTTP cache middleware. By default, it uses the :mod:`dbm`, but you can change it with the @@ -549,15 +545,10 @@ settings: HTTPCACHE_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.11 - Default: ``False`` Whether the HTTP cache will be enabled. -.. versionchanged:: 0.11 - Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -570,9 +561,6 @@ Expiration time for cached requests, in seconds. Cached requests older than this time will be re-downloaded. If zero, cached requests will never expire. -.. versionchanged:: 0.11 - Before 0.11, zero meant cached requests always expire. - .. setting:: HTTPCACHE_DIR HTTPCACHE_DIR @@ -589,8 +577,6 @@ project data dir. For more info see: :ref:`topics-project-structure`. HTTPCACHE_IGNORE_HTTP_CODES ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``[]`` Don't cache response with these HTTP codes. @@ -609,8 +595,6 @@ If enabled, requests not found in the cache will be ignored instead of downloade HTTPCACHE_IGNORE_SCHEMES ^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.10 - Default: ``['file']`` Don't cache responses with these URI schemes. @@ -629,8 +613,6 @@ The class which implements the cache storage backend. HTTPCACHE_DBM_MODULE ^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend @@ -641,8 +623,6 @@ The database module to use in the :ref:`DBM storage backend HTTPCACHE_POLICY ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.18 - Default: ``'scrapy.extensions.httpcache.DummyPolicy'`` The class which implements the cache policy. @@ -652,8 +632,6 @@ The class which implements the cache policy. HTTPCACHE_GZIP ^^^^^^^^^^^^^^ -.. versionadded:: 1.0 - Default: ``False`` If enabled, will compress all cached data with gzip. @@ -664,8 +642,6 @@ This setting is specific to the Filesystem backend. HTTPCACHE_ALWAYS_STORE ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``False`` If enabled, will cache pages unconditionally. @@ -684,8 +660,6 @@ responses you feed to the cache middleware. HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 1.1 - Default: ``[]`` List of Cache-Control directives in responses to be ignored. @@ -735,8 +709,6 @@ HttpProxyMiddleware .. module:: scrapy.downloadermiddlewares.httpproxy :synopsis: Http Proxy Middleware -.. versionadded:: 0.8 - .. reqmeta:: proxy .. class:: HttpProxyMiddleware @@ -817,8 +789,6 @@ RedirectMiddleware settings REDIRECT_ENABLED ^^^^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Redirect middleware will be enabled. @@ -860,8 +830,6 @@ MetaRefreshMiddleware settings METAREFRESH_ENABLED ^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 - Default: ``True`` Whether the Meta Refresh middleware will be enabled. @@ -924,8 +892,6 @@ RetryMiddleware Settings RETRY_ENABLED ^^^^^^^^^^^^^ -.. versionadded:: 0.13 - Default: ``True`` Whether the Retry middleware will be enabled. @@ -1179,8 +1145,6 @@ AjaxCrawlMiddleware Settings AJAXCRAWL_ENABLED ^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.21 - Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0fc83e645..14096ada4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -288,8 +288,6 @@ If zero (or non set), spiders won't be closed by number of passed items. CLOSESPIDER_PAGECOUNT """"""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of responses to crawl. If the spider @@ -302,8 +300,6 @@ number of crawled responses. CLOSESPIDER_ERRORCOUNT """""""""""""""""""""" -.. versionadded:: 0.11 - Default: ``0`` An integer which specifies the maximum number of errors to receive before diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cd4f7cf29..9fb2189e8 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -4,8 +4,6 @@ Feed exports ============ -.. versionadded:: 0.10 - One of the most frequently required features when implementing scrapers is being able to store the scraped data properly and, quite often, that means generating an "export file" with the scraped data (commonly called "export diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d0136137f..30b1945d0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -553,18 +553,6 @@ fields with form data from :class:`Response` objects. The other parameters of this class method are passed directly to the :class:`FormRequest` ``__init__`` method. - .. versionadded:: 0.10.3 - The ``formname`` parameter. - - .. versionadded:: 0.17 - The ``formxpath`` parameter. - - .. versionadded:: 1.1.0 - The ``formcss`` parameter. - - .. versionadded:: 1.1.0 - The ``formid`` parameter. - Request usage examples ---------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2924c0566..d010a0023 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1076,8 +1076,6 @@ See :ref:`topics-extensions-ref-memusage`. MEMUSAGE_CHECK_INTERVAL_SECONDS ------------------------------- -.. versionadded:: 1.1 - Default: ``60.0`` Scope: ``scrapy.extensions.memusage`` @@ -1358,8 +1356,6 @@ The class that will be used for loading spiders, which must implement the SPIDER_LOADER_WARN_ONLY ----------------------- -.. versionadded:: 1.3.3 - Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c6cbdba76..fc114a63f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -146,8 +146,6 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_start_requests(start_requests, spider) - .. versionadded:: 0.15 - This method is called with the start requests of the spider, and works similarly to the :meth:`process_spider_output` method, except that it doesn't have a response associated and must return only requests (not @@ -341,8 +339,6 @@ RefererMiddleware settings REFERER_ENABLED ^^^^^^^^^^^^^^^ -.. versionadded:: 0.15 - Default: ``True`` Whether to enable referer middleware. @@ -352,8 +348,6 @@ Whether to enable referer middleware. REFERRER_POLICY ^^^^^^^^^^^^^^^ -.. versionadded:: 1.4 - Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. reqmeta:: referrer_policy From 0e579182319504ba7bfd0c09333fe92f70c6d312 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Fri, 28 Aug 2020 13:58:32 +0200 Subject: [PATCH 114/119] test(Slot): cover __repr__ Issue: #4324 --- scrapy/core/downloader/__init__.py | 2 +- tests/test_core_downloader.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/test_core_downloader.py diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 12a9db6dd..4f7ab594f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -43,7 +43,7 @@ class Slot: cls_name = self.__class__.__name__ return (f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r}") + f"randomize_delay={self.randomize_delay!r})") def __str__(self): return ( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py new file mode 100644 index 000000000..113ea8f19 --- /dev/null +++ b/tests/test_core_downloader.py @@ -0,0 +1,10 @@ +from twisted.trial import unittest + +from scrapy.core.downloader import Slot + + +class SlotTest(unittest.TestCase): + + def test_repr(self): + slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) + self.assertEqual(repr(slot), 'Slot(concurrency=8, delay=0.10, randomize_delay=True)') From de640f41ecfa1a87b39d9e9268c396fc97353fc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 28 Aug 2020 18:27:36 +0500 Subject: [PATCH 115/119] Merge back tests/py36/_test_crawl.py. --- conftest.py | 2 -- tests/py36/_test_crawl.py | 57 --------------------------------------- tests/spiders.py | 53 ++++++++++++++++++++++++++++++++++++ tests/test_crawl.py | 6 ++--- 4 files changed, 56 insertions(+), 62 deletions(-) delete mode 100644 tests/py36/_test_crawl.py diff --git a/conftest.py b/conftest.py index b39d644a5..be97b7714 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ collect_ignore = [ *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), ] for line in open('tests/ignores.txt'): diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py deleted file mode 100644 index 162a53760..000000000 --- a/tests/py36/_test_crawl.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio - -from scrapy import Request -from tests.spiders import SimpleSpider - - -class AsyncDefAsyncioGenSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen' - - async def parse(self, response): - await asyncio.sleep(0.2) - yield {'foo': 42} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_loop' - - async def parse(self, response): - for i in range(10): - await asyncio.sleep(0.1) - yield {'foo': i} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_complex' - initial_reqs = 4 - following_reqs = 3 - depth = 2 - - def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}, - dont_filter=True, - callback=cb) - - def start_requests(self): - for i in range(1, self.initial_reqs + 1): - yield self._get_req(i) - - async def parse(self, response): - index = response.meta['index'] - yield {'index': index} - if index < 10 ** self.depth: - for new_index in range(10 * index, 10 * index + self.following_reqs): - yield self._get_req(new_index) - yield self._get_req(index, cb=self.parse2) - await asyncio.sleep(0.1) - yield {'index': index + 5} - - async def parse2(self, response): - await asyncio.sleep(0.1) - yield {'index2': response.meta['index']} diff --git a/tests/spiders.py b/tests/spiders.py index 8a0ee44b7..8a85d2b51 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index, cb=None): + return Request(self.mockserver.url("/status?n=200&request=%d" % index), + meta={'index': index}, + dont_filter=True, + callback=cb) + + def start_requests(self): + for i in range(1, self.initial_reqs + 1): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) + await asyncio.sleep(0.1) + yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c1b918baf..ba8c3fd3c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -19,6 +19,9 @@ from scrapy.http.response import Response from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( + AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenLoopSpider, + AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, @@ -407,7 +410,6 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - from tests.py36._test_crawl import AsyncDefAsyncioGenSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -423,7 +425,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: @@ -442,7 +443,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) From ffdf6fe100cf0ac99a726eab9d927e65f4c01364 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 29 Aug 2020 07:21:48 +0200 Subject: [PATCH 116/119] use f-strings for the newly merged code from master Issue: #4324 --- tests/spiders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index 3c01c0217..106392ea6 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -177,7 +177,7 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): depth = 2 def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), + return Request(self.mockserver.url(f"/status?n=200&request={index}"), meta={'index': index}, dont_filter=True, callback=cb) From 8123c427373778398036bb375bf03b3cd6b240de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 28 Aug 2020 18:31:09 +0500 Subject: [PATCH 117/119] Simplify running spiders in CrawlSpiderTestCase. --- tests/test_crawl.py | 56 +++++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index ba8c3fd3c..a8c9b6d7f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -335,6 +335,19 @@ class CrawlSpiderTestCase(TestCase): def tearDown(self): self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks + def _run_spider(self, spider_cls): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(spider_cls) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + return log, items, crawler.stats + @defer.inlineCallbacks def test_crawlspider_with_parse(self): self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) @@ -379,15 +392,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_items_list(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) self.assertIn("Got response 200", str(log)) self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) @@ -410,27 +415,17 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 1) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_loop(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) self.assertIn("Got response 200", str(log)) - itemcount = crawler.stats.get_value('item_scraped_count') + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 10) for i in range(10): self.assertIn({'foo': i}, items) @@ -438,15 +433,8 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): - items = [] - - def _on_item_scraped(item): - items.append(item) - - crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) - crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) - itemcount = crawler.stats.get_value('item_scraped_count') + _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) + itemcount = stats.get_value('item_scraped_count') self.assertEqual(itemcount, 156) # some random items for i in [1, 4, 21, 22, 207, 311]: @@ -457,9 +445,7 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): - crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider) - with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): self.assertIn("Got response 200, req_id %d" % req_id, str(log)) From 90ca9350f56c8c450c3385590d77390ad1d77365 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Sat, 29 Aug 2020 08:03:03 +0000 Subject: [PATCH 118/119] Update scrapy/commands/check.py Co-authored-by: Mikhail Korobov --- scrapy/commands/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 7e848dc97..ae21d86e6 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -17,7 +17,7 @@ class TextTestResult(_TextTestResult): plural = "s" if run != 1 else "" writeln(self.separator2) - writeln(f"Ran {run} contract{plural} in {stop - start:.3f}") + writeln(f"Ran {run} contract{plural} in {stop - start:.3f}s") writeln() infos = [] From 307e35c6641c0d9cf4b251996ebefd1347c2c1fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 1 Sep 2020 06:04:00 -0300 Subject: [PATCH 119/119] Improve check for invalid cookie in CookiesMiddleware (#4772) --- scrapy/downloadermiddlewares/cookies.py | 2 +- tests/test_downloadermiddleware_cookies.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index e2b7dd901..87f8152a4 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -74,7 +74,7 @@ class CookiesMiddleware: """ decoded = {} for key in ("name", "value", "path", "domain"): - if not cookie.get(key): + if cookie.get(key) is None: if key in ("name", "value"): msg = "Invalid cookie found in request {}: {} ('{}' is missing)" logger.warning(msg.format(request, cookie, key)) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 010577415..a3de307ee 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -322,6 +322,9 @@ class CookiesMiddlewareTest(TestCase): cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] req2 = Request('http://example.org/2', cookies=cookies2) assert self.mw.process_request(req2, self.spider) is None + cookies3 = [{'name': 'foo', 'value': None}, {'name': 'key', 'value': ''}] + req3 = Request('http://example.org/3', cookies=cookies3) + assert self.mw.process_request(req3, self.spider) is None lc.check( ("scrapy.downloadermiddlewares.cookies", "WARNING", @@ -331,6 +334,11 @@ class CookiesMiddlewareTest(TestCase): "WARNING", "Invalid cookie found in request :" " {'name': 'foo'} ('value' is missing)"), + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo', 'value': None} ('value' is missing)"), ) self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') + self.assertCookieValEqual(req3.headers['Cookie'], 'key=')