From 114437c1693c85125c4275387616d398e68ec20f Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Wed, 4 Nov 2015 02:29:28 +0800 Subject: [PATCH 01/44] added: Doc for `scrapy.http.TextResponse.urljoin` --- docs/topics/request-response.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75b98d3b3..ba3d697ef 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -666,6 +666,14 @@ TextResponse objects The same as :attr:`text`, but available as a method. This method is kept for backwards compatibility; please prefer ``response.text``. + .. method:: TextResponse.urljoin(url) + + Constructs an absolute url by combining the Response's base url with + a possible relative url. The base url shall be extracted from the + ```` tag, or just the Response's :attr:`url` if there is no such + tag. + + HtmlResponse objects -------------------- From 412f8526029f63fbebd6dae3bad7240dee8dc090 Mon Sep 17 00:00:00 2001 From: Arvind Prasanna <1108710+aprasanna@users.noreply.github.com> Date: Tue, 6 Mar 2018 23:58:27 -0500 Subject: [PATCH 02/44] A few typo fixes and some grammatical enhancements --- docs/news.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1629510b2..be4cac3f3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1888,7 +1888,7 @@ Scrapy 0.14.3 - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`) - update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`) - added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`) -- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`) +- do not discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`) - do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`) - fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`) - fixed minor defect in link extractors documentation (:commit:`ba14f38`) @@ -1984,7 +1984,7 @@ Code rearranged and removed - Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot - Removed support for default field values in Scrapy items (:rev:`2616`) - Removed experimental crawlspider v2 (:rev:`2632`) -- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe filtering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) - Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) - Removed deprecated Execution Queue (:rev:`2704`) - Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) @@ -2054,7 +2054,7 @@ New features and improvements - Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) - Support for overriding default request headers per spider (#181) - Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) -- Splitted Debian package into two packages - the library and the service (#187) +- The Debian package has been split into two packages - library and the service (#187) - Scrapy log refactoring (#188) - New extension for keeping persistent spider contexts among different runs (#203) - Added `dont_redirect` request.meta key for avoiding redirects (#233) @@ -2075,7 +2075,7 @@ API changes - ``url`` and ``body`` attributes of Request objects are now read-only (#230) - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) -- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Offsite middleware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) - Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` From 6d6243afbb16cb5b7d401a0b2ea7a174b7be71b8 Mon Sep 17 00:00:00 2001 From: leobalestri <33645316+leobalestri@users.noreply.github.com> Date: Sun, 16 Feb 2020 23:45:41 -0800 Subject: [PATCH 03/44] Update install.rst Minor grammar and typo fixes --- docs/intro/install.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 51b41b4d7..a08dedbd0 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -186,7 +186,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python, install a new, updated version +* *(Recommended)* **Don't** use system python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: @@ -231,9 +231,9 @@ PyPy We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. -Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. -This means that these dependecies will be built during installation. -On OS X, you are likely to face an issue with building Cryptography dependency, +Most scrapy dependencies now have binary wheels for CPython, but not for PyPy. +This means that these dependencies will be built during installation. +On OS X, you are likely to face an issue with building Cryptography dependency. The solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command From 231c9ddef8be9d749dcc2684f07ea9bb8bd02aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 28 Feb 2020 18:50:45 +0100 Subject: [PATCH 04/44] Update docs/intro/install.rst --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index a08dedbd0..b71379e4d 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -186,7 +186,7 @@ prevents ``pip`` from updating system packages. This has to be addressed to successfully install Scrapy and its dependencies. Here are some proposed solutions: -* *(Recommended)* **Don't** use system python. Install a new, updated version +* *(Recommended)* **Don't** use system Python. Install a new, updated version that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: From f2c800c5c9f88b4b583181e9cf49eb3cd8d538f0 Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 11:14:54 +0100 Subject: [PATCH 05/44] Improve open_in_browser base tag injection (#5319) --- scrapy/utils/response.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3ef7b463..8b109dced 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,8 +3,9 @@ This module provides some useful functions for working with scrapy.http.Response objects """ import os -import webbrowser +import re import tempfile +import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary @@ -80,8 +81,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b'' - body = body.replace(b'', to_bytes(repl)) + repl = fr'\1' + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) ext = '.html' elif isinstance(response, TextResponse): ext = '.txt' From 75ed765476a2ac66ea8f52e7b29186864f65535c Mon Sep 17 00:00:00 2001 From: Samuel Marchal Date: Mon, 15 Nov 2021 14:31:24 +0100 Subject: [PATCH 06/44] Test coverage for open_in_browser base tag injection (#5319) --- tests/test_utils_response.py | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d6f4c0bb5..0a09f6109 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -83,3 +83,56 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(response_status_message(200), '200 OK') self.assertEqual(response_status_message(404), '404 Not Found') self.assertEqual(response_status_message(573), "573 Unknown Status") + + def test_inject_base_url(self): + url = "http://www.example.com" + + def check_base_url(burl): + path = urlparse(burl).path + if not os.path.exists(path): + path = burl.replace('file://', '') + with open(path, "rb") as f: + bbody = f.read() + self.assertEqual(bbody.count(b''), 1) + return True + + r1 = HtmlResponse(url, body=b""" + + Dummy +

Hello world.

+ """) + r2 = HtmlResponse(url, body=b""" + + Dummy + Hello world. + """) + r3 = HtmlResponse(url, body=b""" + + Dummy + +
Hello header
+

Hello world.

+ + """) + r4 = HtmlResponse(url, body=b""" + + + Dummy +

Hello world.

+ """) + r5 = HtmlResponse(url, body=b""" + + + + Standard head + +

Hello world.

+ """) + + assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" + assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head" + assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag" + assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment" + assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment" From 6ec66c96fb962a276c2d285cd5ffa34d84925df1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 12:25:45 +0500 Subject: [PATCH 07/44] Fix and pin pylint. --- pylintrc | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 699686e16..2cdd6321e 100644 --- a/pylintrc +++ b/pylintrc @@ -112,6 +112,7 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, + use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/tox.ini b/tox.ini index e4514f512..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint + pylint==2.12.1 commands = pylint conftest.py docs extras scrapy setup.py tests From 4cc039628eb4861f98f7997e90125745e30f8687 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Nov 2021 19:52:03 +0500 Subject: [PATCH 08/44] Fix typing of middleware methods. --- scrapy/core/downloader/middleware.py | 5 ++++- scrapy/core/spidermw.py | 3 ++- scrapy/middleware.py | 16 ++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a5619d8a4..289147466 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,7 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union +from typing import Callable, Union, cast from twisted.internet import defer from twisted.python.failure import Failure @@ -37,6 +37,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): @defer.inlineCallbacks def process_request(request: Request): for method in self.methods['process_request']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -55,6 +56,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods['process_response']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, response=response, spider=spider)) if not isinstance(response, (Response, Request)): raise _InvalidOutput( @@ -69,6 +71,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(failure: Failure): exception = failure.value for method in self.methods['process_exception']: + method = cast(Callable, method) response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider)) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 7e58521ac..7cdc28284 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union +from typing import Any, Callable, Generator, Iterable, Union, cast from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -47,6 +47,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_input(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Any: for method in self.methods['process_spider_input']: + method = cast(Callable, method) try: result = method(response=response, spider=spider) if result is not None: diff --git a/scrapy/middleware.py b/scrapy/middleware.py index bbec38086..e8f60287a 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict +from typing import Callable, Deque, Dict, Optional, cast, Iterable from twisted.internet.defer import Deferred @@ -21,7 +21,8 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - self.methods: Dict[str, Deque[Callable]] = defaultdict(deque) + # Optional because process_spider_output and process_spider_exception can be None + self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -64,14 +65,17 @@ class MiddlewareManager: self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname: str, obj, *args) -> Deferred: - return process_parallel(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_parallel(methods, obj, *args) def _process_chain(self, methodname: str, obj, *args) -> Deferred: - return process_chain(self.methods[methodname], obj, *args) + methods = cast(Iterable[Callable], self.methods[methodname]) + return process_chain(methods, obj, *args) def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - return process_chain_both(self.methods[cb_methodname], - self.methods[eb_methodname], obj, *args) + cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) + eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) + return process_chain_both(cb_methods, eb_methods, obj, *args) def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From eb62906c3e4c1e1f8e3e6c7965a04d5e65c61907 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Dec 2021 17:40:41 +0500 Subject: [PATCH 09/44] Extract utils.log.log_reactor_info(). --- scrapy/utils/log.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0441c0358..9887ecc40 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -152,6 +152,10 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + log_reactor_info() + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor From 6483dfdbe17cd66c409435b95a05850a3c94b5ee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Dec 2021 19:53:39 +0500 Subject: [PATCH 10/44] Move install_shutdown_handlers() from __init__() to start(). --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..357f14dc0 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -278,7 +278,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -318,6 +317,7 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() From d6a384b3cfdb36cd19b32942479487a9e47c244b Mon Sep 17 00:00:00 2001 From: yogender26 <95638485+yogender26@users.noreply.github.com> Date: Thu, 23 Dec 2021 04:09:05 +0530 Subject: [PATCH 11/44] corrrection of coma (#5347) --- scrapy/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 6e77551c6..5f1dabd33 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -43,14 +43,14 @@ class ScrapyCommand: def long_desc(self): """A long description of the command. Return short description when not - available. It cannot contain newlines, since contents will be formatted + available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): """An extensive help for the command. It will be shown when using the - "help" command. It can contain newlines, since no post-formatting will + "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() From 46ef9cf771789f1db513bbf2f65243d3320ce695 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 22 Dec 2021 21:24:59 +0500 Subject: [PATCH 12/44] Don't install non-working shutdown handlers in `scrapy shell`. --- scrapy/commands/shell.py | 2 +- scrapy/crawler.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..de81986d8 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 357f14dc0..e54ad9750 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -297,7 +297,7 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -308,6 +308,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -317,7 +320,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - install_shutdown_handlers(self._signal_shutdown) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() From 60c8838554a79e70c22a7c6a57baedfcaf521444 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:07:18 +0500 Subject: [PATCH 13/44] Move installing the reactor from CrawlerProcess to Crawler. --- scrapy/crawler.py | 31 ++++++++++++++++++++----------- scrapy/utils/log.py | 1 - tests/test_crawler.py | 5 ++++- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e54ad9750..95cfb1bd1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -69,6 +70,19 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if self.settings.get("TWISTED_REACTOR"): + install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if self.settings.get("TWISTED_REACTOR"): + verify_installed_reactor(self.settings["TWISTED_REACTOR"]) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +167,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +260,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -297,6 +306,11 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool @@ -341,8 +355,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 9887ecc40..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -152,7 +152,6 @@ def log_scrapy_info(settings: Settings) -> None: if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) - log_reactor_info() def log_reactor_info() -> None: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..118cb631b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -271,6 +271,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -279,9 +280,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) + yield runner.crawl(NoRequestsSpider) @defer.inlineCallbacks # https://twistedmatrix.com/trac/ticket/9766 @@ -301,6 +303,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): runner = CrawlerProcess(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) + yield runner.crawl(NoRequestsSpider) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_false(self): From 041699b54cfa6cde9f886a98ff300e3276e2eaad Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:14:47 +0500 Subject: [PATCH 14/44] Remove tests that want to modify the test process reactor. --- tests/test_crawler.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 118cb631b..f445c181e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -285,33 +285,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) yield runner.crawl(NoRequestsSpider) - @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - yield runner.crawl(NoRequestsSpider) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - class ScriptRunnerMixin: def run_script(self, script_name, *script_args): From ebcafdf4a9e0692bf301546b6d60465b3b2c4b06 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:35:26 +0500 Subject: [PATCH 15/44] Add tests for TWISTED_REACTOR in custom_settings. --- .../twisted_reactor_custom_settings.py | 14 +++++++++++ ...wisted_reactor_custom_settings_conflict.py | 22 +++++++++++++++++ .../twisted_reactor_custom_settings_same.py | 21 ++++++++++++++++ tests/test_crawler.py | 24 +++++++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings.py create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_same.py diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..9a6c01d72 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class PollReactorSpider(scrapy.Spider): + name = 'poll_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(PollReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..1f5a44010 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f445c181e..6d6763aec 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') From 002513438204eea5062b5a1d75fb4f261880da4f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 16:45:17 +0500 Subject: [PATCH 16/44] Completely skip WindowsRunSpiderCommandTest outside Windows. --- tests/test_commands.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..efe9b0531 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -765,6 +765,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -777,35 +778,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() From 9c4bfb48362f736fce81b71a6ca1fa0b3600231d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 17:17:36 +0500 Subject: [PATCH 17/44] Remove an unused import. --- tests/test_crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6d6763aec..d68c50026 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -7,7 +7,6 @@ import warnings from unittest import skipIf from pytest import raises, mark -from testfixtures import LogCapture from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version From d4565318c7061c2ccd17fa5d5eabcacef8c34826 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 23 Dec 2021 17:40:31 +0500 Subject: [PATCH 18/44] Fix a reactor test on Windows. --- .../twisted_reactor_custom_settings_conflict.py | 8 ++++---- tests/test_crawler.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py index 9a6c01d72..3f219098c 100644 --- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -2,10 +2,10 @@ import scrapy from scrapy.crawler import CrawlerProcess -class PollReactorSpider(scrapy.Spider): - name = 'poll_reactor' +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' custom_settings = { - "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", } @@ -17,6 +17,6 @@ class AsyncioReactorSpider(scrapy.Spider): process = CrawlerProcess() -process.crawl(PollReactorSpider) +process.crawl(SelectReactorSpider) process.crawl(AsyncioReactorSpider) process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d68c50026..e7d5c8132 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -381,8 +381,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") - self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') From 940cc0776ff86f726e79c2ab2018f4b83a833936 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Dec 2021 17:12:50 +0500 Subject: [PATCH 19/44] Add docs about TWISTED_REACTOR and other per-process settings. --- docs/topics/practices.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..bd0dd8ce0 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -102,6 +102,17 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + .. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: @@ -193,6 +204,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: From a986792def6df1b2bbdf1bc996308d3afd8528c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 24 Dec 2021 19:43:14 +0500 Subject: [PATCH 20/44] Add more docs for TWISTED_REACTOR. --- docs/topics/settings.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 210c1def7..cff6d80cb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1638,10 +1638,18 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform will be used. This is to maintain backward compatibility and +avoid possible problems caused by using a non-default reactor. + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. For additional information, see :doc:`core/howto/choosing-reactor`. From 7380888cad2c255e6f4a7efb6fe4cfd1240fb42c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 30 Dec 2021 18:55:16 +0500 Subject: [PATCH 21/44] Fix a warning message. (#5359) --- scrapy/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 24873f75d..24a6187b9 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -164,7 +164,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, message = ( 'The -t command line option is deprecated in favor of ' 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.', + 'documentation of the -o and -O options for more information.' ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From 64261d9e389737621caa85f320cf81ef2aef1faa Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 15:45:59 +0500 Subject: [PATCH 22/44] Slight refactoring. --- scrapy/crawler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 95cfb1bd1..a638254f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,17 +71,18 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + reactor_class = self.settings.get("TWISTED_REACTOR") if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) else: from twisted.internet import default default.install() log_reactor_info() - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) + if reactor_class: + verify_installed_reactor(reactor_class) self.extensions = ExtensionManager.from_crawler(self) From b81938684b55fca29e48faa766f2b6f6e3ab5d6a Mon Sep 17 00:00:00 2001 From: Andrey Oskin Date: Fri, 31 Dec 2021 21:49:18 +1100 Subject: [PATCH 23/44] Docs: correct process repetition start step (#5356) The process repeats from step 3, the scheduler feeds request to the engine. Steps 1 and 2 are not parts of the loop as their incarnations steps 7 and 8 are parts of the loop. --- docs/topics/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 71d027c86..0c3a7ed88 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -67,7 +67,7 @@ this: the :ref:`Scheduler ` and asks for possible next Requests to crawl. -9. The process repeats (from step 1) until there are no more requests from the +9. The process repeats (from step 3) until there are no more requests from the :ref:`Scheduler `. Components From e4bdd1cb958b7d89b86ea66f0af1cec2d91a6d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Miech?= Date: Fri, 31 Dec 2021 11:57:12 +0100 Subject: [PATCH 24/44] downloader.webclient: make reactor import local (#5357) --- scrapy/core/downloader/webclient.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 915cb5fe3..06cb96489 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -3,7 +3,7 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.http import HTTPClient -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.protocol import ClientFactory from scrapy.http import Headers @@ -170,6 +170,7 @@ class ScrapyHTTPClientFactory(ClientFactory): p.followRedirect = self.followRedirect p.afterFoundGet = self.afterFoundGet if self.timeout: + from twisted.internet import reactor timeoutCall = reactor.callLater(self.timeout, p.timeout) self.deferred.addBoth(self._cancelTimeout, timeoutCall) return p From 57dc58123b98e2026025cc87bdee474bf0656dcb Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 31 Dec 2021 17:15:08 +0500 Subject: [PATCH 25/44] Remove the experimental note about asyncio (#5332) --- docs/topics/asyncio.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 28241ae24..402352721 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -10,11 +10,6 @@ Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the asyncio reactor `, you may use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. -.. warning:: :mod:`asyncio` support in Scrapy is experimental, and not yet - recommended for production environments. Future Scrapy versions - may introduce related changes without a deprecation period or - warning. - .. _install-asyncio: Installing the asyncio reactor From a2763c608d33a9254034d650457b7efa7b434ec0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 18:35:00 +0500 Subject: [PATCH 26/44] Remove unused MiddlewareManager._process_chain_both(). --- scrapy/middleware.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index e8f60287a..2eb1d8609 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.exceptions import NotConfigured from scrapy.settings import Settings from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.defer import process_parallel, process_chain, process_chain_both +from scrapy.utils.defer import process_parallel, process_chain logger = logging.getLogger(__name__) @@ -72,11 +72,6 @@ class MiddlewareManager: methods = cast(Iterable[Callable], self.methods[methodname]) return process_chain(methods, obj, *args) - def _process_chain_both(self, cb_methodname: str, eb_methodname: str, obj, *args) -> Deferred: - cb_methods = cast(Iterable[Callable], self.methods[cb_methodname]) - eb_methods = cast(Iterable[Callable], self.methods[eb_methodname]) - return process_chain_both(cb_methods, eb_methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: return self._process_parallel('open_spider', spider) From 6eaceec735d551f5b777bc641ff8d85dbb3ba98c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 31 Dec 2021 20:14:24 +0500 Subject: [PATCH 27/44] Implement docs suggestions. --- docs/news.rst | 22 ++++++++++++++++++++++ docs/topics/practices.rst | 11 ----------- docs/topics/settings.rst | 13 ++----------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 509366c17..2afe318f6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,3 +1,25 @@ +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. + + .. _news: Release notes diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index bd0dd8ce0..1a9d56143 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -102,17 +102,6 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -.. note:: - .. versionchanged:: VERSION - - The Twisted reactor is now installed when - :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a - :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, - :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now - honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions - they are silently ignored when set there and you need to set these settings - in some other way. - .. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cff6d80cb..f6c95c502 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1639,17 +1639,8 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which means that Scrapy will install the default reactor defined by Twisted for the -current platform will be used. This is to maintain backward compatibility and -avoid possible problems caused by using a non-default reactor. - -.. note:: - .. versionchanged:: VERSION - - Previously this setting had no effect in a spider - :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but - if you :ref:`run several spiders in one process `, - they must not have different values for this setting, because they will use - a single reactor instance. +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. From c5ab58056c29c2c35b183572b0780acfbb15dfe8 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Sat, 1 Jan 2022 00:38:10 +0500 Subject: [PATCH 28/44] Set WindowsSelectorEventLoopPolicy on Windows (#5315) --- .github/workflows/tests-windows.yml | 3 ++ docs/topics/asyncio.rst | 28 +++++++++++++++++++ scrapy/utils/reactor.py | 5 ++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 3 ++ tests/test_commands.py | 11 +++----- tests/test_crawler.py | 16 ----------- tests/test_downloader_handlers.py | 16 +++++++++++ tests/test_utils_asyncio.py | 7 +---- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 6fabf5cde..ab7385118 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,9 @@ jobs: - python-version: "3.10" env: TOXENV: py + - python-version: "3.10" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v2 diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 402352721..8712d4268 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -36,6 +36,34 @@ use it instead of the default asyncio event loop. .. _asyncio-await-dfd: +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations: :class:`~asyncio.SelectorEventLoop` (default before Python +3.8, required when using Twisted) and :class:`~asyncio.ProactorEventLoop` +(default since Python 3.8, cannot work with Twisted). So on Python 3.8+ the +event loop class needs to be changed. Scrapy since VERSION does this +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`, but if you install the reactor +by other means or use an older Scrapy version you need to call the following +code before installing the reactor:: + + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +You can put this in the same function that installs the reactor, if you do that +yourself, or in some code that runs before the reactor is installed, e.g. +``settings.py``. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + Awaiting on Deferreds ===================== diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6723d9b37..96395543c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,5 @@ import asyncio +import sys from contextlib import suppress from twisted.internet import asyncioreactor, error @@ -57,6 +58,10 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): + if sys.version_info >= (3, 8) and sys.platform == "win32": + policy = asyncio.get_event_loop_policy() + if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 8568bd8b8..f2a93074b 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,6 +1,9 @@ import asyncio +import sys from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) import scrapy diff --git a/tests/test_commands.py b/tests/test_commands.py index 75098a77a..81d1a1cab 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -674,9 +674,6 @@ class MySpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' @@ -699,15 +696,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_custom_asyncio_loop_enabled_false(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' ]) import asyncio - loop = asyncio.new_event_loop() + if sys.platform != 'win32': + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() self.assertIn(f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}", log) def test_output(self): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index be067155e..7bc4fba40 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,7 +4,6 @@ import platform import subprocess import sys import warnings -from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -284,9 +283,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -328,17 +324,11 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -377,9 +367,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) @@ -404,9 +391,6 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9c11820e5..a1ea4c679 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import contextlib import os import shutil +import sys import tempfile from typing import Optional, Type from unittest import mock @@ -287,6 +288,12 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -296,6 +303,11 @@ class HttpTestCase(unittest.TestCase): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + # https://twistedmatrix.com/trac/ticket/10279 + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) # client connects, server send headers and some body bytes but hangs spider = Spider('foo') meta = {'download_timeout': 0.5} @@ -1055,6 +1067,10 @@ class BaseFTPTestCase(unittest.TestCase): class FTPTestCase(BaseFTPTestCase): def test_invalid_credentials(self): + if self.reactor_pytest == "asyncio" and sys.platform == "win32": + raise unittest.SkipTest( + "This test produces DirtyReactorAggregateError on Windows with asyncio" + ) from twisted.protocols.ftp import ConnectionLost meta = dict(self.req_meta) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a2114bd18..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,6 +1,4 @@ -import platform -import sys -from unittest import skipIf, TestCase +from unittest import TestCase from pytest import mark @@ -14,9 +12,6 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') - # https://twistedmatrix.com/trac/ticket/9766 - @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), - "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") From 4bdaa54af4470582845206a9fa21779f80a4b123 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 28 Jan 2022 15:39:32 +0200 Subject: [PATCH 29/44] response_httprepr memory issue fixed (#4972) * response_httprepr replaced by response.body * unused import deleted * get_header_size function added * response size calculation updated * flake8 codestyle fix * added counting status code, line breaks to response size * get_status size: list to tuple, comments added * test added: comparing new response size counting method with old `len(response_httprepr)` * downloader stats : unreachable code deleted * `get_status_size` optimized * comment added * tests.test_downloadermiddleware_stats: statement formatting updated * scrapy.utils.response: `response_httprepr` -> deprecated * tests.test_downloadermiddleware_stats: flake8 fix --- scrapy/downloadermiddlewares/stats.py | 22 +++++++++++++++++++--- scrapy/utils/response.py | 2 ++ tests/test_downloadermiddleware_stats.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 5479cd0e2..25fb1ed9d 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,7 +1,22 @@ from scrapy.exceptions import NotConfigured +from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from scrapy.utils.response import response_httprepr -from scrapy.utils.python import global_object_name + +from twisted.web import http + + +def get_header_size(headers): + size = 0 + for key, value in headers.items(): + if isinstance(value, (list, tuple)): + for v in value: + size += len(b": ") + len(key) + len(v) + return size + len(b'\r\n') * (len(headers.keys()) - 1) + + +def get_status_size(response_status): + return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15 + # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> " class DownloaderStats: @@ -24,7 +39,8 @@ class DownloaderStats: def process_response(self, request, response, spider): self.stats.inc_value('downloader/response_count', spider=spider) self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider) - reslen = len(response_httprepr(response)) + reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4 + # response.body + b"\r\n"+ response.header + b"\r\n" + response.status self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8b109dced..741dce350 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -14,6 +14,7 @@ from scrapy.http.response import Response from twisted.web import http from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.decorators import deprecated from w3lib import html @@ -51,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str: return f'{status_int} {to_unicode(message)}' +@deprecated def response_httprepr(response: Response) -> bytes: """Return raw HTTP representation (as bytes) of the given response. This is provided only for reference, since it's not the exact stream of bytes diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 1f2616e35..9e75f0a50 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,8 +1,10 @@ +from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler @@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase): self.mw.process_response(self.req, self.res, self.spider) self.assertStatsEqual('downloader/response_count', 1) + def test_response_len(self): + body = (b'', b'not_empty') # empty/notempty body + headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'}) # 0 headers, 1h and 2h + test_responses = [ # form test responses with all combinations of body/headers + Response( + url='scrapytest.org', + status=200, + body=r[0], + headers=r[1] + ) + for r in product(body, headers) + ] + for test_response in test_responses: + self.crawler.stats.set_value('downloader/response_bytes', 0) + self.mw.process_response(self.req, test_response, self.spider) + self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) self.assertStatsEqual('downloader/exception_count', 1) From 30d5779ea94ed1e9343a4590895a3f5e65e444b9 Mon Sep 17 00:00:00 2001 From: "Sixuan (Cherie) Wu" <73203695+inspurwusixuan@users.noreply.github.com> Date: Fri, 28 Jan 2022 09:30:30 -0800 Subject: [PATCH 30/44] Fix FEED_URI_PARAMS: custom params throws KeyError (#4966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix FEED_URI_PARAMS: custom params throws KeyError closes #4962 * another try FEED_URI_PARAMS * add warning message and change default function * Add tests for FEED_URI_PARAMS * FEED_URI_PARAMS: warn if the params dict has been modified in-place * [Doc] FEED_URI_PARAMS: modifying params in-place is deprecated * Remove whileline * Rename parameters for lambda function * Type hints for FeedExporter._get_uri_params Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- docs/topics/feed-exports.rst | 7 +- scrapy/extensions/feedexport.py | 26 +++-- tests/test_feedexport.py | 163 ++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 116967280..7994027d2 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -322,7 +322,7 @@ Post-Processing Scrapy provides an option to activate plugins to post-process feeds before they are exported to feed storages. In addition to using :ref:`builtin plugins `, you -can create your own :ref:`plugins `. +can create your own :ref:`plugins `. These plugins can be activated through the ``postprocessing`` option of a feed. The option must be passed a list of post-processing plugins in the order you want @@ -366,7 +366,7 @@ Each plugin is a class that must implement the following methods: Close the target file object. -To pass a parameter to your plugin, use :ref:`feed options `. You +To pass a parameter to your plugin, use :ref:`feed options `. You can then access those parameters from the ``__init__`` method of your plugin. @@ -744,6 +744,9 @@ The function signature should be as follows: :param spider: source spider of the feed items :type spider: scrapy.Spider + .. caution:: The function should return a new dictionary, modifying + the received ``params`` in-place is deprecated. + For example, to include the :attr:`name ` of the source spider in the feed URI: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 370723368..e7097b7a1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,14 +11,14 @@ import sys import warnings from datetime import datetime from tempfile import NamedTemporaryFile -from typing import Any, Optional, Tuple +from typing import Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from zope.interface import implementer, Interface -from scrapy import signals +from scrapy import signals, Spider from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available @@ -524,7 +524,12 @@ class FeedExporter: raise TypeError(f"{feedcls.__qualname__}.{method_name} returned None") return instance - def _get_uri_params(self, spider, uri_params, slot=None): + def _get_uri_params( + self, + spider: Spider, + uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], + slot: Optional[_FeedSlot] = None, + ) -> dict: params = {} for k in dir(spider): params[k] = getattr(spider, k) @@ -532,9 +537,18 @@ class FeedExporter: params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - uripar_function = load_object(uri_params) if uri_params else lambda x, y: None - uripar_function(params, spider) - return params + original_params = params.copy() + uripar_function = load_object(uri_params_function) if uri_params_function else lambda params, _: params + new_params = uripar_function(params, spider) + if new_params is None or original_params != params: + warnings.warn( + 'Modifying the params dictionary in-place in the function defined in ' + 'the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS ' + 'setting is deprecated. The function must return a new dictionary ' + 'instead.', + category=ScrapyDeprecationWarning + ) + return new_params if new_params is not None else params def _load_filter(self, feed_options): # load the item filter if declared else load the default filter class diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 253f3119c..f0acf1941 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2608,3 +2608,166 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): ), ) ) + + +class URIParamsTest: + + spider_name = "uri_params_spider" + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + raise NotImplementedError + + def test_default(self): + settings = self.build_settings( + uri='file:///tmp/%(name)s', + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_none(self): + def uri_params(params, spider): + pass + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + 'Modifying the params dictionary in-place in the ' + 'function defined in the FEED_URI_PARAMS setting or ' + 'in the uri_params key of the FEEDS setting is ' + 'deprecated. The function must return a new ' + 'dictionary instead.' + ), + ) + ) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_empty_dict(self): + def uri_params(params, spider): + return {} + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + with self.assertRaises(KeyError): + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + def test_params_as_is(self): + def uri_params(params, spider): + return params + + settings = self.build_settings( + uri='file:///tmp/%(name)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + def test_custom_param(self): + def uri_params(params, spider): + return {**params, 'foo': self.spider_name} + + settings = self.build_settings( + uri='file:///tmp/%(foo)s', + uri_params=uri_params, + ) + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider(self.spider_name) + spider.crawler = crawler + with warnings.catch_warnings(record=True) as w: + feed_exporter.open_spider(spider) + messages = tuple( + str(item.message) for item in w + if item.category is ScrapyDeprecationWarning + ) + self.assertEqual(messages, tuple()) + + self.assertEqual( + feed_exporter.slots[0].uri, + f'file:///tmp/{self.spider_name}' + ) + + +class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + extra_settings = {} + if uri_params: + extra_settings['FEED_URI_PARAMS'] = uri_params + return { + 'FEED_URI': uri, + **extra_settings, + } + + +class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + + def build_settings(self, uri='file:///tmp/foobar', uri_params=None): + options = { + 'format': 'jl', + } + if uri_params: + options['uri_params'] = uri_params + return { + 'FEEDS': { + uri: options, + }, + } From fe43411bc4d0164a0f0ecc596c23b59c99d31f17 Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Fri, 4 Feb 2022 05:57:57 -0300 Subject: [PATCH 31/44] Fix TypeError on using pathlib.Path as key on FEEDS settings (#5384) --- scrapy/settings/__init__.py | 6 +++++- tests/test_cmdline/__init__.py | 4 ++++ tests/test_cmdline/settings.py | 9 +++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..6b1ad0828 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -375,9 +375,13 @@ class BaseSettings(MutableMapping): return len(self.attributes) def _to_dict(self): - return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + return {self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items()} + def _get_key(self, key_value): + return (key_value if isinstance(key_value, (bool, float, int, str, type(None))) + else str(key_value)) + def copy_to_dict(self): """ Make a copy of current settings and convert to a dict. diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 591075a98..8233e0101 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -64,3 +64,7 @@ class CmdlineTest(unittest.TestCase): settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) + + def test_pathlib_path_as_feeds_key(self): + self.assertEqual(self._execute('settings', '--get', 'FEEDS'), + json.dumps({"items.csv": {"format": "csv", "fields": ["price", "name"]}})) diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py index 8a719ddf2..b0ac6e98b 100644 --- a/tests/test_cmdline/settings.py +++ b/tests/test_cmdline/settings.py @@ -1,5 +1,14 @@ +from pathlib import Path + EXTENSIONS = { 'tests.test_cmdline.extensions.TestExtension': 0, } TEST1 = 'default' + +FEEDS = { + Path('items.csv'): { + 'format': 'csv', + 'fields': ['price', 'name'], + }, +} From 9be878fc09cf71bb2cb98695f5042cb344bd2e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 4 Feb 2022 12:27:39 +0100 Subject: [PATCH 32/44] CI: stop using tox-pip-version (#5389) --- .github/workflows/checks.yml | 4 ---- .github/workflows/tests-ubuntu.yml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80df9469d..98fa44c7f 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,6 @@ jobs: - python-version: 3.8 env: TOXENV: pylint - TOX_PIP_VERSION: 20.3.3 - python-version: 3.6 env: TOXENV: typing @@ -38,8 +37,5 @@ jobs: - name: Run check env: ${{ matrix.env }} run: | - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ea50e644..1fc8d914b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -46,7 +46,6 @@ jobs: - python-version: 3.8 env: TOXENV: extra-deps - TOX_PIP_VERSION: 20.3.3 steps: - uses: actions/checkout@v2 @@ -73,9 +72,6 @@ jobs: $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [[ ! -z "$TOX_PIP_VERSION" ]]; then - pip install tox-pip-version - fi pip install -U tox tox From 55ae2109c95e497d4a730afeb9caf71aa78a7723 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 5 Feb 2022 13:02:02 -0300 Subject: [PATCH 33/44] Remove deprecated TextResponse.body_as_unicode --- scrapy/http/response/text.py | 9 --------- tests/test_http_response.py | 18 ------------------ 2 files changed, 27 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 27bd55c07..89516b9b6 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,7 +6,6 @@ See documentation in docs/topics/request-response.rst """ import json -import warnings from contextlib import suppress from typing import Generator, Tuple from urllib.parse import urljoin @@ -16,7 +15,6 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -66,13 +64,6 @@ class TextResponse(Response): or self._body_declared_encoding() ) - def body_as_unicode(self): - """Return body as unicode""" - warnings.warn('Response.body_as_unicode() is deprecated, ' - 'please use Response.text instead.', - ScrapyDeprecationWarning, stacklevel=2) - return self.text - def json(self): """ .. versionadded:: 2.2 diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 0ec5257e1..2986f884f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,10 +1,8 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings from w3lib.encoding import resolve_encoding -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -134,9 +132,6 @@ class BaseResponseTest(unittest.TestCase): assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertEqual(response.body_as_unicode(), body_unicode) self.assertEqual(response.text, body_unicode) def _assert_response_encoding(self, response, encoding): @@ -346,12 +341,6 @@ class TextResponseTest(BaseResponseTest): original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') - # check body_as_unicode - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(r1.body_as_unicode(), str)) - self.assertEqual(r1.body_as_unicode(), unicode_string) - # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) @@ -683,13 +672,6 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') - def test_body_as_unicode_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), 'Hello') - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - def test_json_response(self): json_body = b"""{"ip": "109.187.217.200"}""" json_response = self.response_class("http://www.example.com", body=json_body) From 38d2a154ec79767558f699ec663697ccc7f64ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Ferenc=20Gyarmati?= Date: Sun, 6 Feb 2022 18:52:15 +0100 Subject: [PATCH 34/44] docs: use https scheme for each quotes.toscrape.com url occurrence --- docs/intro/examples.rst | 2 +- docs/intro/overview.rst | 4 +-- docs/intro/tutorial.rst | 52 ++++++++++++++++----------------- docs/topics/developer-tools.rst | 20 ++++++------- docs/topics/logging.rst | 2 +- docs/topics/settings.rst | 4 +-- docs/topics/signals.rst | 2 +- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst index 96363c7d5..edff894c6 100644 --- a/docs/intro/examples.rst +++ b/docs/intro/examples.rst @@ -7,7 +7,7 @@ Examples The best way to learn is with examples, and Scrapy is no exception. For this reason, there is an example Scrapy project named quotesbot_, that you can use to play and learn more about Scrapy. It contains two spiders for -http://quotes.toscrape.com, one using CSS selectors and another one using XPath +https://quotes.toscrape.com, one using CSS selectors and another one using XPath expressions. The quotesbot_ project is available at: https://github.com/scrapy/quotesbot. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 405bf845d..f3d652621 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,7 +20,7 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -http://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination:: import scrapy @@ -28,7 +28,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): name = 'quotes' start_urls = [ - 'http://quotes.toscrape.com/tag/humor/', + 'https://quotes.toscrape.com/tag/humor/', ] def parse(self, response): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ca5856881..5697b9608 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -7,7 +7,7 @@ Scrapy Tutorial In this tutorial, we'll assume that Scrapy is already installed on your system. If that's not the case, see :ref:`intro-install`. -We are going to scrape `quotes.toscrape.com `_, a website +We are going to scrape `quotes.toscrape.com `_, a website that lists quotes from famous authors. This tutorial will walk you through these tasks: @@ -93,8 +93,8 @@ This is the code for our first Spider. Save it in a file named def start_requests(self): urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -143,9 +143,9 @@ similar to this:: 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Spider opened 2016-12-16 21:24:05 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) 2016-12-16 21:24:05 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023 - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (404) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-12-16 21:24:05 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-1.html 2016-12-16 21:24:05 [quotes] DEBUG: Saved file quotes-2.html 2016-12-16 21:24:05 [scrapy.core.engine] INFO: Closing spider (finished) @@ -184,8 +184,8 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -207,7 +207,7 @@ Extracting data The best way to learn how to extract data with Scrapy is trying selectors using the :ref:`Scrapy shell `. Run:: - scrapy shell 'http://quotes.toscrape.com/page/1/' + scrapy shell 'https://quotes.toscrape.com/page/1/' .. note:: @@ -217,18 +217,18 @@ using the :ref:`Scrapy shell `. Run:: On Windows, use double quotes instead:: - scrapy shell "http://quotes.toscrape.com/page/1/" + scrapy shell "https://quotes.toscrape.com/page/1/" You will see something like:: [ ... Scrapy log here ... ] - 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2016-09-19 12:09:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) [s] Available Scrapy objects: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request - [s] response <200 http://quotes.toscrape.com/page/1/> + [s] request + [s] response <200 https://quotes.toscrape.com/page/1/> [s] settings [s] spider [s] Useful shortcuts: @@ -241,7 +241,7 @@ object: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html') + response = load_response('https://quotes.toscrape.com/page/1/', 'quotes1.html') >>> response.css('title') [] @@ -355,7 +355,7 @@ Extracting quotes and authors Now that you know a bit about selection and extraction, let's complete our spider by writing the code to extract the quotes from the web page. -Each quote in http://quotes.toscrape.com is represented by HTML elements that look +Each quote in https://quotes.toscrape.com is represented by HTML elements that look like this: .. code-block:: html @@ -379,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'http://quotes.toscrape.com' + $ scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: @@ -444,8 +444,8 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', - 'http://quotes.toscrape.com/page/2/', + 'https://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/2/', ] def parse(self, response): @@ -458,9 +458,9 @@ in the callback, as you can see below:: If you run this spider, it will output the extracted data with the log:: - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['life', 'love'], 'author': 'André Gide', 'text': '“It is better to be hated for what you are than to be loved for what you are not.”'} - 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 http://quotes.toscrape.com/page/1/> + 2016-09-19 18:57:19 [scrapy.core.scraper] DEBUG: Scraped from <200 https://quotes.toscrape.com/page/1/> {'tags': ['edison', 'failure', 'inspirational', 'paraphrased'], 'author': 'Thomas A. Edison', 'text': "“I have not failed. I've just found 10,000 ways that won't work.”"} @@ -505,7 +505,7 @@ Following links =============== Let's say, instead of just scraping the stuff from the first two pages -from http://quotes.toscrape.com, you want quotes from all the pages in the website. +from https://quotes.toscrape.com, you want quotes from all the pages in the website. Now that you know how to extract data from pages, let's see how to follow links from them. @@ -549,7 +549,7 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -600,7 +600,7 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'http://quotes.toscrape.com/page/1/', + 'https://quotes.toscrape.com/page/1/', ] def parse(self, response): @@ -654,7 +654,7 @@ this time for scraping author information:: class AuthorSpider(scrapy.Spider): name = 'author' - start_urls = ['http://quotes.toscrape.com/'] + start_urls = ['https://quotes.toscrape.com/'] def parse(self, response): author_page_links = response.css('.author + a') @@ -727,7 +727,7 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'http://quotes.toscrape.com/' + url = 'https://quotes.toscrape.com/' tag = getattr(self, 'tag', None) if tag is not None: url = url + 'tag/' + tag @@ -747,7 +747,7 @@ with a specific tag, building the URL based on the argument:: If you pass the ``tag=humor`` argument to this spider, you'll notice that it will only visit URLs from the ``humor`` tag, such as -``http://quotes.toscrape.com/tag/humor``. +``https://quotes.toscrape.com/tag/humor``. You can :ref:`learn more about handling spider arguments here `. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 057b1ec62..96475899f 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -81,18 +81,18 @@ clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you copy XPaths to selected elements. Let's try it out. -First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal: +First open the Scrapy shell at https://quotes.toscrape.com/ in a terminal: .. code-block:: none - $ scrapy shell "http://quotes.toscrape.com/" + $ scrapy shell "https://quotes.toscrape.com/" Then, back to your web browser, right-click on the ``span`` tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so: .. invisible-code-block: python - response = load_response('http://quotes.toscrape.com/', 'quotes.html') + response = load_response('https://quotes.toscrape.com/', 'quotes.html') >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() ['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] @@ -227,7 +227,7 @@ interests us is the one request called ``quotes?page=1`` with the type ``json``. If we click on this request, we see that the request URL is -``http://quotes.toscrape.com/api/quotes?page=1`` and the response +``https://quotes.toscrape.com/api/quotes?page=1`` and the response is a JSON-object that contains our quotes. We can also right-click on the request and open ``Open in new tab`` to get a better overview. @@ -247,7 +247,7 @@ also request each page to get every quote on the site:: name = 'quote' allowed_domains = ['quotes.toscrape.com'] page = 1 - start_urls = ['http://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] def parse(self, response): data = json.loads(response.text) @@ -255,7 +255,7 @@ also request each page to get every quote on the site:: yield {"quote": quote["text"]} if data["has_next"]: self.page += 1 - url = f"http://quotes.toscrape.com/api/quotes?page={self.page}" + url = f"https://quotes.toscrape.com/api/quotes?page={self.page}" yield scrapy.Request(url=url, callback=self.parse) This spider starts at the first page of the quotes-API. With each @@ -280,7 +280,7 @@ request:: from scrapy import Request request = Request.from_curl( - "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" + "curl 'https://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce" "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X" "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" @@ -304,8 +304,8 @@ daunting and pages can be very complex, but it (mostly) boils down to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools -.. _quotes.toscrape.com: http://quotes.toscrape.com -.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll -.. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 +.. _quotes.toscrape.com: https://quotes.toscrape.com +.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll +.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index d593c74c6..3bf23d5f5 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -218,7 +218,7 @@ For example, let's say you're scraping a website which returns many HTTP 404 and 500 responses, and you want to hide all messages like this:: 2016-12-16 22:00:06 [scrapy.spidermiddlewares.httperror] INFO: Ignoring - response <500 http://quotes.toscrape.com/page/1-34/>: HTTP status code + response <500 https://quotes.toscrape.com/page/1-34/>: HTTP status code is not handled or not allowed The first thing to note is a logger name - it is in brackets: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f6c95c502..4e105642d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1597,7 +1597,7 @@ In order to use the reactor installed by Scrapy:: def start_requests(self): reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) @@ -1625,7 +1625,7 @@ which raises :exc:`Exception`, becomes:: from twisted.internet import reactor reactor.callLater(self.timeout, self.stop) - urls = ['http://quotes.toscrape.com/page/1'] + urls = ['https://quotes.toscrape.com/page/1'] for url in urls: yield scrapy.Request(url=url, callback=self.parse) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 63ad3a9ad..2fbd0b51c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -60,7 +60,7 @@ Let's take an example:: class SignalSpider(scrapy.Spider): name = 'signals' - start_urls = ['http://quotes.toscrape.com/page/1/'] + start_urls = ['https://quotes.toscrape.com/page/1/'] @classmethod def from_crawler(cls, crawler, *args, **kwargs): From bbfa185664cef79299b48cb0ae22065439bb07fc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 6 Feb 2022 18:12:28 -0300 Subject: [PATCH 35/44] Remove deprecated BaseItem class --- scrapy/exporters.py | 4 +-- scrapy/item.py | 34 ++------------------ scrapy/utils/misc.py | 4 +-- tests/test_item.py | 75 +------------------------------------------- 4 files changed, 8 insertions(+), 109 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 36cca2d05..1c26e81db 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -13,7 +13,7 @@ from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -315,7 +315,7 @@ class PythonItemExporter(BaseItemExporter): return serializer(value) def _serialize_value(self, value): - if isinstance(value, _BaseItem): + if isinstance(value, Item): return self.export_item(value) elif is_item(value): return dict(self._serialize_item(value)) diff --git a/scrapy/item.py b/scrapy/item.py index 2ccd7ad18..839bee3fa 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -15,39 +15,11 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -class _BaseItem(object_ref): - """ - Temporary class used internally to avoid the deprecation - warning raised by isinstance checks using BaseItem. - """ - pass - - -class _BaseItemMeta(ABCMeta): - def __instancecheck__(cls, instance): - if cls is BaseItem: - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__instancecheck__(instance) - - -class BaseItem(_BaseItem, metaclass=_BaseItemMeta): - """ - Deprecated, please use :class:`scrapy.item.Item` instead - """ - - def __new__(cls, *args, **kwargs): - if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): - warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) - - class Field(dict): """Container of field metadata""" -class ItemMeta(_BaseItemMeta): +class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. .. _metaclass: https://realpython.com/python-metaclasses @@ -74,7 +46,7 @@ class ItemMeta(_BaseItemMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, BaseItem): +class DictItem(MutableMapping, object_ref): fields: Dict[str, Field] = {} @@ -118,7 +90,7 @@ class DictItem(MutableMapping, BaseItem): def __iter__(self): return iter(self._values) - __hash__ = BaseItem.__hash__ + __hash__ = object_ref.__hash__ def keys(self): return self._values.keys() diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 11c4206c2..1221b39b2 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -14,11 +14,11 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode -from scrapy.item import _BaseItem +from scrapy.item import Item from scrapy.utils.deprecate import ScrapyDeprecationWarning -_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes +_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes def arg_to_iter(arg): diff --git a/tests/test_item.py b/tests/test_item.py index c94bb44af..7d82fbffe 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -3,7 +3,7 @@ from unittest import mock from warnings import catch_warnings, filterwarnings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -318,79 +318,6 @@ class DictItemTest(unittest.TestCase): self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) -class BaseItemTest(unittest.TestCase): - - def test_isinstance_check(self): - - class SubclassedBaseItem(BaseItem): - pass - - class SubclassedItem(Item): - pass - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), BaseItem)) - self.assertTrue(isinstance(Item(), BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), BaseItem)) - - # make sure internal checks using private _BaseItem class succeed - self.assertTrue(isinstance(BaseItem(), _BaseItem)) - self.assertTrue(isinstance(SubclassedBaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), _BaseItem)) - self.assertTrue(isinstance(SubclassedItem(), _BaseItem)) - - def test_deprecation_warning(self): - """ - Make sure deprecation warnings are logged whenever BaseItem is used, - either instantiated or in an isinstance check - """ - with catch_warnings(record=True) as warnings: - BaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - - class SubclassedBaseItem(BaseItem): - pass - - SubclassedBaseItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertFalse(isinstance("foo", BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - with catch_warnings(record=True) as warnings: - self.assertTrue(isinstance(BaseItem(), BaseItem)) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - -class ItemNoDeprecationWarningTest(unittest.TestCase): - def test_no_deprecation_warning(self): - """ - Make sure deprecation warnings are NOT logged whenever BaseItem subclasses are used. - """ - class SubclassedItem(Item): - pass - - with catch_warnings(record=True) as warnings: - Item() - SubclassedItem() - _BaseItem() - self.assertFalse(isinstance("foo", _BaseItem)) - self.assertFalse(isinstance("foo", Item)) - self.assertFalse(isinstance("foo", SubclassedItem)) - self.assertTrue(isinstance(_BaseItem(), _BaseItem)) - self.assertTrue(isinstance(Item(), Item)) - self.assertTrue(isinstance(SubclassedItem(), SubclassedItem)) - self.assertEqual(len(warnings), 0) - if __name__ == "__main__": unittest.main() From c8c1edd43b04ce7a2d9b1da198af26ba271cb1d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 6 Feb 2022 18:27:41 -0300 Subject: [PATCH 36/44] Flake8 adjustments --- tests/test_item.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_item.py b/tests/test_item.py index 7d82fbffe..a12e425e0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,6 +1,6 @@ import unittest from unittest import mock -from warnings import catch_warnings, filterwarnings +from warnings import catch_warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta @@ -318,6 +318,5 @@ class DictItemTest(unittest.TestCase): self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - if __name__ == "__main__": unittest.main() From fca49cca929de035fb5d179d83f7f79da22fd205 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 6 Feb 2022 18:31:55 -0300 Subject: [PATCH 37/44] Remove deprecated DictItem class --- docs/conf.py | 1 - scrapy/item.py | 55 +++++++++++++++++++--------------------------- tests/test_item.py | 31 +------------------------- 3 files changed, 23 insertions(+), 64 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 406c4d94a..d5e139e66 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -272,7 +272,6 @@ coverage_ignore_pyobjects = [ r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]', # helper functions # Never documented before, and deprecated now. - r'^scrapy\.item\.DictItem$', r'^scrapy\.linkextractors\.FilteringLinkExtractor$', # Implementation detail of LxmlLinkExtractor diff --git a/scrapy/item.py b/scrapy/item.py index 839bee3fa..2521ac829 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -9,9 +9,7 @@ from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat from typing import Dict -from warnings import warn -from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -46,15 +44,30 @@ class ItemMeta(ABCMeta): return super().__new__(mcs, class_name, bases, new_attrs) -class DictItem(MutableMapping, object_ref): +class Item(MutableMapping, object_ref, metaclass=ItemMeta): + """ + Base class for scraped items. - fields: Dict[str, Field] = {} + In Scrapy, an object is considered an ``item`` if it is an instance of either + :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a + spider callback is evaluated, only instances of :class:`Item` or + :class:`dict` are passed to :ref:`item pipelines `. - def __new__(cls, *args, **kwargs): - if issubclass(cls, DictItem) and not issubclass(cls, Item): - warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) + If you need instances of a custom class to be considered items by Scrapy, + you must inherit from either :class:`Item` or :class:`dict`. + + Items must declare :class:`Field` attributes, which are processed and stored + in the ``fields`` attribute. This restricts the set of allowed field names + and prevents typos, raising ``KeyError`` when referring to undefined fields. + Additionally, fields can be used to define metadata and control the way + data is processed internally. Please refer to the :ref:`documentation + about fields ` for additional information. + + Unlike instances of :class:`dict`, instances of :class:`Item` may be + :ref:`tracked ` to debug memory leaks. + """ + + fields: Dict[str, Field] def __init__(self, *args, **kwargs): self._values = {} @@ -105,27 +118,3 @@ class DictItem(MutableMapping, object_ref): """Return a :func:`~copy.deepcopy` of this item. """ return deepcopy(self) - - -class Item(DictItem, metaclass=ItemMeta): - """ - Base class for scraped items. - - In Scrapy, an object is considered an ``item`` if it is an instance of either - :class:`Item` or :class:`dict`, or any subclass. For example, when the output of a - spider callback is evaluated, only instances of :class:`Item` or - :class:`dict` are passed to :ref:`item pipelines `. - - If you need instances of a custom class to be considered items by Scrapy, - you must inherit from either :class:`Item` or :class:`dict`. - - Items must declare :class:`Field` attributes, which are processed and stored - in the ``fields`` attribute. This restricts the set of allowed field names - and prevents typos, raising ``KeyError`` when referring to undefined fields. - Additionally, fields can be used to define metadata and control the way - data is processed internally. Please refer to the :ref:`documentation - about fields ` for additional information. - - Unlike instances of :class:`dict`, instances of :class:`Item` may be - :ref:`tracked ` to debug memory leaks. - """ diff --git a/tests/test_item.py b/tests/test_item.py index a12e425e0..25f2aea0a 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,9 +1,7 @@ import unittest from unittest import mock -from warnings import catch_warnings -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta +from scrapy.item import ABCMeta, Field, Item, ItemMeta class ItemTest(unittest.TestCase): @@ -254,18 +252,6 @@ class ItemTest(unittest.TestCase): item['tags'].append('tag2') assert item['tags'] != copied_item['tags'] - def test_dictitem_deprecation_warning(self): - """Make sure the DictItem deprecation warning is not issued for - Item""" - with catch_warnings(record=True) as warnings: - Item() - self.assertEqual(len(warnings), 0) - - class SubclassedItem(Item): - pass - SubclassedItem() - self.assertEqual(len(warnings), 0) - class ItemMetaTest(unittest.TestCase): @@ -303,20 +289,5 @@ class ItemMetaClassCellRegression(unittest.TestCase): super().__init__(*args, **kwargs) -class DictItemTest(unittest.TestCase): - - def test_deprecation_warning(self): - with catch_warnings(record=True) as warnings: - DictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedDictItem(DictItem): - pass - SubclassedDictItem() - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - - if __name__ == "__main__": unittest.main() From b282a7af012a4804eb91bdd850df3b86065b3fd6 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 8 Feb 2022 01:25:08 +0500 Subject: [PATCH 38/44] Temporarily pin Twisted to an older version in CI (#5401) --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 2031a2d92..cf7855cf9 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,8 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 + # Temporary until the tests are updated + Twisted<22.1.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From 4bda0976b28917405f54c781afda5ea55b65b16b Mon Sep 17 00:00:00 2001 From: Laerte <5853172+Laerte@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:57:19 -0300 Subject: [PATCH 39/44] Fix csviter call, add parse_rows test (#5394) --- scrapy/spiders/feed.py | 2 +- tests/test_spider.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index bef2d6b24..79e12e030 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -123,7 +123,7 @@ class CSVFeedSpider(Spider): process_results methods for pre and post-processing purposes. """ - for row in csviter(response, self.delimiter, self.headers, self.quotechar): + for row in csviter(response, self.delimiter, self.headers, quotechar=self.quotechar): ret = iterate_spider_output(self.parse_row(response, row)) for result_item in self.process_results(response, ret): yield result_item diff --git a/tests/test_spider.py b/tests/test_spider.py index a7c3ee048..689349999 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -21,6 +21,7 @@ from scrapy.spiders import ( ) from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler +from tests import get_testdata class SpiderTest(unittest.TestCase): @@ -167,6 +168,23 @@ class CSVFeedSpiderTest(SpiderTest): spider_class = CSVFeedSpider + def test_parse_rows(self): + body = get_testdata('feeds', 'feed-sample6.csv') + response = Response("http://example.org/dummy.csv", body=body) + + class _CrawlSpider(self.spider_class): + name = "test" + delimiter = "," + quotechar = "'" + + def parse_row(self, response, row): + return row + + spider = _CrawlSpider() + rows = list(spider.parse_rows(response)) + assert rows[0] == {'id': '1', 'name': 'alpha', 'value': 'foobar'} + assert len(rows) == 4 + class CrawlSpiderTest(SpiderTest): From fd55f62207bbbb18d7758c8e2ef46fe9115eb2c5 Mon Sep 17 00:00:00 2001 From: Raihan Nismara <31585789+raihan71@users.noreply.github.com> Date: Tue, 8 Feb 2022 21:36:25 +0700 Subject: [PATCH 40/44] Update Logo in README.rst (#5258) --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 05f10bb6c..6b563d638 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,4 @@ -.. image:: /artwork/scrapy-logo.jpg - :width: 400px +.. image:: https://scrapy.org/img/scrapylogo.png ====== Scrapy From 1e1cfc26dbdde5fc3035169884c4d1218844d0de Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 8 Feb 2022 21:01:16 +0500 Subject: [PATCH 41/44] Copy resource classes from twisted.web.test.test_webclient. --- pytest.ini | 2 - tests/mockserver.py | 74 ++++++++++++++++++++++++++++--- tests/test_downloader_handlers.py | 12 +++-- tests/test_webclient.py | 18 ++++---- 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/pytest.ini b/pytest.ini index fa5d6b34f..ae2ed2029 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,5 +21,3 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed -filterwarnings= - ignore::DeprecationWarning:twisted.web.test.test_webclient diff --git a/tests/mockserver.py b/tests/mockserver.py index ab9aec6a6..72d7e0241 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -14,10 +14,9 @@ from twisted.internet import defer, reactor, ssl from twisted.internet.task import deferLater from twisted.names import dns, error from twisted.names.server import DNSServerFactory -from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web import resource, server from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File -from twisted.web.test.test_webclient import PayloadResource from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode @@ -35,7 +34,70 @@ def getarg(request, name, default=None, type=None): return default -class LeafResource(Resource): +# most of the following resources are copied from twisted.web.test.test_webclient +class ForeverTakingResource(resource.Resource): + """ + L{ForeverTakingResource} is a resource which never finishes responding + to requests. + """ + + def __init__(self, write=False): + resource.Resource.__init__(self) + self._write = write + + def render(self, request): + if self._write: + request.write(b"some bytes") + return server.NOT_DONE_YET + + +class ErrorResource(resource.Resource): + def render(self, request): + request.setResponseCode(401) + if request.args.get(b"showlength"): + request.setHeader(b"content-length", b"0") + return b"" + + +class NoLengthResource(resource.Resource): + def render(self, request): + return b"nolength" + + +class HostHeaderResource(resource.Resource): + """ + A testing resource which renders itself as the value of the host header + from the request. + """ + + def render(self, request): + return request.requestHeaders.getRawHeaders(b"host")[0] + + +class PayloadResource(resource.Resource): + """ + A testing resource which renders itself as the contents of the request body + as long as the request body is 100 bytes long, otherwise which renders + itself as C{"ERROR"}. + """ + + def render(self, request): + data = request.content.read() + contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0] + if len(data) != 100 or int(contentLength) != 100: + return b"ERROR" + return data + + +class BrokenDownloadResource(resource.Resource): + def render(self, request): + # only sends 3 bytes even though it claims to send 5 + request.setHeader(b"content-length", b"5") + request.write(b"abc") + return b"" + + +class LeafResource(resource.Resource): isLeaf = True @@ -175,10 +237,10 @@ class ArbitraryLengthPayloadResource(LeafResource): return request.content.read() -class Root(Resource): +class Root(resource.Resource): def __init__(self): - Resource.__init__(self) + resource.Resource.__init__(self) self.putChild(b"status", Status()) self.putChild(b"follow", Follow()) self.putChild(b"delay", Delay()) @@ -187,7 +249,7 @@ class Root(Resource): self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"xpayload", resource.EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a1ea4c679..2bb53950d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -15,8 +15,6 @@ from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed from twisted.web.http import _DataLoss -from twisted.web.test.test_webclient import (ForeverTakingResource, HostHeaderResource, - NoLengthResource, PayloadResource) from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers @@ -34,7 +32,15 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto -from tests.mockserver import MockServer, ssl_context_factory, Echo +from tests.mockserver import ( + Echo, + ForeverTakingResource, + HostHeaderResource, + MockServer, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) from tests.spiders import SingleRequestSpider diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 6e4cb9b6e..a6d55cb38 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -21,14 +21,6 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks -from twisted.web.test.test_webclient import ( - ForeverTakingResource, - ErrorResource, - NoLengthResource, - HostHeaderResource, - PayloadResource, - BrokenDownloadResource, -) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -36,7 +28,15 @@ from scrapy.http import Request, Headers from scrapy.settings import Settings from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes, to_unicode -from tests.mockserver import ssl_context_factory +from tests.mockserver import ( + BrokenDownloadResource, + ErrorResource, + ForeverTakingResource, + HostHeaderResource, + NoLengthResource, + PayloadResource, + ssl_context_factory, +) def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): From 77547a1ab554a5b9afa7d7a343d8f90ef1d4cfe8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 8 Feb 2022 21:06:02 +0500 Subject: [PATCH 42/44] Revert "Temporarily pin Twisted to an older version in CI (#5401)" This reverts commit b282a7af012a4804eb91bdd850df3b86065b3fd6. --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index cf7855cf9..2031a2d92 100644 --- a/tox.ini +++ b/tox.ini @@ -19,8 +19,6 @@ deps = mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # Extras botocore>=1.4.87 - # Temporary until the tests are updated - Twisted<22.1.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From e2e2ffd0d162cfed5a2e82e9fb9472dbf233c919 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 9 Feb 2022 11:52:07 -0800 Subject: [PATCH 43/44] Move from optparse to argparse (#5374) --- scrapy/cmdline.py | 14 +++---- scrapy/commands/__init__.py | 75 ++++++++++++++++++++++++------------ scrapy/commands/check.py | 8 ++-- scrapy/commands/fetch.py | 10 ++--- scrapy/commands/genspider.py | 20 +++++----- scrapy/commands/parse.py | 44 ++++++++++----------- scrapy/commands/settings.py | 20 +++++----- scrapy/commands/shell.py | 12 +++--- scrapy/commands/version.py | 4 +- scrapy/commands/view.py | 3 +- tests/test_command_parse.py | 19 +++++++++ tests/test_commands.py | 38 ++++++++++++++---- 12 files changed, 168 insertions(+), 99 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 91482ce01..491c4beab 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,13 +1,13 @@ import sys import os -import optparse +import argparse import cProfile import inspect import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -123,8 +123,6 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) sys.exit(0) @@ -133,12 +131,14 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser.usage = f"scrapy {cmdname} {cmd.syntax()}" - parser.description = cmd.long_desc() + parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) + opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) cmd.crawler_process = CrawlerProcess(settings) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 5f1dabd33..fb304b8c0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -2,7 +2,7 @@ Base class for Scrapy commands """ import os -from optparse import OptionGroup +import argparse from typing import Any, Dict from twisted.python import failure @@ -59,22 +59,20 @@ class ScrapyCommand: """ Populate option parse with options available for this command """ - group = OptionGroup(parser, "Global Options") - group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") - group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help=f"log level (default: {self.settings['LOG_LEVEL']})") - group.add_option("--nolog", action="store_true", - help="disable logging completely") - group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") - group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") - group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") - - parser.add_option_group(group) + group = parser.add_argument_group(title='Global Options') + group.add_argument("--logfile", metavar="FILE", + help="log file. if omitted stderr will be used") + group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None, + help=f"log level (default: {self.settings['LOG_LEVEL']})") + group.add_argument("--nolog", action="store_true", + help="disable logging completely") + group.add_argument("--profile", metavar="FILE", default=None, + help="write python cProfile stats to FILE") + group.add_argument("--pidfile", metavar="FILE", + help="write process ID to FILE") + group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE", + help="set/override setting (may be repeated)") + group.add_argument("--pdb", action="store_true", help="enable pdb on failure") def process_options(self, args, opts): try: @@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand): """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") - parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items") + parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_argument("-o", "--output", metavar="FILE", action="append", + help="append scraped items to the end of FILE (use - for stdout)") + parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", + help="dump scraped items into FILE, overwriting any existing file") + parser.add_argument("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand): opts.overwrite_output, ) self.settings.set('FEEDS', feeds, priority='cmdline') + + +class ScrapyHelpFormatter(argparse.HelpFormatter): + """ + Help Formatter for scrapy command line help messages. + """ + def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + super().__init__(prog, indent_increment=indent_increment, + max_help_position=max_help_position, width=width) + + def _join_parts(self, part_strings): + parts = self.format_part_strings(part_strings) + return super()._join_parts(parts) + + def format_part_strings(self, part_strings): + """ + Underline and title case command line help message headers. + """ + if part_strings and part_strings[0].startswith("usage: "): + part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):] + headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')] + for index in headings[::-1]: + char = '-' if "Global Options" in part_strings[index] else '=' + part_strings[index] = part_strings[index][:-2].title() + underline = ''.join(["\n", (char * len(part_strings[index])), "\n"]) + part_strings.insert(index + 1, underline) + return part_strings diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ae21d86e6..a16f4beb7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -49,10 +49,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="only list contracts, without checking them") - parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true', - help="print contract tests for all spiders") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="only list contracts, without checking them") + parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true', + help="print contract tests for all spiders") def run(self, args, opts): # load contracts diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 95f87e8c3..9b2ebb37f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -26,11 +26,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("--spider", dest="spider", help="use this spider") + parser.add_argument("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2082a4974..ed5f588e9 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -44,16 +44,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") - parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") - parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") - parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") - parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + parser.add_argument("-l", "--list", dest="list", action="store_true", + help="List available templates") + parser.add_argument("-e", "--edit", dest="edit", action="store_true", + help="Edit spider after creating it") + parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE", + help="Dump template to standard output") + parser.add_argument("-t", "--template", dest="template", default="basic", + help="Uses a custom template.") + parser.add_argument("--force", dest="force", action="store_true", + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 52118db1b..a3f6b96f4 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser): BaseRunSpiderCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") - parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") - parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") - parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") - parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") - parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") - parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") - parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") - parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") - parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + parser.add_argument("--spider", dest="spider", default=None, + help="use this spider without looking for one") + parser.add_argument("--pipelines", action="store_true", + help="process items through pipelines") + parser.add_argument("--nolinks", dest="nolinks", action="store_true", + help="don't show links to follow (extracted requests)") + parser.add_argument("--noitems", dest="noitems", action="store_true", + help="don't show scraped items") + parser.add_argument("--nocolour", dest="nocolour", action="store_true", + help="avoid using pygments to colorize the output") + parser.add_argument("-r", "--rules", dest="rules", action="store_true", + help="use CrawlSpider rules to discover the callback") + parser.add_argument("-c", "--callback", dest="callback", + help="use this callback for parsing, instead looking for a callback") + parser.add_argument("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") + parser.add_argument("--cbkwargs", dest="cbkwargs", + help="inject extra callback kwargs into the Request, it must be a valid raw json string") + parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, + help="maximum depth for parsing requests [default: %default]") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 8d49e440f..1b2e2601e 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -18,16 +18,16 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") - parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") - parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") - parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") - parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + parser.add_argument("--get", dest="get", metavar="SETTING", + help="print raw setting value") + parser.add_argument("--getbool", dest="getbool", metavar="SETTING", + help="print setting value, interpreted as a boolean") + parser.add_argument("--getint", dest="getint", metavar="SETTING", + help="print setting value, interpreted as an integer") + parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING", + help="print setting value, interpreted as a float") + parser.add_argument("--getlist", dest="getlist", metavar="SETTING", + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index de81986d8..f67a5886a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -33,12 +33,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, - help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_argument("-c", dest="code", + help="evaluate the code in the shell, print the result and exit") + parser.add_argument("--spider", dest="spider", + help="use this spider") + parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1237610cb..c6a3c273a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -16,8 +16,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + parser.add_argument("--verbose", "-v", dest="verbose", action="store_true", + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index c8f873334..b1f52abe2 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,3 +1,4 @@ +import argparse from scrapy.commands import fetch from scrapy.utils.response import open_in_browser @@ -12,7 +13,7 @@ class Command(fetch.Command): def add_options(self, parser): super().add_options(parser) - parser.remove_option("--headers") + parser.add_argument('--headers', help=argparse.SUPPRESS) def _print_response(self, response, opts): open_in_browser(response) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index ed3848d88..f21ee971d 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,9 @@ import os +import argparse from os.path import join, abspath, isfile, exists from twisted.internet import defer +from scrapy.commands import parse +from scrapy.settings import Settings from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest from scrapy.utils.python import to_unicode @@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} content = '[\n{},\n{"foo": "bar"}\n]' with open(file_path, 'r') as f: self.assertEqual(f.read(), content) + + def test_parse_add_options(self): + command = parse.Command() + command.settings = Settings() + parser = argparse.ArgumentParser( + prog='scrapy', formatter_class=argparse.HelpFormatter, + conflict_handler='resolve', prefix_chars='-' + ) + command.add_options(parser) + namespace = parser.parse_args( + ['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name] + ) + self.assertTrue(namespace.nolinks) + self.assertEqual(namespace.depth, 2) + self.assertEqual(namespace.spider, self.spider_name) + self.assertTrue(namespace.verbose) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7473b53df..7cd19b29a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,6 +1,6 @@ import inspect import json -import optparse +import argparse import os import platform import re @@ -23,7 +23,7 @@ from twisted.python.versions import Version from twisted.trial import unittest import scrapy -from scrapy.commands import ScrapyCommand +from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase): def setUp(self): self.command = ScrapyCommand() self.command.settings = Settings() - self.parser = optparse.OptionParser( - formatter=optparse.TitledHelpFormatter(), - conflict_handler='resolve', - ) + self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') self.command.add_options(self.parser) def test_settings_json_string(self): feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' - opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) + opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) self.command.process_options(args, opts) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + def test_help_formatter(self): + formatter = ScrapyHelpFormatter(prog='scrapy') + part_strings = ['usage: scrapy genspider [options] \n\n', + '\n', 'optional arguments:\n', '\n', 'Global Options:\n'] + self.assertEqual( + formatter._join_parts(part_strings), + ('Usage\n=====\n scrapy genspider [options] \n\n\n' + 'Optional Arguments\n==================\n\n' + 'Global Options\n--------------\n') + ) + class ProjectTest(unittest.TestCase): project_name = 'testproject' @@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest): self.assertNotIn('Unhandled Error', log) +class ViewCommandTest(CommandTest): + + def test_methods(self): + command = view.Command() + command.settings = Settings() + parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-', + formatter_class=ScrapyHelpFormatter, + conflict_handler='resolve') + command.add_options(parser) + self.assertEqual(command.short_desc(), + "Open URL in browser, as seen by Scrapy") + self.assertIn("URL using the Scrapy downloader and show its", + command.long_desc()) + + class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): From 5d7c0a5f861327c1a51ffcb3a39e283eb999fae7 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 10 Feb 2022 14:50:12 +0500 Subject: [PATCH 44/44] Use toscrape.com instead of example.com in test_command_check. (#5407) --- tests/test_command_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 34f5e59dd..c3d705194 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -19,11 +19,11 @@ import scrapy class CheckSpider(scrapy.Spider): name = '{self.spider_name}' - start_urls = ['http://example.com'] + start_urls = ['http://toscrape.com'] def parse(self, response, **cb_kwargs): \"\"\" - @url http://example.com + @url http://toscrape.com {contracts} \"\"\" {parse_def}