From cfed9b6659c90e0799361911b1d72ed127edf471 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 15 Jul 2015 17:27:57 +0200 Subject: [PATCH 001/211] Allow passing Python objects to middleware dict settings --- scrapy/middleware.py | 5 ++++- scrapy/utils/misc.py | 7 ++++++- tests/test_middleware.py | 13 +++++++++++++ tests/test_utils_misc/__init__.py | 4 +++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2ef5f30e2..690ec6a55 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,4 +1,5 @@ from collections import defaultdict +from inspect import isclass import logging import pprint @@ -31,7 +32,9 @@ class MiddlewareManager(object): for clspath in mwlist: try: mwcls = load_object(clspath) - if crawler and hasattr(mwcls, 'from_crawler'): + if not isclass(mwcls): + mw = mwcls + elif crawler and hasattr(mwcls, 'from_crawler'): mw = mwcls.from_crawler(crawler) elif hasattr(mwcls, 'from_settings'): mw = mwcls.from_settings(settings) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 303a413d8..75f42cc17 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -31,10 +31,15 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be a class, function, variable o instance. + If ``path`` is not a string, it will be returned. + + The object can be a class, function, variable, or instance. path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' """ + if not isinstance(path, six.string_types): + return path + try: dot = path.rindex('.') except ValueError: diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6d885330..4e3c67d20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -91,3 +91,16 @@ class MiddlewareManagerTest(unittest.TestCase): mwman = TestMiddlewareManager.from_settings(settings) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) + + def test_instances_from_settings(self): + settings = Settings() + myM3 = M3() + class InstanceTestMiddlewareManager(MiddlewareManager): + @classmethod + def _get_mwlist_from_settings(cls, settings): + return [ 'tests.test_middleware.M1', M2, myM3 ] + mwman = InstanceTestMiddlewareManager.from_settings(settings) + self.assertIsInstance(mwman.middlewares[0], M1) + self.assertIsInstance(mwman.middlewares[1], M2) + self.assertIs(mwman.middlewares[2], myM3) + diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..06af3c009 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -11,7 +11,9 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + not_a_string = int(1000) + self.assertIs(load_object(not_a_string), not_a_string) self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') From e5b8def0b86102eacdb9a49942e88047a2250527 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 1 Jun 2015 17:39:41 +0200 Subject: [PATCH 002/211] Redraft SEP-021 --- sep/sep-021.rst | 338 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 289 insertions(+), 49 deletions(-) diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 628a95dd2..ce500fc00 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -17,19 +17,31 @@ Scrapy currently supports many hooks and mechanisms for extending its functionality, but no single entry point for enabling and configuring them. Instead, the hooks are spread over: -* Spider middlewares (SPIDER_MIDDLEWARES) -* Downloader middlewares (DOWNLOADER_MIDDLEWARES) -* Downloader handlers (DOWNLOADER_HANDLERS) -* Item pipelines (ITEM_PIPELINES) -* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) -* Generic extensions (EXTENSIONS) -* CLI commands (COMMANDS_MODULE) +* Spider middlewares (``SPIDER_MIDDLEWARES``) +* Downloader middlewares (``DOWNLOADER_MIDDLEWARES``) +* Downloader handlers (``DOWNLOADER_HANDLERS``) +* Item pipelines (``ITEM_PIPELINES``) +* Feed exporters and storages (``FEED_EXPORTERS``, ``FEED_STORAGES``) +* Overrideable components (``DUPEFILTER_CLASS``, ``STATS_CLASS``, + ``SCHEDULER``, ``SPIDER_MANAGER_CLASS``, ``ITEM_PROCESSOR``, etc.) +* Generic extensions (``EXTENSIONS``) +* CLI commands (``COMMANDS_MODULE``) + +This approach has several shortfalls: + +* Enabling an extension often requires modifying many settings, often in a + coordinated way, which is complex and error prone. +* Extension developers have little control over ensuring their library + dependencies and configuration requirements are met, especially since most + extensions never 'see' a fully-configured crawler before it starts running. +* The user is burdened with supervising potential interplay of extensions, + especially non-included ones, ranging from setting name clashes to mutually + excluding dependencies/configuration requirements. + +*Add-ons* search to remedy these shortcomings by enhancing Scrapy's extension +management, making it easy-to-use and transparent for users while giving more +configuration control to developers. -One problem of this approach is that enabling an extension often requires -modifying many settings, often in a coordinated way, which is complex and error -prone. Add-ons are meant to fix this by providing a simple mechanism for -enabling extensions. Design goals and non-goals ========================== @@ -37,8 +49,8 @@ Design goals and non-goals Goals: * simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (ie. modifying + adding or removing lines in a configuration file +* backward compatibility with enabling extension the "old way" (i.e. modifying settings directly) Non-goals: @@ -46,62 +58,290 @@ Non-goals: * a way to publish, distribute or discover extensions (use pypi for that) -Managing add-ons -================ +User experience: managing add-ons +================================= -Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]`` -section. +Add-ons are enabled and configured either via Scrapy's settings, or (for add-ons +not bound to any project) in ``scrapy.cfg``. -To enable the "httpcache" addon, either shipped with Scrapy or in the Python -search path, create an entry for it in your ``scrapy.cfg``, like this:: +In the settings, add-ons can be enabled by adding either their name (for +built-in add-ons), their Python path, or their file path, to a +``INSTALLED_ADDONS`` setting. If necessary, each add-on can be configured by +providing a dictionary-valued setting with the uppercase add-on name. For +example, to enable and configure the built-in ``httpcache`` add-on and enable +(without configuring) two custom add-ons, one via Python path and one via file +path, add these entries to your settings module:: - [addons] - httpcache = + INSTALLED_ADDONS = ( + 'httpcache', + 'mymodule.filters.myfilter', + 'mymodule/filters/otherfilter.py', + ) -You may also specify the full path to an add-on (which may be either a .py file -or a folder containing __init__.py):: + HTTPCACHE = { + 'ignore_http_codes': [404, 503], + } - [addons] - mongodb_pipeline = /path/to/mongodb_pipeline.py +In ``scrapy.cfg``, add-ons are enabled and configured with one section per +add-on. The section names correspond to the entries of ``INSTALLED_ADDONS``. +The configuration from above could look like this:: + + [addon:httpcache] + ignore_http_codes = 404,503 + + [addon:mymodule.filters.myfilter] + + [addon:mymodule/filters/otherfilter.py] -Writing add-ons -=============== +Developer experience: writing add-ons +===================================== -Add-ons are Python modules that implement the following callbacks. +Add-ons are (any) Python *objects* that implement Scrapy's *add-on interface*. +The interface is enforced through ``zope.interface``. This leaves the choice of +Python object up the developer. Examples: -addon_configure ---------------- +* for a small pipeline, the add-on interface could be implemented in the same + class that also implements the ``open/close_spider`` and ``process_item`` + callbacks +* for larger add-ons, or for clearer structure, the interface could be provided + by a stand-alone module -Receives the Settings object and modifies it to enable the required components. -If it raises an exception, Scrapy will print it and exit. +The absolute minimum interface consists of just two attributes: -Examples:: +* ``NAME``: string with add-on name +* ``VERSION``: PEP-440 style version string - def addon_configure(settings): - settings.overrides['DOWNLADER_MIDDLEWARES'].update({ - 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, - }) +To be any useful, an add-on should implement at least one of the following +callback methods: + +* ``update_addons()``: adds and configures other add-ons +* ``update_settings()``: sets configuration (such as default values for this + add-on and required settings for other extensions) and enables needed + components. +* ``check_configuration()``: receives the fully-initialized ``Crawler`` + instance before it starts running, performs additional dependency and + configuration requirement checks + +Additionally, an add-on may (and should, where appropriate) provide one or more +variables that can be used for automated detection of possible dependency +clashes: + +* ``REQUIRES``: list of built-in or custom components required by this add-on, + as PEP-440 strings +* ``MODIFIES``: list of components whose functionality is affected or replaced + by this add-on (a custom HTTP cache should list ``httpcache`` here) +* ``PROVIDES``: list of components provided by this add-on (e.g. ``mongodb`` + for an extension that provides generic read/write access to a MongoDB + database, releasing other components from having to provide their own + database access methods) + +update_addons() +----------------- + +Called: +~~~~~~~ + +Shortly after initialisation of the ``Crawler`` object. + +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``addons``: the add-on manager, providing methods to add and configure add-ons + +Purpose: +~~~~~~~~ + +* Configure and enable related add-ons, useful for 'umbrella add-ons' which + chain-load other add-ons based on the configuration + +Examples: +~~~~~~~~~ :: - def addon_configure(settings): + def update_addons(config, addons): + if 'httpcache' not in addons.enabled: + addons.add('httpcache', {'expiration_secs': 60}) + +or:: + + def update_addons(config, addons): + if 'otheraddon' in addons.enabled: + addons.configs['otheraddon']['some_config_name'] = True + +update_settings() +----------------- + +Called: +~~~~~~~ + +Directly after the ``update_addons()`` callback of all add-ons has been called. + +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``settings``: the crawler's ``Settings`` instance containing all project + settings + +Purpose: +~~~~~~~~ + +* Modify ``settings`` to enable required components +* Expose some add-on specific configuration (``config``) into the global + settings namespace (``settings``) if necessary +* Raise exception if components can not be properly configured (e.g. on missing + dependencies); Scrapy will print this exception *and exit* (making users + explicitly acknowledge that the add-on does not work by forcing them to + disable it). + +Side note: +~~~~~~~~~~ + +The ``MiddlewareManager.from_settings()`` method will receive a slight +modification to allow directly placing Python objects instead of class paths +in the middleware dict settings. This way, add-ons can place already +instantiated components into the settings. This allows keeping configuration +as local to components as possible and avoids cluttering up the global +settings namespace. Furthermore, it allows reusing components (e.g. using +two instances of the same mongodb pipeline to write to different locations). + +Examples: +~~~~~~~~~ + +:: + + def update_settings(config, settings): + # Don't care where this module is located + settings.set['DOWNLADER_MIDDLEWARES']({ + __name__ + '.downloadermw.coolmw': 900, + }) + + # Instantiate components to not expose settings into + # the global namespace + from .pipelines import MySQLPipeline + mysqlpl = MySQLPipeline(password = config['password']) + settings.set['ITEM_PIPELINES']({ + mysqlpl: 200, + }) + +or:: + + def update_settings(config, settings): + # Assuming this class also has a process_item() method + settings.set['ITEM_PIPELINES']({ + self: 200, + }) + +or:: + + def update_settings(config, settings): try: import boto except ImportError: raise RuntimeError("boto library is required") +check_configuration() +--------------------- -crawler_ready -------------- +Called: +~~~~~~~ -``crawler_ready`` receives a Crawler object after it has been initialized and -is meant to be used to perform post-initialization checks like making sure the -extension and its dependencies were configured properly. If it raises an -exception, Scrapy will print and exit. +Shortly before the crawler starts crawling. -Examples:: +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``crawler``: fully-initialized ``Crawler`` object, ready to start crawling + +Purpose: +~~~~~~~~ + +* Perform post-initialization checks like making sure the extension and its + dependencies were configured properly. +* Raise exception if a critical check failed; Scrapy will print this exception + *and exit* (see ``update_settings()`` purpose for rationale on this). + +Examples: +~~~~~~~~~ + +:: + + def check_configuration(config, crawler): + if 'some.other.addon' not in crawler.addons.enabled: + raise RuntimeError("Some other add-on required to use this add-on") + + +Implementation +============== + +A new core component, the *add-on manager*, is introduced to Scrapy. It +facilitates loading add-ons, gathering and providing information on them, +calling their callbacks at appropriate times, and performing basic checks for +dependency and configuration clashes. + +Layout +------ + +A new ``AddonManager`` class is introduced, providing methods to + +* add and remove add-ons, +* search for add-ons by name +* read enabled add-ons and their configurations from the settings module and + from ``settings.py``, +* enable and disable add-ons +* check for possible dependency incompatibilites by inspecting the collected + ``REQUIRES``, ``MODIFIES`` and ``PROVIDES`` add-on variables +* call the add-on callbacks + +Integration into start-up process +--------------------------------- + +The settings used to crawl are not complete until the spider-specific settings +have been loaded in ``Crawler.__init__()``. Add-on management follows this +approach and only starts loading add-ons when the crawler is initialised. + +Instantiation and the calls ``update_addons()`` and ``update_settings()`` happen +in ``Crawler.__init__()``. The final checks (i.e. the callback to +``check_configuration()``) is coded into the ``Crawler.crawl()`` method after +creating the engine. + +Finding add-ons +--------------- + +Add-on localisation is governed by the add-on paths given in +``INSTALLED_ADDONS`` (or by the section names if using ``scrapy.cfg``). If +nothing is found at the given path, it is tried again with ``addons.`` +prepended (i.e. pointing to the project's ``addons`` folder or module), then +with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` +submodule). If the object found has an ``_addon`` attribute, that attribute +will be treated as the found add-on. This allows, for example, to change the +add-on based on the Python version. + +Updating existing extensions +---------------------------- + +An ``Addon`` class is introduced that add-on developers may or may not subclass +depending on how much of the 'default functionality' they want. Naturally, it +does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` +exposes the add-on configuration into the global settings namespace with an +appropriate name, e.g. this section from ``scrapy.cfg``:: + + [httpcache] + dir = /some/dir + +would expose ``HTTPCACHE_DIR``. + +Add-on modules will be written for all built-in extensions and placed in +``scrapy.addons``. For many default Scrapy components, it will be sufficient to +create a subclass of ``Addon`` with minor or no method modifications. The +component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). + +Later, the global settings namespace could be cleaned up in a backwards +-incompatible fashion by deprecating support for the global setting names, e.g. +``HTTPCACHE_DIR``, and instead instantiate the components with the add-on +configuration in ``update_settings()``. - def crawler_ready(crawler): - if 'some.other.addon' not in crawler.extensions.enabled: - raise RuntimeError("Some other addon is required to use this addon") From d8af395d7654ba659aa29060cf3cf0ed3dfe1174 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 19 Aug 2015 16:16:53 +0200 Subject: [PATCH 003/211] Introduce add-ons via AddonManager and Addon base class --- docs/topics/api.rst | 11 + docs/topics/settings.rst | 9 + scrapy/addons/__init__.py | 497 ++++++++++++++++++ scrapy/interfaces.py | 23 +- scrapy/settings/__init__.py | 1 + scrapy/settings/default_settings.py | 2 + scrapy/utils/conf.py | 12 +- scrapy/utils/misc.py | 42 +- scrapy/utils/project.py | 12 + tests/test_addons/__init__.py | 388 ++++++++++++++ tests/test_addons/addonmod.py | 16 + tests/test_addons/addons.py | 40 ++ tests/test_addons/cfg.cfg | 5 + tests/test_addons/project/__init__.py | 0 tests/test_addons/project/addons/__init__.py | 0 tests/test_addons/project/addons/addonmod.py | 7 + tests/test_addons/project/addons/addonmod2.py | 7 + tests/test_addons/scrapy_addons/__init__.py | 0 tests/test_addons/scrapy_addons/addonmod.py | 7 + tests/test_addons/scrapy_addons/addonmod2.py | 7 + tests/test_addons/scrapy_addons/addonmod3.py | 7 + tests/test_utils_misc/__init__.py | 26 +- tests/test_utils_misc/testmod.py | 1 + tests/test_utils_misc/testpkg/__init__.py | 1 + tests/test_utils_misc/testpkg/submod.py | 1 + tests/test_utils_project.py | 27 + 26 files changed, 1142 insertions(+), 7 deletions(-) create mode 100644 scrapy/addons/__init__.py create mode 100644 tests/test_addons/__init__.py create mode 100644 tests/test_addons/addonmod.py create mode 100644 tests/test_addons/addons.py create mode 100644 tests/test_addons/cfg.cfg create mode 100644 tests/test_addons/project/__init__.py create mode 100644 tests/test_addons/project/addons/__init__.py create mode 100644 tests/test_addons/project/addons/addonmod.py create mode 100644 tests/test_addons/project/addons/addonmod2.py create mode 100644 tests/test_addons/scrapy_addons/__init__.py create mode 100644 tests/test_addons/scrapy_addons/addonmod.py create mode 100644 tests/test_addons/scrapy_addons/addonmod2.py create mode 100644 tests/test_addons/scrapy_addons/addonmod3.py create mode 100644 tests/test_utils_misc/testmod.py create mode 100644 tests/test_utils_misc/testpkg/__init__.py create mode 100644 tests/test_utils_misc/testpkg/submod.py create mode 100644 tests/test_utils_project.py diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 42c0133c1..0c22b3ce9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -149,6 +149,17 @@ Settings API .. autoclass:: BaseSettings :members: +.. _topics-api-addonmanager: + +AddonManager API +================ + +.. module:: scrapy.addons + :synopsis: Add-on manager + +.. autoclass:: AddonManager + :members: + .. _topics-api-spiderloader: SpiderLoader API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index aa0417e1a..afcb8dd21 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -585,6 +585,15 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. +.. setting:: INSTALLED_ADDONS + +INSTALLED_ADDONS +---------------- + +Default: ``()`` + +A tuple containing paths to the add-ons enabled in your project. For more +information, see :ref:`topics-addons`. .. setting:: ITEM_PIPELINES diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py new file mode 100644 index 000000000..59e59e15f --- /dev/null +++ b/scrapy/addons/__init__.py @@ -0,0 +1,497 @@ +from collections import defaultdict, Mapping +from importlib import import_module +from inspect import isclass +import os +import six +import warnings + +from pkg_resources import WorkingSet, Distribution, Requirement +import zope.interface +from zope.interface.verify import verifyObject + +from scrapy.exceptions import NotConfigured +from scrapy.interfaces import IAddon +from scrapy.settings import BaseSettings +from scrapy.utils.conf import config_from_filepath, get_config +from scrapy.utils.misc import load_module_or_object +from scrapy.utils.project import get_project_path + + +@zope.interface.implementer(IAddon) +class Addon(object): + + basic_settings = None + """``dict`` of settings that will be exported via :meth:`export_basics`.""" + + default_config = None + """``dict`` with default configuration.""" + + config_mapping = None + """``dict`` with mappings from config names to setting names. The given + setting names will be taken as given, i.e. they will be neither prefixed + nor uppercased. + """ + + component_type = None + """Component setting into which to export via :meth:`export_component`. Can + be any of the dictionary-like component setting names (e.g. + ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in + :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``, + :meth:`export_component` will do nothing. + """ + + component_key = None + """Key to be used in the component dictionary setting when exporting via + :meth:`export_component`. This is only useful for the settings that have + no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``. + """ + + component_order = 0 + """Component order to use when not given in the add-on configuration. Has + no effect for component types that use :attr:`component_key`. + """ + + component = None + """Component to be inserted via :meth:`export_component`. This can be + anything that can be used in the dictionary-like component settings, i.e. + a class path, a class, or an instance. If ``None``, it is assumed that the + add-on itself is also provides the component interface, and ``self`` will be + used. + """ + + settings_prefix = None + """Prefix with which the add-on configuration will be exported into the + global settings namespace via :meth:`export_config`. If ``None``, + :attr:`name` will be used. If ``False``, no configuration will be exported. + """ + + def export_component(self, config, settings): + """Export the component in :attr:`component` into the dictionary-like + component setting derived from :attr:`component_type`. + + Where applicable, the order parameter of the component (i.e. the + dictionary value) will be retrieved from the ``order`` add-on + configuration value. + + :param config: Add-on configuration from which to read component order + :type config: ``dict`` + + :param settings: Settings object into which to export component + :type settings: :class:`~scrapy.settings.Settings` + """ + if self.component_type: + comp = self.component or self + if self.component_key: + # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'} + k = self.component_key + v = comp + else: + # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} + k = comp + v = config.get('order', self.component_order) + settings.set(self.component_type, {k: v}, 'addon') + + def export_basics(self, settings): + """Export the :attr:`basic_settings` attribute into the settings object. + + All settings will be exported with ``addon`` priority (see + :ref:`topics-api-settings`). + + :param settings: Settings object into which to expose the basic settings + :type settings: :class:`~scrapy.settings.Settings` + """ + for setting, value in six.iteritems(self.basic_settings or {}): + settings.set(setting, value, 'addon') + + def export_config(self, config, settings): + """Export the add-on configuration, all keys in caps and with + :attr:`settings_prefix` or :attr:`name` prepended, into the settings + object. + + For example, the add-on configuration ``{'key': 'value'}`` will export + the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings + will be exported with ``addon`` priority (see + :ref:`topics-api-settings`). + + :param config: Add-on configuration to be exposed + :type config: ``dict`` + + :param settings: Settings object into which to export the configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + if self.settings_prefix is False: + return + conf = self.default_config or {} + conf.update(config) + prefix = self.settings_prefix or self.name + # Since default exported config is case-insensitive (everything will be + # uppercased), make mapped config case-insensitive as well + conf_mapping = {k.lower(): v + for k, v in six.iteritems(self.config_mapping or {})} + for key, val in six.iteritems(conf): + if key.lower() in conf_mapping: + key = conf_mapping[key.lower()] + else: + key = (prefix + '_' + key).upper() + settings.set(key, val, 'addon') + + def update_settings(self, config, settings): + """Export both the basic settings and the add-on configuration. I.e., + call :meth:`export_basics` and :meth:`export_config`. + + For more advanced add-ons, you may want to override this callback. + + :param config: Add-on configuration + :type config: ``dict`` + + :param settings: Crawler settings object + :type settings: :class:`~scrapy.settings.Settings` + """ + self.export_component(config, settings) + self.export_basics(settings) + self.export_config(config, settings) + + +class AddonManager(Mapping): + """This class facilitates loading and storing :ref:`topics-addons`. + + You can treat it like a read-only dictionary in which keys correspond to + add-on names and values correspond to the add-on objects:: + + addons = AddonManager() + # ... load some add-ons here + print addons.enabled # prints names of all enabled add-ons + print addons['TestAddon'].version # prints version of add-on with name + # 'TestAddon' + + """ + + def __init__(self): + self._addons = {} + self.configs = {} + self._disable_on_add = [] + + def __getitem__(self, name): + return self._addons[name] + + def __delitem__(self, name): + del self._addons[name] + del self.configs[name] + + def __iter__(self): + return iter(self._addons) + + def __len__(self): + return len(self._addons) + + def add(self, addon, config=None): + """Store an add-on. + + If ``addon`` is a string, it will be treated as add-on path and passed + to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object + implementing or providing Scrapy's add-on interface. The interface + will be enforced through ``zope.interface``'s ``verifyObject()``. + + If ``addon`` is a class, it will be instantiated. You can avoid this + (for example if you have implemented the add-on callbacks as class + methods) by declaring -- via ``zope.interface`` -- that your class + directly *provides* ``scrapy.interfaces.IAddon``. + + :param addon: The add-on object (or path) to be stored + :type addon: Any Python object providing the add-on interface or ``str`` + + :param config: The add-on configuration dictionary + :type config: ``dict`` + """ + addon = self.get_addon(addon) + if isclass(addon) and not IAddon.providedBy(addon): + addon = addon() + if not IAddon.providedBy(addon): + zope.interface.alsoProvides(addon, IAddon) + # zope.interface's exceptions are already quite helpful. Still, should + # we catch them and log an error message? + verifyObject(IAddon, addon) + name = addon.name + if name in self: + raise ValueError("Addon '{}' already loaded".format(name)) + self._addons[name] = addon + self.configs[name] = config or {} + if name in self._disable_on_add: + self.configs[name]['_enabled'] = False + self._disable_on_add.remove(name) + + def remove(self, addon): + """Remove an add-on. + + If ``addon`` is the name of a stored add-on, that add-on will be + removed. Otherwise, you can use the argument in the same fashion as + in :meth:`add`. + + :param addon: The add-on name, object, or path to be removed + :type addon: Any Python object providing the add-on interface or ``str`` + """ + if addon in self: + del self[addon] + elif hasattr(addon, 'name') and addon.name in self: + del self[addon.name] + else: + try: + del self[self.get_addon(addon).name] + except NameError: + raise KeyError + + @staticmethod + def get_addon(path): + """Get an add-on object by its Python or file path. + + ``path`` is assumed to be either a Python or a file path of a Scrapy + add-on. If no object is found at ``path``, it is tried again first with + ``projectname.addons`` prepended (pointing to the current project's + ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to + Scrapy's built-in add-ons). These convenience shortcuts will only work + with Python paths, not file paths. + + If the object or module pointed to by ``path`` has an attribute named + ``_addon`` that attribute will be assumed to be the add-on. + :meth:`get_addon` will keep following ``_addon`` attributes until it + finds an object that does not have an attribute named ``_addon``. + + :param path: Python or file path to an add-on + :type path: ``str`` + """ + if isinstance(path, six.string_types): + prefixes = ['', 'scrapy.addons.'] + try: + prefixes.insert(1, get_project_path() + '.addons.') + except NotConfigured: + warnings.warn("Unable to locate project Python path") + for prefix in prefixes: + fullpath = prefix + path + try: + obj = load_module_or_object(fullpath) + except NameError: + pass + else: + break + else: + raise NameError("Could not find add-on '%s'" % path) + else: + obj = path + if hasattr(obj, '_addon'): + obj = AddonManager.get_addon(obj._addon) + return obj + + def load_dict(self, addonsdict): + """Load add-ons and configurations from given dictionary. + + Each add-on should be an entry in the dictionary, where the key + corresponds to the add-on path. The value should be a dictionary + representing the add-on configuration. + + Example add-on dictionary:: + + addonsdict = { + 'path.to.addon1': { + 'setting1': 'value', + 'setting2': 42, + }, + 'path/to/addon2.py': { + 'addon2setting': True, + }, + } + + :param addonsdict: dictionary where keys correspond to add-on paths \ + and values correspond to their configuration + :type addonsdict: ``dict`` + """ + for addonpath, addoncfg in six.iteritems(addonsdict): + self.add(addonpath, addoncfg) + + def load_settings(self, settings): + """Load add-ons and configurations from settings object. + + This will invoke :meth:`get_addon` for every add-on path in the + ``INSTALLED_ADDONS`` setting. For each of these add-ons, the + configuration will be read from the dictionary setting whose name + matches the uppercase add-on name. + + :param settings: The :class:`~scrapy.settings.Settings` object from \ + which to read the add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + paths = settings.getlist('INSTALLED_ADDONS') + addons = [self.get_addon(path) for path in paths] + configs = [settings.getdict(addon.name.upper()) for addon in addons] + for a, c in zip(addons, configs): + self.add(a, c) + + def load_cfg(self, cfg=None): + """Load add-ons and configurations from given ``ConfigParser`` object or + config file path. + + Each add-on should have its own section, where the section has a name in + the form ``addon:my_addon_path``. The add-on object is searched for via + the :meth:`get_addon` method, ``my_addon_path`` can be either a Python + or a file path. + + If ``cfg`` is ``None``, ``scrapy.cfg`` will be used. + + :param cfg: ``ConfigParser`` object or config file path from which to \ + read add-on configuration + :type cfg: ``ConfigParser`` or ``str`` + """ + if cfg is None: + cfg = get_config() + elif isinstance(cfg, six.string_types): + cfg = config_from_filepath(cfg) + for secname in cfg.sections(): + if secname.startswith("addon:"): + addonkey = secname.split("addon:", 1)[1] + addoncfg = dict(cfg.items(secname)) + self.add(addonkey, addoncfg) + + def check_dependency_clashes(self): + """Check for incompatibilities in add-on dependencies. + + Add-ons can provide information about their dependencies in their + ``provides``, ``modifies`` and ``requires`` attributes. This method will + raise an ``ImportError`` if + + * a component required by an add-on is not provided by any other add-on, + or + * a component modified by an add-on is not provided by any other add-on, + or + * the same component is provided by more than one add-on, + + and warn when a component required by an add-on is modified by any other + add-on. + """ + # Collect all active add-ons and the components they provide + ws = WorkingSet('') + def add_dist(project_name, version, **kwargs): + if project_name in ws.entry_keys.get('scrapy', []): + raise ImportError("Component {} provided by multiple add-ons" + "".format(project_name)) + else: + dist = Distribution(project_name=project_name, version=version, + **kwargs) + ws.add(dist, entry='scrapy') + for name in self: + ver = self[name].version + add_dist(name, ver) + for provides_name in getattr(self[name], 'provides', []): + add_dist(provides_name, ver) + + # Collect all required and modified components + def compile_attribute_dict(attribute_name): + attrs = defaultdict(list) + for name in self: + for entry in getattr(self[name], attribute_name, []): + attrs[entry].append(name) + return attrs + modified = compile_attribute_dict('modifies') + required = compile_attribute_dict('requires') + + req_or_mod = set(required.keys()).union(modified.keys()) + for reqstr in req_or_mod: + req = Requirement.parse(reqstr) + # May raise VersionConflict. Do we want to catch it and raise + # our own exception or is it helpful enough? + if ws.find(req) is None: + raise ImportError( + "Add-ons {} require or modify missing component {}" + "".format(required[reqstr]+modified[reqstr], reqstr)) + + mod_and_req = set(required.keys()).intersection(modified.keys()) + for conflict in mod_and_req: + warnings.warn("Component '{}', required by add-ons {}, is modified " + "by add-ons {}".format(conflict, required[conflict], + modified[conflict])) + + def disable(self, addon): + """Disable an add-on, i.e. prevent its callbacks from being called. + + If you disable an add-on before it is loaded, it will be disabled as + soon as it is added to the :class:`AddonManager`. + + :param addon: Name of the add-on to be disabled + :type addon: ``str`` + """ + if addon in self: + self.configs[addon]['_enabled'] = False + else: + self._disable_on_add.append(addon) + + def enable(self, addon): + """Re-enable a disabled add-on. + + Will raise ``ValueError`` if the add-on is neither already loaded nor + marked for being disabled on adding. + + :param addon: Name of the add-on to be enabled + :type addon: ``str`` + """ + if addon in self: + self.configs[addon]['_enabled'] = True + elif addon in self._disable_on_add: + self._disable_on_add.remove(addon) + else: + raise ValueError("Add-ons need to be added before they can be " + "enabled") + + @property + def disabled(self): + """Names of disabled add-ons""" + return ([a for a in self if not self.configs[a].get('_enabled', True)] + + self._disable_on_add) + + @property + def enabled(self): + """Names of enabled add-ons""" + return [a for a in self if self.configs[a].get('_enabled', True)] + + def _call_if_exists(self, obj, cbname, *args, **kwargs): + if obj is None: + return + try: + cb = getattr(obj, cbname) + except AttributeError: + return + else: + cb(*args, **kwargs) + + def _call_addon(self, addonname, cbname, *args, **kwargs): + if self.configs[addonname].get('_enabled', True): + self._call_if_exists(self[addonname], cbname, + self.configs[addonname], *args, **kwargs) + + def update_addons(self): + """Call ``update_addons()`` of all held add-ons. + + This will also call ``update_addons()`` of all add-ons that are added + last minute during the ``update_addons()`` routine of other add-ons. + """ + called_addons = set() + while called_addons != set(self): + for name in set(self).difference(called_addons): + called_addons.add(name) + self._call_addon(name, 'update_addons', self) + + def update_settings(self, settings): + """Call ``update_settings()`` of all held add-ons. + + :param settings: The :class:`~scrapy.settings.Settings` object to be \ + updated + :type settings: :class:`~scrapy.settings.Settings` + """ + for name in self: + self._call_addon(name, 'update_settings', settings) + + def check_configuration(self, crawler): + """Call ``check_configuration()`` of all held add-ons. + + :param crawler: the fully-initialized crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + """ + for name in self: + self._call_addon(name, 'check_configuration', crawler) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index eb93c6f7e..75b72899e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,6 +1,6 @@ -from zope.interface import Interface +import zope.interface -class ISpiderLoader(Interface): +class ISpiderLoader(zope.interface.Interface): def from_settings(settings): """Return an instance of the class for the given settings""" @@ -20,3 +20,22 @@ class ISpiderLoader(Interface): # ISpiderManager is deprecated, don't use it! # An alias is kept for backwards compatibility. ISpiderManager = ISpiderLoader + + +class IAddon(zope.interface.Interface): + """Scrapy add-on""" + + name = zope.interface.Attribute("""Add-on name""") + version = zope.interface.Attribute("""Add-on version string (PEP440)""") + + # XXX: Can methods be declared optional? I.e., can I enforce the signature + # but not the existence of a method? + + #def update_addons(config, addons): + # """Enables and configures other add-ons""" + + #def update_settings(config, settings): + # """Modifies `settings` to enable and configure required components""" + + #def check_configuration(config, crawler): + # """Performs post-initialization checks on fully configured `crawler`""" diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 342d2585e..be9f740eb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -14,6 +14,7 @@ from . import default_settings SETTINGS_PRIORITIES = { 'default': 0, 'command': 10, + 'addon': 15, 'project': 20, 'spider': 30, 'cmdline': 40, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8435b0354..a230750fb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -167,6 +167,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +INSTALLED_ADDONS = () + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e8af90f11..5a5418104 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -82,14 +82,20 @@ def init_env(project='default', set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): - """Get Scrapy config file as a SafeConfigParser""" - sources = get_sources(use_closest) +def config_from_filepath(sources): + """Create a SafeConfigParser and read in the given `sources`, which can be + either a filename or a list of filenames.""" cfg = SafeConfigParser() cfg.read(sources) return cfg +def get_config(use_closest=True): + """Get Scrapy config file as a SafeConfigParser""" + sources = get_sources(use_closest) + return config_from_filepath(sources) + + def get_sources(use_closest=True): xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \ os.path.expanduser('~/.config') diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 75f42cc17..9461d93e9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,5 +1,8 @@ """Helper functions which doesn't fit anywhere else""" +import itertools +import os.path import re +import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -56,6 +59,26 @@ def load_object(path): return obj +def load_module_or_object(path): + """Load python module or (non-module) object from given path. + + Path can be both a Python or a file path. + """ + try: + return import_module(path) + except ImportError: + pass + try: + return load_object(path) + except (ValueError, NameError, ImportError): + pass + try: + return get_module_from_filepath(path) + except ImportError: + pass + raise NameError("Could not load '%s'" % path) + + def walk_modules(path): """Loads a module and all its submodules from a the given module path and returns them. If *any* module throws an exception while importing, that @@ -78,6 +101,23 @@ def walk_modules(path): return mods +def get_module_from_filepath(path): + """Load and return a python module/package from a file path""" + path = path.rstrip("/") + if path.endswith('.py'): + path = path.rsplit('.py', 1)[0] + basefolder, modname = os.path.split(path) + # XXX: There are other ways to import modules from a full path which don't + # need to modify PYTHONPATH, see + # https://stackoverflow.com/questions/67631/ + # These methods differ between py2 and py3, and apparently the + # py3 method was deprecated in Python 3.4 + sys.path.insert(0, basefolder) + mod = import_module(modname) + sys.path.pop(0) + return mod + + def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: @@ -118,7 +158,7 @@ def md5sum(file): m.update(d) return m.hexdigest() + def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False - diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a15a0d90f..a1266c879 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -71,3 +71,15 @@ def get_project_settings(): settings.setdict(env_overrides, priority='project') return settings + +def get_project_path(): + """Return the Python path of the current project. + + This fails when the settings module does not live in the project's root. + """ + if not inside_project(): + raise NotConfigured("Not inside a project") + settings_module_path = os.environ.get(ENVVAR) + if not settings_module_path: + raise NotConfigured("Unable to locate project's python path") + return settings_module_path.rsplit('.', 1)[0] diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py new file mode 100644 index 000000000..c98f0ab65 --- /dev/null +++ b/tests/test_addons/__init__.py @@ -0,0 +1,388 @@ +import os.path +import six +from six.moves.configparser import SafeConfigParser +import sys +from tests import mock +import unittest +import warnings + +from pkg_resources import VersionConflict +import zope.interface +from zope.interface.verify import verifyObject +from zope.interface.exceptions import BrokenImplementation + +import scrapy.addons +from scrapy.addons import Addon, AddonManager +from scrapy.crawler import Crawler +from scrapy.interfaces import IAddon +from scrapy.settings import BaseSettings, Settings + +from . import addons +from . import addonmod + + +class AddonTest(unittest.TestCase): + + def setUp(self): + self.rawaddon = Addon() + class AddonWithAttributes(Addon): + name = 'Test' + version = '1.0' + self.testaddon = AddonWithAttributes() + + def test_interface(self): + # Raw Addon should fail exactly b/c name and version are not given + self.assertFalse(hasattr(self.rawaddon, 'name')) + self.assertFalse(hasattr(self.rawaddon, 'version')) + self.assertRaises(BrokenImplementation, verifyObject, IAddon, + self.rawaddon) + verifyObject(IAddon, self.testaddon) + + def test_export_component(self): + settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default') + self.testaddon.component_type = None + self.testaddon.export_component({}, settings) + self.assertEqual(len(settings['ITEM_PIPELINES']), 0) + self.testaddon.component_type = 'ITEM_PIPELINES' + self.testaddon.component = 'test.component' + self.testaddon.export_component({}, settings) + six.assertCountEqual(self, settings['ITEM_PIPELINES'], + ['test.component']) + self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.testaddon.component_order = 313 + self.testaddon.export_component({}, settings) + self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) + self.testaddon.component_type = 'DOWNLOAD_HANDLERS' + self.testaddon.component_key = 'http' + self.testaddon.export_component({}, settings) + self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], + 'test.component') + + def test_export_basics(self): + settings = BaseSettings() + self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.export_basics(settings) + self.assertEqual(settings['TESTKEY'], 313) + self.assertEqual(settings['OTHERKEY'], True) + self.assertEqual(settings.getpriority('TESTKEY'), 15) + + def test_export_config(self): + settings = BaseSettings() + self.testaddon.settings_prefix = None + self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} + self.testaddon.default_config = {'key': 55, 'defaultkey': 100} + self.testaddon.export_config({'key': 313, 'OTHERKEY': True, + 'mapped_KEY': 99}, settings) + self.assertEqual(settings['TEST_KEY'], 313) + self.assertEqual(settings['TEST_DEFAULTKEY'], 100) + self.assertEqual(settings['TEST_OTHERKEY'], True) + self.assertNotIn('MAPPED_key', settings) + self.assertNotIn('MAPPED_KEY', settings) + self.assertEqual(settings['MAPPING_WORKED'], 99) + self.assertEqual(settings.getpriority('TEST_KEY'), 15) + + self.testaddon.settings_prefix = 'PREF' + self.testaddon.export_config({'newkey': 99}, settings) + self.assertEqual(settings['PREF_NEWKEY'], 99) + + with mock.patch.object(settings, 'set') as mock_set: + self.testaddon.settings_prefix = False + self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.assertEqual(mock_set.call_count, 0) + + def test_update_settings(self): + settings = BaseSettings() + settings.set('TEST_KEY1', 'default', priority='default') + settings.set('TEST_KEY2', 'project', priority='project') + self.testaddon.settings_prefix = None + self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} + addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.update_settings(addon_config, settings) + self.assertEqual(settings['OTHERTEST_KEY'], 'addon') + self.assertEqual(settings['TEST_KEY1'], 'addon') + self.assertEqual(settings['TEST_KEY2'], 'project') + self.assertEqual(settings['TEST_KEY3'], 'addon') + + +class AddonManagerTest(unittest.TestCase): + + TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg') + ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') + + def setUp(self): + self.manager = AddonManager() + + def test_add(self): + manager = AddonManager() + manager.add(addonmod, {'key': 'val1'}) + manager.add('tests.test_addons.addons.GoodAddon') + six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertRaises(ValueError, manager.add, addonmod) + + def test_add_dont_instantiate_providing_classes(self): + class ProviderGoodAddon(addons.GoodAddon): + pass + zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + manager = AddonManager() + manager.add(ProviderGoodAddon) + self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + + def test_add_verifies(self): + brokenaddon = self.manager.get_addon( + 'tests.test_addons.addons.BrokenAddon') + self.assertRaises(zope.interface.exceptions.BrokenImplementation, + self.manager.add, + brokenaddon) + + def test_add_adds_missing_interface_declaration(self): + class GoodAddonWithoutDeclaration(object): + name = 'GoodAddonWithoutDeclaration' + version = '1.0' + self.manager.add(GoodAddonWithoutDeclaration) + + def test_remove(self): + manager = AddonManager() + def test_gets_removed(removearg): + manager.add(addonmod) + self.assertIn('AddonModule', manager) + manager.remove(removearg) + self.assertNotIn('AddonModule', manager) + test_gets_removed('AddonModule') + test_gets_removed(addonmod) + test_gets_removed('tests.test_addons.addonmod') + test_gets_removed(self.ADDONMODPATH) + self.assertRaises(KeyError, manager.remove, 'nonexistent') + self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) + + def test_get_addon(self): + goodaddon = self.manager.get_addon( + 'tests.test_addons.addons.GoodAddon') + self.assertIs(goodaddon, addons.GoodAddon) + + loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) + # XXX: The module is in fact imported twice under different names into + # sys.modules, is there a good assertion for module equality? + self.assertEqual(loaded_addonmod.name, addonmod.name) + + # Does not provide interface, but has _addon attribute pointing to + # GoodAddon instance + addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') + goodaddon = self.manager.get_addon(addonspath) + # XXX: Again, the imported class and addons.GoodAddon are different + # since they are imported twice. How to use isInstance? + self.assertEqual(goodaddon.name, addons.GoodAddon.name) + + self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + + def test_get_addon_forward(self): + class SomeCls(object): + _addon = 'tests.test_addons.addons.GoodAddon' + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) + + def test_get_addon_nested(self): + x = addons.GoodAddon('outer') + x._addon = addons.GoodAddon('middle') + x._addon._addon = addons.GoodAddon('inner') + self.assertIs(self.manager.get_addon(x), x._addon._addon) + + @mock.patch.object(scrapy.addons, 'get_project_path', + return_value='tests.test_addons.project') + def test_get_addon_prefixes(self, get_project_path_mock): + # From python path + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + # From project 'addons' folder + self.assertEqual(self.manager.get_addon('addonmod2').FROM, + 'test_addons.project.addons.addonmod2') + # Assert prefix priority '' > 'project.addons' + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + # From scrapy's 'addons' + from . import scrapy_addons + with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}): + self.assertEqual(self.manager.get_addon('addonmod3').FROM, + 'test_addons.scrapy_addons.addonmod3') + # Assert prefix priority 'project.addons' > 'scrapy.addons' + self.assertEqual(self.manager.get_addon('addonmod2').FROM, + 'test_addons.project.addons.addonmod2') + # Assert prefix priority '' > 'scrapy.addons.' + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + def test_load_dict_load_settings(self): + def _test_load_method(func, *args, **kwargs): + manager = AddonManager() + getattr(manager, func)(*args, **kwargs) + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], + ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertIn('key', manager.configs['AddonModule']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + + addonsdict = { + self.ADDONMODPATH: { + 'key': 'val1', + }, + 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, + } + _test_load_method('load_dict', addonsdict) + + settings = BaseSettings() + settings.set('INSTALLED_ADDONS', [ + self.ADDONMODPATH, + 'tests.test_addons.addons.GoodAddon', + ]) + settings.set('ADDONMODULE', {'key': 'val1'}) + settings.set('GOODADDON', {'key': 'val2'}) + _test_load_method('load_settings', settings) + + def test_load_cfg(self): + manager = AddonManager() + manager.load_cfg(self.TESTCFGPATH) + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val2') + + def test_enabled_disabled(self): + manager = AddonManager() + manager.add(addons.GoodAddon('FirstAddon')) + manager.add(addons.GoodAddon('SecondAddon')) + self.assertEqual(set(manager.enabled), + set(('FirstAddon', 'SecondAddon'))) + self.assertEqual(manager.disabled, []) + manager.disable('FirstAddon') + self.assertEqual(manager.enabled, ['SecondAddon']) + self.assertEqual(manager.disabled, ['FirstAddon']) + manager.enable('FirstAddon') + self.assertEqual(set(manager.enabled), + set(('FirstAddon', 'SecondAddon'))) + self.assertEqual(manager.disabled, []) + + def test_enable_before_add(self): + manager = AddonManager() + self.assertRaises(ValueError, manager.enable, 'FirstAddon') + manager.disable('FirstAddon') + manager.enable('FirstAddon') + manager.add(addons.GoodAddon('FirstAddon')) + self.assertIn('FirstAddon', manager.enabled) + + def test_disable_before_add(self): + manager = AddonManager() + manager.disable('FirstAddon') + manager.add(addons.GoodAddon('FirstAddon')) + self.assertEqual(manager.disabled, ['FirstAddon']) + + def test_callbacks(self): + first_addon = addons.GoodAddon('FirstAddon') + second_addon = addons.GoodAddon('SecondAddon') + + manager = AddonManager() + manager.add(first_addon, {'test': 'first'}) + manager.add(second_addon, {'test': 'second'}) + crawler = mock.create_autospec(Crawler) + settings = BaseSettings() + + with mock.patch.object(first_addon, 'update_addons') as ua_first, \ + mock.patch.object(second_addon, 'update_addons') as ua_second, \ + mock.patch.object(first_addon, 'update_settings') as us_first, \ + mock.patch.object(second_addon, 'update_settings') as us_second, \ + mock.patch.object(first_addon, 'check_configuration') as cc_first, \ + mock.patch.object(second_addon, 'check_configuration') as cc_second: + manager.update_addons() + ua_first.assert_called_once_with(manager.configs['FirstAddon'], + manager) + ua_second.assert_called_once_with(manager.configs['SecondAddon'], + manager) + manager.update_settings(settings) + us_first.assert_called_once_with(manager.configs['FirstAddon'], + settings) + us_second.assert_called_once_with(manager.configs['SecondAddon'], + settings) + manager.check_configuration(crawler) + cc_first.assert_called_once_with(manager.configs['FirstAddon'], + crawler) + cc_second.assert_called_once_with(manager.configs['SecondAddon'], + crawler) + self.assertEqual(ua_first.call_count, 1) + self.assertEqual(ua_second.call_count, 1) + self.assertEqual(us_first.call_count, 1) + self.assertEqual(us_second.call_count, 1) + + us_first.reset_mock() + us_second.reset_mock() + manager.disable('FirstAddon') + manager.update_settings(settings) + self.assertEqual(us_first.call_count, 0) + manager.enable('FirstAddon') + manager.update_settings(settings) + self.assertEqual(us_first.call_count, 1) + self.assertEqual(us_second.call_count, 2) + + def test_update_addons_last_minute_add(self): + class AddedAddon(addons.GoodAddon): + name = 'AddedAddon' + + class FirstAddon(addons.GoodAddon): + name = 'FirstAddon' + def update_addons(self, config, addons): + addons.add(AddedAddon()) + + manager = AddonManager() + first_addon = FirstAddon() + with mock.patch.object(first_addon, 'update_addons', + wraps=first_addon.update_addons) as ua_first, \ + mock.patch.object(AddedAddon, 'update_addons') as ua_added: + manager.add(first_addon, {'non-empty': 'dict'}) + manager.update_addons() + six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) + ua_first.assert_called_once_with(manager.configs['FirstAddon'], + manager) + ua_added.assert_called_once_with(manager.configs['AddedAddon'], + manager) + + def test_check_dependency_clashes_attributes(self): + provides = addons.GoodAddon("ProvidesAddon") + provides.provides = ('test', ) + provides2 = addons.GoodAddon("ProvidesAddon2") + provides2.provides = ('test', ) + requires = addons.GoodAddon("RequiresAddon") + requires.requires = ('test', ) + requires_name = addons.GoodAddon("RequiresNameAddon") + requires_name.requires = ('ProvidesAddon', ) + requires_newer = addons.GoodAddon("RequiresNewerAddon") + requires_newer.requires = ('test>=2.0', ) + modifies = addons.GoodAddon("ModifiesAddon") + modifies.modifies = ('test', ) + + def check_with(*addons): + manager = AddonManager() + for a in addons: + manager.add(a) + return manager.check_dependency_clashes() + + self.assertRaises(ImportError, check_with, requires) + self.assertRaises(ImportError, check_with, modifies) + self.assertRaises(ImportError, check_with, provides, provides2) + self.assertRaises(VersionConflict, check_with, provides, requires_newer) + with warnings.catch_warnings(record=True) as w: + check_with(provides, modifies) + check_with(provides) + check_with(provides, requires) + check_with(provides, requires_name) + self.assertEqual(len(w), 0) + check_with(requires, provides, modifies) + self.assertEqual(len(w), 1) diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py new file mode 100644 index 000000000..8ecf4b81d --- /dev/null +++ b/tests/test_addons/addonmod.py @@ -0,0 +1,16 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = "test_addons.addonmod" + +name = "AddonModule" +version = "1.0" + +def update_settings(config, settings): + pass + +def check_configuration(config, crawler): + pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py new file mode 100644 index 000000000..f3442b192 --- /dev/null +++ b/tests/test_addons/addons.py @@ -0,0 +1,40 @@ +import zope.interface + +from scrapy.addons import Addon +from scrapy.interfaces import IAddon + + +class Addon(object): + FROM = 'test_addons.addons' + + +@zope.interface.declarations.implementer(IAddon) +class GoodAddon(object): + + name = 'GoodAddon' + version = '1.0' + + def __init__(self, name=None, version=None): + if name is not None: + self.name = name + if version is not None: + self.version = version + + def update_addons(self, config, addons): + pass + + def update_settings(self, config, settings): + pass + + def check_configuration(self, config, crawler): + pass + + +@zope.interface.declarations.implementer(IAddon) +class BrokenAddon(object): + + name = 'BrokenAddon' + # No version + + +_addon = GoodAddon() diff --git a/tests/test_addons/cfg.cfg b/tests/test_addons/cfg.cfg new file mode 100644 index 000000000..98c4f0f25 --- /dev/null +++ b/tests/test_addons/cfg.cfg @@ -0,0 +1,5 @@ +[addon:tests.test_addons.addons.GoodAddon] +key = val1 + +[addon:tests/test_addons/addonmod.py] +key = val2 diff --git a/tests/test_addons/project/__init__.py b/tests/test_addons/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/project/addons/__init__.py b/tests/test_addons/project/addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/project/addons/addonmod.py b/tests/test_addons/project/addons/addonmod.py new file mode 100644 index 000000000..66ca644f8 --- /dev/null +++ b/tests/test_addons/project/addons/addonmod.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.project.addons.addonmod' diff --git a/tests/test_addons/project/addons/addonmod2.py b/tests/test_addons/project/addons/addonmod2.py new file mode 100644 index 000000000..0dbdd70ff --- /dev/null +++ b/tests/test_addons/project/addons/addonmod2.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.project.addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/__init__.py b/tests/test_addons/scrapy_addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/scrapy_addons/addonmod.py b/tests/test_addons/scrapy_addons/addonmod.py new file mode 100644 index 000000000..fa479aa68 --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod' diff --git a/tests/test_addons/scrapy_addons/addonmod2.py b/tests/test_addons/scrapy_addons/addonmod2.py new file mode 100644 index 000000000..da053af4a --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod2.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/addonmod3.py b/tests/test_addons/scrapy_addons/addonmod3.py new file mode 100644 index 000000000..c64521478 --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod3.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod3' diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 06af3c009..f33562b7d 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,8 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import load_object, arg_to_iter, walk_modules +from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, + walk_modules, get_module_from_filepath) __doctests__ = ['scrapy.utils.misc'] @@ -17,6 +18,15 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + def test_load_module_or_object(self): + testmod = load_module_or_object(__name__ + '.testmod') + self.assertTrue(hasattr(testmod, 'TESTVAR')) + testmod = load_module_or_object( + os.path.join(os.path.dirname(__file__), 'testmod.py')) + self.assertTrue(hasattr(testmod, 'TESTVAR')) + obj = load_object('scrapy.utils.misc.load_object') + self.assertIs(obj, load_object) + def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') expected = [ @@ -57,6 +67,20 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) + def test_get_module_from_filepath(self): + testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') + testmod = get_module_from_filepath(testmodpath) + self.assertTrue(hasattr(testmod, 'TESTVAR')) + + testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') + testpkg = get_module_from_filepath(testpkgpath) + self.assertTrue(hasattr(testpkg, 'TESTVAR2')) + # Check submodule access + import testpkg.submod + self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) + self.assertIs(testpkg.submod.TESTVAR3, + load_object(testpkg.__name__ + ".submod.TESTVAR3")) + def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testmod.py b/tests/test_utils_misc/testmod.py new file mode 100644 index 000000000..eb540335f --- /dev/null +++ b/tests/test_utils_misc/testmod.py @@ -0,0 +1 @@ +TESTVAR = True diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py new file mode 100644 index 000000000..12cc2f6d9 --- /dev/null +++ b/tests/test_utils_misc/testpkg/__init__.py @@ -0,0 +1 @@ +TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py new file mode 100644 index 000000000..8a07e3592 --- /dev/null +++ b/tests/test_utils_misc/testpkg/submod.py @@ -0,0 +1 @@ +TESTVAR3 = True diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py new file mode 100644 index 000000000..cea4d9950 --- /dev/null +++ b/tests/test_utils_project.py @@ -0,0 +1,27 @@ +import os +from tests import mock +import unittest + +from scrapy.exceptions import NotConfigured +from scrapy.utils.project import get_project_path, inside_project + + +class UtilsProjectTestCase(unittest.TestCase): + + @mock.patch('scrapy.utils.project.inside_project', return_value=True) + def test_get_project_path(self, mock_ip): + def _test(settingsmod, expected): + with mock.patch.dict('os.environ', + {'SCRAPY_SETTINGS_MODULE': settingsmod}): + self.assertEqual(get_project_path(), expected) + _test('project.settings', 'project') + _test('project.othername', 'project') + _test('nested.project.settings', 'nested.project') + + with mock.patch.dict('os.environ', {}, clear=True): + self.assertRaises(NotConfigured, get_project_path) + + mock_ip.return_value = False + with mock.patch.dict('os.environ', + {'SCRAPY_SETTINGS_MODULE': 'some.settings'}): + self.assertRaises(NotConfigured, get_project_path) From 07455b1883cacab141572df4310315bb53e65ec9 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 19 Aug 2015 16:17:09 +0200 Subject: [PATCH 004/211] Integrate add-ons into start-up process --- scrapy/cmdline.py | 6 +++++- scrapy/crawler.py | 19 ++++++++++++++----- scrapy/utils/test.py | 4 ++-- tests/test_crawl.py | 17 +++++++++++++++++ tests/test_crawler.py | 24 ++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 35050c13d..b403df570 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,6 +6,7 @@ import inspect import pkg_resources import scrapy +from scrapy.addons import AddonManager from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree from scrapy.commands import ScrapyCommand @@ -118,6 +119,9 @@ def execute(argv=None, settings=None): conf.settings = settings # ------------------------------------------------------------------ + addons = AddonManager() + addons.load_cfg() + inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) @@ -139,7 +143,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + cmd.crawler_process = CrawlerProcess(settings, addons) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index bdcfa9d0c..8107a50aa 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -6,6 +6,7 @@ import warnings from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement +from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) class Crawler(object): - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, addons=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -31,6 +32,12 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) + self.addons = addons if addons is not None else AddonManager() + self.addons.load_settings(self.settings) + self.addons.update_addons() + self.addons.check_dependency_clashes() + self.addons.update_settings(self.settings) + self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) @@ -69,6 +76,7 @@ class Crawler(object): try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() + self.addons.check_configuration(self) start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) @@ -111,10 +119,11 @@ class CrawlerRunner(object): ":meth:`crawl` and managed by this class." ) - def __init__(self, settings=None): + def __init__(self, settings=None, addons=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings + self.addons = addons self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() @@ -181,7 +190,7 @@ class CrawlerRunner(object): def _create_crawler(self, spidercls): if isinstance(spidercls, six.string_types): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + return Crawler(spidercls, self.settings, self.addons) def stop(self): """ @@ -223,8 +232,8 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None): - super(CrawlerProcess, self).__init__(settings) + def __init__(self, settings=None, addons=None): + super(CrawlerProcess, self).__init__(settings, addons) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings) log_scrapy_info(self.settings) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 51edfd353..0e11ec7d1 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,7 +20,7 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") -def get_crawler(spidercls=None, settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None, addons=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -28,7 +28,7 @@ def get_crawler(spidercls=None, settings_dict=None): from scrapy.crawler import CrawlerRunner from scrapy.spiders import Spider - runner = CrawlerRunner(settings_dict) + runner = CrawlerRunner(settings_dict, addons) return runner.create_crawler(spidercls or Spider) def get_pythonpath(): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..7358009e9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,6 +6,7 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy.addons import Addon, AddonManager from scrapy.http import Request from scrapy.crawler import CrawlerRunner from tests import mock @@ -266,3 +267,19 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + + @defer.inlineCallbacks + def test_abort_on_addon_failed_check(self): + class FailedCheckAddon(Addon): + name = 'FailedCheckAddon' + version = '1.0' + def check_configuration(self, config, crawler): + raise ValueError + addonmgr = AddonManager() + addonmgr.add(FailedCheckAddon()) + crawler = self.runner.create_crawler(SimpleSpider) + crawler.addons = addonmgr + # Doesn't work in 'precise' test environment: + #with self.assertRaises(ValueError): + # yield crawler.crawl() + yield self.assertFailure(crawler.crawl(), ValueError) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..dfad11405 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -2,6 +2,7 @@ import warnings import unittest import scrapy +from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader @@ -51,6 +52,29 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) + def test_populate_addons_settings(self): + class TestAddon(Addon): + name = 'TestAddon' + version = '1.0' + addonconfig = {'TEST1': 'addon', 'TEST2': 'addon', 'TEST3': 'addon'} + class TestAddon2(Addon): + name = 'testAddon2' + version = '1.0' + addonconfig2 = {'TEST': 'addon2'} + + settings = Settings() + settings.set('TESTADDON_TEST1', 'project', priority='project') + settings.set('TESTADDON_TEST2', 'default', priority='default') + addonmgr = AddonManager() + addonmgr.add(TestAddon(), addonconfig) + addonmgr.add(TestAddon2(), addonconfig2) + crawler = Crawler(DefaultSpider, settings, addonmgr) + + self.assertEqual(crawler.settings['TESTADDON_TEST1'], 'project') + self.assertEqual(crawler.settings['TESTADDON_TEST2'], 'addon') + self.assertEqual(crawler.settings['TESTADDON_TEST3'], 'addon') + self.assertEqual(crawler.settings['TESTADDON2_TEST'], 'addon2') + def test_crawler_accepts_dict(self): crawler = Crawler(DefaultSpider, {'foo': 'bar'}) self.assertEqual(crawler.settings['foo'], 'bar') From d91647c38b9504ca514a7cfc4d6c4fdb54c3d853 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 17 Aug 2015 02:48:12 +0200 Subject: [PATCH 005/211] Add built-in add-ons --- scrapy/addons/__init__.py | 3 + scrapy/addons/builtins.py | 293 +++++++++++++++++++++++++++++ tests/test_addons/test_builtins.py | 42 +++++ 3 files changed, 338 insertions(+) create mode 100644 scrapy/addons/builtins.py create mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 59e59e15f..420d46b68 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -495,3 +495,6 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, 'check_configuration', crawler) + + +from scrapy.addons.builtins import * diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py new file mode 100644 index 000000000..ff7902afb --- /dev/null +++ b/scrapy/addons/builtins.py @@ -0,0 +1,293 @@ +import scrapy +from scrapy.addons import Addon + +__all__ = ['make_builtin_addon', + + 'depth', 'httperror', 'offsite', 'referer', 'urllength', + + 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', + 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', + 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', + 'stats', 'useragent', + + 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', + 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', + 'statsmailer', 'telnetconsole', + ] + + +def make_builtin_addon(addon_name, comp_type, comp, order=0, + addon_default_config=None, addon_version=None): + class ThisAddon(Addon): + name = addon_name + version = addon_version or scrapy.__version__ + component_type = comp_type + component = comp + component_order = order + default_config = addon_default_config or {} + + return ThisAddon + + +# XXX: Below are CLASSES that have lowercase names. This is in line with the +# original SEP-021 but violates PEP8. +# We might consider prepending all built-in addon names with scrapy_ or similar +# to reduce the chance of name clashes. + +# SPIDER MIDDLEWARES + +depth = make_builtin_addon( + 'depth', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.depth.DepthMiddleware', + 900, +) + +httperror = make_builtin_addon( + 'httperror', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', + 50, +) + +offsite = make_builtin_addon( + 'offsite', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', + 500, +) + +referer = make_builtin_addon( + 'referer', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.referer.RefererMiddleware', + 700, + {'enabled': True}, +) + +urllength = make_builtin_addon( + 'urllength', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', + 800, +) + + +# DOWNLOADER MIDDLEWARES + +ajaxcrawl = make_builtin_addon( + 'ajaxcrawl', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware', + 560, +) + +chunked = make_builtin_addon( + 'chunked', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware', + 830, +) + +cookies = make_builtin_addon( + 'cookies', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', + 700, + {'enabled': True}, +) + +defaultheaders = make_builtin_addon( + 'defaultheaders', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', + 550, +) +# Assume every config entry is a header +def defaultheaders_export_config(self, config, settings): + conf = self.default_config or {} + conf.update(config) + settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') +defaultheaders.export_config = defaultheaders_export_config + +downloadtimeout = make_builtin_addon( + 'downloadtimeout', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', + 350, +) +downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', + 'download_timeout': 'DOWNLOAD_TIMEOUT'} + +httpauth = make_builtin_addon( + 'httpauth', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', + 300, +) + +httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, +) + +httpcompression = make_builtin_addon( + 'httpcompression', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', + 590, + {'enabled': True}, +) +httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} + +httpproxy = make_builtin_addon( + 'httpproxy', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', + 750, +) + +metarefresh = make_builtin_addon( + 'metarefresh', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', + 580, + {'enabled': True}, +) +metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} + +redirect = make_builtin_addon( + 'redirect', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', + 600, + {'enabled': True}, +) + +retry = make_builtin_addon( + 'retry', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.retry.RetryMiddleware', + 500, + {'enabled': True}, +) + +robotstxt = make_builtin_addon( + 'robotstxt', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + 100, + {'obey': True}, +) + +stats = make_builtin_addon( + 'stats', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.stats.DownloaderStats', + 850, +) + +useragent = make_builtin_addon( + 'useragent', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', + 400, +) +useragent.config_mapping = {'user_agent': 'USER_AGENT'} + + +# ITEM PIPELINES + + +# EXTENSIONS + +autothrottle = make_builtin_addon( + 'throttle', + 'EXTENSIONS', + 'scrapy.extensions.throttle.AutoThrottle', + 0, + {'enabled': True}, +) + +corestats = make_builtin_addon( + 'corestats', + 'EXTENSIONS' + 'scrapy.extensions.corestats.CoreStats', + 0, +) + +closespider = make_builtin_addon( + 'closespider', + 'EXTENSIONS' + 'scrapy.extensions.closespider.CloseSpider', + 0, +) + +debugger = make_builtin_addon( + 'debugger', + 'EXTENSIONS' + 'scrapy.extensions.debug.Debugger', + 0, +) + +feedexport = make_builtin_addon( + 'feedexport', + 'EXTENSIONS' + 'scrapy.extensions.feedexport.FeedExporter', + 0, +) +feedexport.settings_prefix = 'FEED' + +logstats = make_builtin_addon( + 'logstats', + 'EXTENSIONS' + 'scrapy.extensions.logstats.LogStats', + 0, +) + +memdebug = make_builtin_addon( + 'memdebug', + 'EXTENSIONS' + 'scrapy.extensions.memdebug.MemoryDebugger', + 0, + {'enabled': True}, +) + +memusage = make_builtin_addon( + 'memusage', + 'EXTENSIONS' + 'scrapy.extensions.memusage.MemoryUsage', + 0, + {'enabled': True}, +) + +spiderstate = make_builtin_addon( + 'spiderstate', + 'EXTENSIONS' + 'scrapy.extensions.spiderstate.SpiderState', + 0, +) + +stacktracedump = make_builtin_addon( + 'stacktracedump', + 'EXTENSIONS' + 'scrapy.extensions.debug.StackTraceDump', + 0, +) + +statsmailer = make_builtin_addon( + 'statsmailer', + 'EXTENSIONS' + 'scrapy.extensions.statsmailer.StatsMailer', + 0, +) + +telnetconsole = make_builtin_addon( + 'telnetconsole', + 'EXTENSIONS' + 'scrapy.telnet.TelnetConsole', + 0, +) diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py new file mode 100644 index 000000000..607c911fb --- /dev/null +++ b/tests/test_addons/test_builtins.py @@ -0,0 +1,42 @@ +import unittest + +import scrapy +import scrapy.addons +from scrapy.addons.builtins import make_builtin_addon +from scrapy.settings import Settings + + +class BuiltinAddonsTest(unittest.TestCase): + + def test_make_builtin_addon(self): + httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, + ) + self.assertEqual(httpcache.name, 'httpcache') + self.assertEqual(httpcache.component_type, 'DOWNLOADER_MIDDLEWARES') + self.assertEqual(httpcache.component, 'scrapy.downloadermiddlewares.' + 'httpcache.HttpCacheMiddleware') + self.assertEqual(httpcache.component_order, 900) + self.assertEqual(httpcache.default_config, {'enabled': True}) + self.assertEqual(httpcache.version, scrapy.__version__) + httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, + '99.9', + ) + self.assertEqual(httpcache.version, '99.9') + + def test_defaultheaders_export_config(self): + settings = Settings() + dh = scrapy.addons.defaultheaders() + dh.export_config({'X-Test-Header': 'val'}, settings) + self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) + self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], + 'val') From 2946b674144833254f3ec6dd6aece0156f71a21b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 28 Jul 2015 12:58:05 +0200 Subject: [PATCH 006/211] Document add-ons --- docs/index.rst | 4 + docs/topics/addons.rst | 387 ++++++++++++++++++++++++++++++++++++++ scrapy/addons/__init__.py | 4 +- 3 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 docs/topics/addons.rst diff --git a/docs/index.rst b/docs/index.rst index 0d21f5d40..3e8a220e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -206,6 +206,7 @@ Extending Scrapy :hidden: topics/architecture + topics/addons topics/downloader-middleware topics/spider-middleware topics/extensions @@ -217,6 +218,9 @@ Extending Scrapy :doc:`topics/architecture` Understand the Scrapy architecture. +:doc:`topics/addons` + Enable and configure built-in and third-party extensions. + :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst new file mode 100644 index 000000000..39ef286eb --- /dev/null +++ b/docs/topics/addons.rst @@ -0,0 +1,387 @@ +.. _topics-addons: + +======= +Add-ons +======= + +Scrapy's add-on system is a framework which unifies managing and configuring +components that extend Scrapy's core functionality, such as middlewares, +extensions, or pipelines. It provides users with a plug-and-play experience in +Scrapy extension management, and grants extensive configuration control to +developers. + + +Activating and configuring add-ons +================================== + +Add-ons and their configuration live in Scrapy's +:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and +only then, the add-on manager will read a list of enabled add-ons and their +configurations from your settings. There are two places where you can provide +the paths to add-ons you want to enable: + +* the ``INSTALLED_ADDONS`` setting, and +* the ``scrapy.cfg`` file. + +As Scrapy settings can be modified from many places, e.g. in a project's +``settings.py``, in a Spider's ``custom_settings`` attribute, or from the +command line, using the ``INSTALLED_ADDONS`` setting is the preferred way to +manage add-ons. + +The ``INSTALLED_ADDONS`` setting a tuple in which every item is a path to an +add-on. The path can be both a Python or a file path. While more precise, it is +not necessary to specify the full add-on Python path if it is either built into +Scrapy or lives in your project's ``addons`` submodule. + +The configuration of an add-on, if necessary at all, is stored as a dictionary +setting whose name is the uppercase add-on name. + +This is an example where an internal add-on and two third-party add-ons (in this +case with one requiring no configuration) are enabled/configured in a project's +``settings.py``:: + + INSTALLED_ADDONS = ( + 'httpcache', + 'path.to.some.addon', + 'path/to/other/addon.py', + ) + + HTTPCACHE = { + 'expiration_secs': 60, + 'ignore_http_codes': [404, 405], + } + + SOMEADDON = { + 'some_config': True, + } + +It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is +a little friendlier, be aware that this file, and therefore the configuration in +it, is not bound to a particular Scrapy project. While this should not pose a +problem when you use the project on your development machine only, a common +stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``. + +In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the +dictionary keys. I.e., the configuration from above would look like this: + +.. code-block:: cfg + + [addon:httpcache] + expiration_secs = 60 + ignore_http_codes = 404,405 + + [addon:path.to.some.addon] + some_config = true + + [addon:path/to/other/addon.py] + + +Enabling and configuring add-ons within Python code +--------------------------------------------------- + +The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings +and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. +Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is +populated, changing the ``INSTALLED_ADDONS`` setting or any of the add-on +configuration dictionary settings will have no effect. + +If you want to enable, disable, or configure add-ons in Python code, for example +when writing your own add-on, you will have to use the +:class:`~scrapy.addons.AddonManager`. You can access the add-on manager through +either ``crawler.addons`` or, if you are writing an add-on, through the +``addons`` argument of the :meth:`update_addons` callback. The add-on manager +provides many useful methods and attributes to facilitate interacting with the +add-ons framework, e.g.: + +* an :meth:`~scrapy.addons.AddonManager.add` method to load add-ons, +* the :attr:`~scrapy.addons.AddonManager.enabled` list of enabled add-ons, +* :meth:`~scrapy.addons.AddonManager.enable` and + :meth:`~scrapy.addons.AddonManager.disable` methods, +* the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the + configuration of all add-ons + +In this example, we ensure that the ``httpcache`` add-on is loaded, and that +its ``expiration_secs`` configuration is set to ``60``:: + + # addons is an instance of AddonManager + if 'httpcache' not in addons: + addons.add('httpcache', {'expiration_secs': 60}) + else: + addons.configs['httpcache']['expiration_secs'] = 60 + + +Writing your own add-ons +======================== + +Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*. +The interface is enforced through ``zope.interface``. This leaves the choice of +Python object up the developer. Examples: + +* for a small pipeline, the add-on interface could be implemented in the same + class that also implements the ``open/close_spider`` and ``process_item`` + callbacks +* for larger add-ons, or for clearer structure, the interface could be provided + by a stand-alone module + +The absolute minimum interface consists of two attributes: + +.. attribute:: name + + string with add-on name + +.. attribute:: version + + version string (PEP-404, e.g. ``'1.0.1'``) + +Of course, stating just these two attributes will not get you very far. Add-ons +can provide three callback methods that are called at various stages before the +crawling process: + +.. method:: update_settings(config, settings) + + This method is called during the initialization of the + :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks + (e.g. for external Python libraries) and update the + :class:`~scrapy.settings.Settings` object as wished, e.g. enable components + for this add-on or set required configuration of other extensions. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.Settings` + +.. method:: check_configuration(config, crawler) + + This method is called when the :class:`~scrapy.crawler.Crawler` has been + fully initialized, immediately before it starts crawling. You can perform + additional dependency and configuration checks here. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param crawler: Fully initialized Scrapy crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + +.. method:: update_addons(config, addons) + + This method is called immediately before :meth:`update_settings`, and should + be used to enable and configure other *add-ons* only. + + When using this callback, be aware that there is no guarantee in which order + the :meth:`update_addons` callbacks of enabled add-ons will be called. + Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during + this callback will also have their :meth:`update_addons` method called. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param addons: Add-on manager holding all loaded add-ons + :type addons: :class:`~scrapy.addons.AddonManager` + +Additionally, add-ons may (and should, where appropriate) provide one or more +attributes that can be used for limited automated detection of possible +dependency clashes: + +.. attribute:: requires + + list of built-in or custom components needed by this add-on, as strings. + +.. attribute:: modifies + + list of built-in or custom components whose functionality is affected or + replaced by this add-on (a custom HTTP cache should list ``httpcache`` here) + +.. attribute:: provides + + list of components provided by this add-on (e.g. ``mongodb`` for an + extension that provides generic read/write access to a MongoDB database) + +The entries in the :attr:`requires` and :attr:`modifies` attributes can be add-on +names or components from other add-ons' :attr:`provides` attribute. You can +specify :pep:`440`-style information about required versions. Examples:: + + requires = ['httpcache'] + requires = ['otheraddon >= 2.0', 'yetanotheraddon'] + +The Python object or module that is pointed to by an add-on path (e.g. given in +the ``INSTALLED_ADDONS`` setting, or given to +:meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an +add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be +either an add-on or another add-on path. + + +Add-on base class +================= + +Scrapy comes with a built-in base class for add-ons which provides some +convenience functionality: + +* basic settings can be exported via :meth:`~scrapy.addons.Addon.export_basics`, + configurable via :attr:`~scrapy.addons.Addon.basic_settings`. +* a single component (e.g. an item pipeline or a downloader middleware) can be + inserted into Scrapy's settings via + :meth:`~scrapy.addons.Addon.export_component`, configurable via + :attr:`~scrapy.addons.Addon.component_type`, + :attr:`~scrapy.addons.Addon.component_key`, + :attr:`~scrapy.addons.Addon.component`, and the ``order`` key in + :attr:`~scrapy.addons.Addon.default_config`. +* the add-on configuration can be exposed into Scrapy's settings via + :meth:`~scrapy.addons.Addon.export_config`, configurable via + :attr:`~scrapy.addons.Addon.default_config`, + :attr:`~scrapy.addons.Addon.config_mapping`, and + :attr:`~scrapy.addons.Addon.settings_prefix`. + +By default, the base add-on class will expose the add-on configuration into +Scrapy's settings namespace, in caps and with the add-on name prepended. It is +easy to write your own functionality while still being able to use the +convenience functions by overwriting +:meth:`~scrapy.addons.Addon.update_settings`. + +.. module:: scrapy.addons + +.. autoclass:: Addon + :members: + + +Add-on examples +=============== + +Set some basic configuration using the :class:`Addon` base class:: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + component = 'path.to.mypipeline' + component_type = 'ITEM_PIPELINES' + component_order = 200 + basic_settings = { + 'DNSCACHE_ENABLED': False, + } + +Check dependencies:: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + try: + import boto + except ImportError: + raise RuntimeError("myaddon requires the boto library") + else: + self.export_config(config, settings) + +Enable a component that lives relative to the add-on (see +:ref:`topics-api-settings`):: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + component = __name__ + '.downloadermw.coolmw' + component_type = 'DOWNLOADER_MIDDLEWARES' + component_order = 900 + +Instantiate components ad hoc:: + + from path.to.my.pipelines import MySQLPipeline + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + mysqlpl = MySQLPipeline(password=config['password']) + settings.set( + 'ITEM_PIPELINES', + {mysqlpl: 200}, + priority='addon', + ) + +Provide add-on interface along component interface:: + + class MyPipeline(object): + name = 'mypipeline' + version = '1.0' + + def process_item(self, item, spider): + # Do some processing here + return item + + def update_settings(self, config, settings): + settings.set( + 'ITEM_PIPELINES', + {self: 200}, + priority='addon', + ) + +Enable another addon (see :ref:`topics-api-addonmanager`):: + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_addons(self, config, addons): + if 'httpcache' not in addons.enabled: + addons.add('httpcache', {'expiration_secs': 60}) + +Check configuration of fully initialized crawler (see +:ref:`topics-api-crawler`):: + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + settings.set('DNSCACHE_ENABLED', False, priority='addon') + + def check_configuration(self, config, crawler): + if crawler.settings.getbool('DNSCACHE_ENABLED'): + # The spider, some other add-on, or the user messed with the + # DNS cache setting + raise ValueError("myaddon is incompatible with DNS cache") + +Provide add-on interface through a module: + +.. No idea why just using '::' doesn't work for this one +.. code-block:: python + + name = 'AddonModule' + version = '1.0' + + class MyPipeline(object): + # ... + + class MyDownloaderMiddleware(object): + # ... + + def update_settings(config, settings): + settings.set( + 'ITEM_PIPELINES', + {MyPipeline(): 200}, + priority='addon', + } + settings.set( + 'DOWNLOADER_MIDDLEWARES', + {MyDownloaderMiddleware(): 800}, + priority='addon', + } + +Forward to other add-ons depending on Python version:: + + # This could be a Python module, say project/pipelines/mypipeline.py, but + # could also be done inside a class, etc. + import six + + if six.PY3: + # We're running Python 3 + _addon = 'path.to.addon' + else: + _addon = 'path.to.other.addon' diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 420d46b68..15460143c 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -156,13 +156,15 @@ class AddonManager(Mapping): """This class facilitates loading and storing :ref:`topics-addons`. You can treat it like a read-only dictionary in which keys correspond to - add-on names and values correspond to the add-on objects:: + add-on names and values correspond to the add-on objects. Add-on + configurations are saved in the :attr:`config` dictionary attribute:: addons = AddonManager() # ... load some add-ons here print addons.enabled # prints names of all enabled add-ons print addons['TestAddon'].version # prints version of add-on with name # 'TestAddon' + print addons.configs['TestAddon'] # prints configuration of 'TestAddon' """ From 25498c3c210ea5aa1a45aed02618033d832cfc1c Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 24 Aug 2015 18:04:58 +0200 Subject: [PATCH 007/211] Remove unused imports in add-ons --- scrapy/addons/__init__.py | 3 --- tests/test_addons/__init__.py | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 15460143c..b1d6e14cb 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,7 +1,5 @@ from collections import defaultdict, Mapping -from importlib import import_module from inspect import isclass -import os import six import warnings @@ -11,7 +9,6 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings from scrapy.utils.conf import config_from_filepath, get_config from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index c98f0ab65..84870ec52 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,7 +1,5 @@ import os.path import six -from six.moves.configparser import SafeConfigParser -import sys from tests import mock import unittest import warnings @@ -15,7 +13,7 @@ import scrapy.addons from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import BaseSettings from . import addons from . import addonmod From 4ac6a83072f6e507f7f952b9c466419bec43364c Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 26 Aug 2015 01:43:59 +0200 Subject: [PATCH 008/211] Fix class signatures in Extensions docs --- docs/topics/extensions.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 847353868..b29e1802f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -183,7 +183,7 @@ Telnet console extension .. module:: scrapy.extensions.telnet :synopsis: Telnet console -.. class:: scrapy.extensions.telnet.TelnetConsole +.. class:: TelnetConsole Provides a telnet console for getting into a Python interpreter inside the currently running Scrapy process, which can be very useful for debugging. @@ -200,7 +200,7 @@ Memory usage extension .. module:: scrapy.extensions.memusage :synopsis: Memory usage extension -.. class:: scrapy.extensions.memusage.MemoryUsage +.. class:: MemoryUsage .. note:: This extension does not work in Windows. @@ -229,7 +229,7 @@ Memory debugger extension .. module:: scrapy.extensions.memdebug :synopsis: Memory debugger extension -.. class:: scrapy.extensions.memdebug.MemoryDebugger +.. class:: MemoryDebugger An extension for debugging memory usage. It collects information about: @@ -245,7 +245,7 @@ Close spider extension .. module:: scrapy.extensions.closespider :synopsis: Close spider extension -.. class:: scrapy.extensions.closespider.CloseSpider +.. class:: CloseSpider Closes a spider automatically when some conditions are met, using a specific closing reason for each condition. @@ -316,7 +316,7 @@ StatsMailer extension .. module:: scrapy.extensions.statsmailer :synopsis: StatsMailer extension -.. class:: scrapy.extensions.statsmailer.StatsMailer +.. class:: StatsMailer This simple extension can be used to send a notification e-mail every time a domain has finished scraping, including the Scrapy stats collected. The email @@ -332,7 +332,7 @@ Debugging extensions Stack trace dump extension ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.extensions.debug.StackTraceDump +.. class:: StackTraceDump Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_ signal is received. The information dumped is the following: @@ -361,7 +361,7 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal: Debugger extension ~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.extensions.debug.Debugger +.. class:: Debugger Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues From 18c7f3dbe2d225166b5dbd54a3872abfc91e77b0 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 21 Aug 2015 16:29:27 +0200 Subject: [PATCH 009/211] Document built-in add-ons --- docs/topics/addons.rst | 123 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 39ef286eb..4dab15a2a 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -385,3 +385,126 @@ Forward to other add-ons depending on Python version:: _addon = 'path.to.addon' else: _addon = 'path.to.other.addon' + + +Built-in add-on reference +========================= + +Scrapy comes with gateway add-ons that you can use to configure the built-in +middlewares and extensions. For example, to activate and configure the +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`, instead +of placing this in your ``settings.py``:: + + HTTPCACHE_ENABLED = True + HTTPCACHE_EXPIRATION_SECS = 60 + HTTPCACHE_IGNORE_HTTP_CODES = [404] + +you can also use the add-on framework:: + + INSTALLED_ADDONS = ( + # ..., + 'httpcache', + ) + + HTTPCACHE = { + 'expiration_secs': 60, + 'ignore_http_codes': [404], + } + +Note that you *must* enable built-in addons by placing them in your +``INSTALLED_ADDONS`` setting before you can use them for configuring built-in +components. I.e., configuring the ``HTTPCACHE`` setting will have no effect +when ``httpcache`` is not listed in ``INSTALLED_ADDONS``. + +In general, the add-on names match the lowercase name of the component, with its +type suffix removed (i.e. the add-on configuring the +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is called +``httpcache``), and the configuration option names match the names of the +settings they map to, with the component prefix removed (i.e. +``expiration_secs`` maps to :setting:`HTTPCACHE_EXPIRATION_SECS`, as above). +The available add-ons are: + + ++--------------------------------------+--------------------------------------+ +| Add-on | Notes | ++======================================+======================================+ +| **Spider middlewares** | ++--------------------------------------+--------------------------------------+ +| depth (:class:`~scrapy.spidermi\ | | +| ddlewares.depth.DepthMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| httperror (:class:`~scrapy.spid\ | | +| ermiddlewares.httperror.HttpErrorMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| offsite (:class:`~scrapy.spid\ | | +| ermiddlewares.offsite.OffsiteMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| referer (:class:`~scrapy.spid\ | | +| ermiddlewares.referer.RefererMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| urllength (:class:`~scrapy.spid\ | | +| ermiddlewares.urllength.UrlLengthMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| **Downloader middlewares** | ++--------------------------------------+--------------------------------------+ +| ajaxcrawl (:class:`~scrapy.download\ | | +| ermiddlewares.ajaxcrawl.AjaxCrawlMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| chunked (:class:`~scrapy.download\ | | +| ermiddlewares.chunked.ChunkedTrans\ | | +| ferMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| cookies (:class:`~scrapy.download\ | | +| ermiddlewares.cookies.CookiesMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| defaultheaders (:class:`~scrapy.down\| Every configuration entry is treated | +| loadermiddlewares.defaultheaders.Def\| as a default header. | +| aultHeadersMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| **Extensions** | ++--------------------------------------+--------------------------------------+ +| autothrottle | Installing sets | +| (:ref:`topics-autothrottle`) | :setting:`AUTOTHROTTLE_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| corestats (:class:`~scrapy.exten\ | | +| sions.corestats.CoreStats`) | | ++--------------------------------------+--------------------------------------+ +| closespider (:class:`~scrapy.exten\ | | +| sions.closespider.CloseSpider`) | | ++--------------------------------------+--------------------------------------+ +| debugger (:class:`~scrapy.exten\ | | +| sions.debug.Debugger`) | | ++--------------------------------------+--------------------------------------+ +| feedexport (:ref:`topics-feed-expor\ | | +| ts`) | | ++--------------------------------------+--------------------------------------+ +| logstats (:class:`~scrapy.exten\ | | +| sions.logstats.LogStats`) | | ++--------------------------------------+--------------------------------------+ +| memdebug (:class:`~scrapy.exten\ | Installing sets | +| sions.memdebug.MemoryDebugger`) | :setting:`MEMDEBUG_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| memusage (:class:`~scrapy.exten\ | Installing sets | +| sions.memusage.MemoryUsage`) | :setting:`MEMUSAGE_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| spiderstate (:class:`~scrapy.exten\ | | +| sions.spiderstate.SpiderState`) | | ++--------------------------------------+--------------------------------------+ +| stacktracedump (:class:`~scrapy.ext\ | | +| ensions.debug.StackTraceDump`) | | ++--------------------------------------+--------------------------------------+ +| statsmailer (:class:`~scrapy.exten\ | | +| sions.statsmailer.StatsMailer`) | | ++--------------------------------------+--------------------------------------+ +| telnetconsole (:ref:`topics-telnet\ | | +| console`) | | ++--------------------------------------+--------------------------------------+ From d18b6a61d7db295c5f4b1800402a064ed1f40059 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 29 Oct 2015 16:34:52 +0100 Subject: [PATCH 010/211] Add missing AddonManager tests --- tests/test_addons/__init__.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 84870ec52..ab06023e3 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -14,6 +14,7 @@ from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings +from scrapy.utils.conf import config_from_filepath from . import addons from . import addonmod @@ -244,16 +245,22 @@ class AddonManagerTest(unittest.TestCase): _test_load_method('load_settings', settings) def test_load_cfg(self): + def _check_loaded_addons(manager): + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val2') manager = AddonManager() manager.load_cfg(self.TESTCFGPATH) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val2') + _check_loaded_addons(manager) + manager = AddonManager() + preloaded_cfg = config_from_filepath(self.TESTCFGPATH) + manager.load_cfg(preloaded_cfg) + _check_loaded_addons(manager) def test_enabled_disabled(self): manager = AddonManager() @@ -330,6 +337,11 @@ class AddonManagerTest(unittest.TestCase): self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) + # This will become relevant when we let spiders implement the add-on + # interface and should be replaced with a test where + # AddonManager.spidercls = None then. + manager._call_if_exists(None, 'irrelevant') + def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): name = 'AddedAddon' From 9f7fcf5582ed61787180df20a62a301cc09fce4a Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 22:18:10 +0100 Subject: [PATCH 011/211] Make update_classpath() util function return non-string objects --- scrapy/utils/deprecate.py | 3 +++ tests/test_utils_deprecate.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 0fc33e0c4..9293b1480 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,5 +1,6 @@ """Some helpers for deprecation messages""" +import six import warnings import inspect from scrapy.exceptions import ScrapyDeprecationWarning @@ -149,6 +150,8 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" + if not isinstance(path, six.string_types): + return path for prefix, replacement in DEPRECATION_RULES: if path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 3e7236fb1..7a35c424b 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -279,3 +279,7 @@ class UpdateClassPathTest(unittest.TestCase): output = update_classpath('scrapy.unmatched.Path') self.assertEqual(output, 'scrapy.unmatched.Path') self.assertEqual(len(w), 0) + + def test_returns_nonstring(self): + for notastring in [None, True, [1, 2, 3], object()]: + self.assertEqual(update_classpath(notastring), notastring) From f7ed239fcb47663d881e72ffab4386244b215ce4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 22:27:01 +0100 Subject: [PATCH 012/211] Replace INSTALLED_ADDONS tuple setting with ADDONS dictionary setting --- scrapy/addons/__init__.py | 15 ++++++++------- scrapy/settings/default_settings.py | 4 ++-- tests/test_addons/__init__.py | 26 ++++++++++++++++++++++---- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index b1d6e14cb..a1a9a388a 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,4 +1,4 @@ -from collections import defaultdict, Mapping +from collections import defaultdict, Mapping, OrderedDict from inspect import isclass import six import warnings @@ -9,7 +9,8 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.utils.conf import config_from_filepath, get_config +from scrapy.utils.conf import (build_component_list, config_from_filepath, + get_config) from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path @@ -166,7 +167,7 @@ class AddonManager(Mapping): """ def __init__(self): - self._addons = {} + self._addons = OrderedDict() self.configs = {} self._disable_on_add = [] @@ -310,15 +311,15 @@ class AddonManager(Mapping): """Load add-ons and configurations from settings object. This will invoke :meth:`get_addon` for every add-on path in the - ``INSTALLED_ADDONS`` setting. For each of these add-ons, the - configuration will be read from the dictionary setting whose name - matches the uppercase add-on name. + ``ADDONS`` setting. For each of these add-ons, the configuration will be + read from the dictionary setting whose name matches the uppercase add-on + name. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = settings.getlist('INSTALLED_ADDONS') + paths = build_component_list(settings['ADDONS']) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a230750fb..6068a1393 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,6 +18,8 @@ import sys from importlib import import_module from os.path import join, abspath, dirname +ADDONS = {} + AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -167,8 +169,6 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False -INSTALLED_ADDONS = () - ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index ab06023e3..aa8dfbb63 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,3 +1,5 @@ +from collections import OrderedDict +import itertools import os.path import six from tests import mock @@ -236,14 +238,30 @@ class AddonManagerTest(unittest.TestCase): _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('INSTALLED_ADDONS', [ - self.ADDONMODPATH, - 'tests.test_addons.addons.GoodAddon', - ]) + settings.set('ADDONS', {self.ADDONMODPATH: 0, + 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) _test_load_method('load_settings', settings) + def test_load_dict_load_settings_order(self): + def _test_load_method(expected_order, func, *args, **kwargs): + manager = AddonManager() + getattr(manager, func)(*args, **kwargs) + self.assertEqual(list(manager.keys()), expected_order) + + # Get three addons named 0, 1, 2 + addonlist = [addons.GoodAddon(str(x)) for x in range(3)] + # Test both methods for every possible mutation + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.name for a in ordered_addons] + addonsdict = OrderedDict((a, {}) for a in ordered_addons) + _test_load_method(expected_order, 'load_dict', addonsdict) + settings = BaseSettings({ + 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} + }) + _test_load_method(expected_order, 'load_settings', settings) + def test_load_cfg(self): def _check_loaded_addons(manager): six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) From b10caf91a14dcf0d0d4fe89865a53e2542588772 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 23:14:42 +0100 Subject: [PATCH 013/211] Drop support for add-on configuration in scrapy.cfg --- scrapy/addons/__init__.py | 28 +--------------------------- scrapy/cmdline.py | 1 - tests/test_addons/__init__.py | 20 -------------------- tests/test_addons/cfg.cfg | 5 ----- 4 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 tests/test_addons/cfg.cfg diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index a1a9a388a..40a98676d 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -9,8 +9,7 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.utils.conf import (build_component_list, config_from_filepath, - get_config) +from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path @@ -325,31 +324,6 @@ class AddonManager(Mapping): for a, c in zip(addons, configs): self.add(a, c) - def load_cfg(self, cfg=None): - """Load add-ons and configurations from given ``ConfigParser`` object or - config file path. - - Each add-on should have its own section, where the section has a name in - the form ``addon:my_addon_path``. The add-on object is searched for via - the :meth:`get_addon` method, ``my_addon_path`` can be either a Python - or a file path. - - If ``cfg`` is ``None``, ``scrapy.cfg`` will be used. - - :param cfg: ``ConfigParser`` object or config file path from which to \ - read add-on configuration - :type cfg: ``ConfigParser`` or ``str`` - """ - if cfg is None: - cfg = get_config() - elif isinstance(cfg, six.string_types): - cfg = config_from_filepath(cfg) - for secname in cfg.sections(): - if secname.startswith("addon:"): - addonkey = secname.split("addon:", 1)[1] - addoncfg = dict(cfg.items(secname)) - self.add(addonkey, addoncfg) - def check_dependency_clashes(self): """Check for incompatibilities in add-on dependencies. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b403df570..b7c349793 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -120,7 +120,6 @@ def execute(argv=None, settings=None): # ------------------------------------------------------------------ addons = AddonManager() - addons.load_cfg() inproject = inside_project() cmds = _get_commands_dict(settings, inproject) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index aa8dfbb63..f10819685 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -16,7 +16,6 @@ from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from scrapy.utils.conf import config_from_filepath from . import addons from . import addonmod @@ -107,7 +106,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg') ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') def setUp(self): @@ -262,24 +260,6 @@ class AddonManagerTest(unittest.TestCase): }) _test_load_method(expected_order, 'load_settings', settings) - def test_load_cfg(self): - def _check_loaded_addons(manager): - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val2') - manager = AddonManager() - manager.load_cfg(self.TESTCFGPATH) - _check_loaded_addons(manager) - manager = AddonManager() - preloaded_cfg = config_from_filepath(self.TESTCFGPATH) - manager.load_cfg(preloaded_cfg) - _check_loaded_addons(manager) - def test_enabled_disabled(self): manager = AddonManager() manager.add(addons.GoodAddon('FirstAddon')) diff --git a/tests/test_addons/cfg.cfg b/tests/test_addons/cfg.cfg deleted file mode 100644 index 98c4f0f25..000000000 --- a/tests/test_addons/cfg.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[addon:tests.test_addons.addons.GoodAddon] -key = val1 - -[addon:tests/test_addons/addonmod.py] -key = val2 From 8e5d067af1e50a32f8930294199798535357edf2 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Sat, 7 Nov 2015 19:56:47 +0100 Subject: [PATCH 014/211] Drop BaseSettings.get_addon() prefix magic --- scrapy/addons/__init__.py | 27 +++---------------- tests/test_addons/__init__.py | 27 ------------------- tests/test_addons/project/__init__.py | 0 tests/test_addons/project/addons/__init__.py | 0 tests/test_addons/project/addons/addonmod.py | 7 ----- tests/test_addons/project/addons/addonmod2.py | 7 ----- tests/test_addons/scrapy_addons/__init__.py | 0 tests/test_addons/scrapy_addons/addonmod.py | 7 ----- tests/test_addons/scrapy_addons/addonmod2.py | 7 ----- tests/test_addons/scrapy_addons/addonmod3.py | 7 ----- 10 files changed, 4 insertions(+), 85 deletions(-) delete mode 100644 tests/test_addons/project/__init__.py delete mode 100644 tests/test_addons/project/addons/__init__.py delete mode 100644 tests/test_addons/project/addons/addonmod.py delete mode 100644 tests/test_addons/project/addons/addonmod2.py delete mode 100644 tests/test_addons/scrapy_addons/__init__.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod2.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod3.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 40a98676d..ddd18c7fb 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -7,11 +7,9 @@ from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface from zope.interface.verify import verifyObject -from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_module_or_object -from scrapy.utils.project import get_project_path @zope.interface.implementer(IAddon) @@ -244,14 +242,8 @@ class AddonManager(Mapping): """Get an add-on object by its Python or file path. ``path`` is assumed to be either a Python or a file path of a Scrapy - add-on. If no object is found at ``path``, it is tried again first with - ``projectname.addons`` prepended (pointing to the current project's - ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to - Scrapy's built-in add-ons). These convenience shortcuts will only work - with Python paths, not file paths. - - If the object or module pointed to by ``path`` has an attribute named - ``_addon`` that attribute will be assumed to be the add-on. + add-on. If the object or module pointed to by ``path`` has an attribute + named ``_addon`` that attribute will be assumed to be the add-on. :meth:`get_addon` will keep following ``_addon`` attributes until it finds an object that does not have an attribute named ``_addon``. @@ -259,20 +251,9 @@ class AddonManager(Mapping): :type path: ``str`` """ if isinstance(path, six.string_types): - prefixes = ['', 'scrapy.addons.'] try: - prefixes.insert(1, get_project_path() + '.addons.') - except NotConfigured: - warnings.warn("Unable to locate project Python path") - for prefix in prefixes: - fullpath = prefix + path - try: - obj = load_module_or_object(fullpath) - except NameError: - pass - else: - break - else: + obj = load_module_or_object(path) + except NameError: raise NameError("Could not find add-on '%s'" % path) else: obj = path diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index f10819685..4f1074221 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -11,7 +11,6 @@ import zope.interface from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation -import scrapy.addons from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon @@ -187,32 +186,6 @@ class AddonManagerTest(unittest.TestCase): x._addon._addon = addons.GoodAddon('inner') self.assertIs(self.manager.get_addon(x), x._addon._addon) - @mock.patch.object(scrapy.addons, 'get_project_path', - return_value='tests.test_addons.project') - def test_get_addon_prefixes(self, get_project_path_mock): - # From python path - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - - # From project 'addons' folder - self.assertEqual(self.manager.get_addon('addonmod2').FROM, - 'test_addons.project.addons.addonmod2') - # Assert prefix priority '' > 'project.addons' - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - - # From scrapy's 'addons' - from . import scrapy_addons - with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}): - self.assertEqual(self.manager.get_addon('addonmod3').FROM, - 'test_addons.scrapy_addons.addonmod3') - # Assert prefix priority 'project.addons' > 'scrapy.addons' - self.assertEqual(self.manager.get_addon('addonmod2').FROM, - 'test_addons.project.addons.addonmod2') - # Assert prefix priority '' > 'scrapy.addons.' - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() diff --git a/tests/test_addons/project/__init__.py b/tests/test_addons/project/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/project/addons/__init__.py b/tests/test_addons/project/addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/project/addons/addonmod.py b/tests/test_addons/project/addons/addonmod.py deleted file mode 100644 index 66ca644f8..000000000 --- a/tests/test_addons/project/addons/addonmod.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.project.addons.addonmod' diff --git a/tests/test_addons/project/addons/addonmod2.py b/tests/test_addons/project/addons/addonmod2.py deleted file mode 100644 index 0dbdd70ff..000000000 --- a/tests/test_addons/project/addons/addonmod2.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.project.addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/__init__.py b/tests/test_addons/scrapy_addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/scrapy_addons/addonmod.py b/tests/test_addons/scrapy_addons/addonmod.py deleted file mode 100644 index fa479aa68..000000000 --- a/tests/test_addons/scrapy_addons/addonmod.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod' diff --git a/tests/test_addons/scrapy_addons/addonmod2.py b/tests/test_addons/scrapy_addons/addonmod2.py deleted file mode 100644 index da053af4a..000000000 --- a/tests/test_addons/scrapy_addons/addonmod2.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/addonmod3.py b/tests/test_addons/scrapy_addons/addonmod3.py deleted file mode 100644 index c64521478..000000000 --- a/tests/test_addons/scrapy_addons/addonmod3.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod3' From 388c5c4b78509adaf801fb8f66d2d1fdbe1ce9d4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 00:36:16 +0100 Subject: [PATCH 015/211] Fix component exporting for Addon base class --- scrapy/addons/__init__.py | 2 +- tests/test_addons/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index ddd18c7fb..48de99c1a 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -84,7 +84,7 @@ class Addon(object): # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp v = config.get('order', self.component_order) - settings.set(self.component_type, {k: v}, 'addon') + settings[self.component_type].update({k: v}, 'addon') def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 4f1074221..b135dd04d 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -38,7 +38,9 @@ class AddonTest(unittest.TestCase): verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default') + settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), + 'DOWNLOAD_HANDLERS': BaseSettings()}, + 'default') self.testaddon.component_type = None self.testaddon.export_component({}, settings) self.assertEqual(len(settings['ITEM_PIPELINES']), 0) From e924d382380a6bc7285b8a9713ac14588681af3b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 01:09:23 +0100 Subject: [PATCH 016/211] Drop component configuration (copied from default_settings) from built-in add-ons --- scrapy/addons/builtins.py | 244 +++++------------------------ tests/test_addons/test_builtins.py | 21 +-- 2 files changed, 37 insertions(+), 228 deletions(-) diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index ff7902afb..9babdeb6f 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -13,17 +13,14 @@ __all__ = ['make_builtin_addon', 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', 'statsmailer', 'telnetconsole', - ] + ] -def make_builtin_addon(addon_name, comp_type, comp, order=0, - addon_default_config=None, addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, + addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ - component_type = comp_type - component = comp - component_order = order default_config = addon_default_config or {} return ThisAddon @@ -36,73 +33,26 @@ def make_builtin_addon(addon_name, comp_type, comp, order=0, # SPIDER MIDDLEWARES -depth = make_builtin_addon( - 'depth', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.depth.DepthMiddleware', - 900, -) +depth = make_builtin_addon('depth') -httperror = make_builtin_addon( - 'httperror', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 50, -) +httperror = make_builtin_addon('httperror') -offsite = make_builtin_addon( - 'offsite', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', - 500, -) +offsite = make_builtin_addon('offsite') -referer = make_builtin_addon( - 'referer', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.referer.RefererMiddleware', - 700, - {'enabled': True}, -) +referer = make_builtin_addon('referer') -urllength = make_builtin_addon( - 'urllength', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', - 800, -) +urllength = make_builtin_addon('urllength') # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon( - 'ajaxcrawl', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware', - 560, -) +ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) -chunked = make_builtin_addon( - 'chunked', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware', - 830, -) +chunked = make_builtin_addon('chunked') -cookies = make_builtin_addon( - 'cookies', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', - 700, - {'enabled': True}, -) +cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon( - 'defaultheaders', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', - 550, -) +defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} @@ -110,92 +60,31 @@ def defaultheaders_export_config(self, config, settings): settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon( - 'downloadtimeout', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', - 350, -) +downloadtimeout = make_builtin_addon('downloadtimeout') downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', 'download_timeout': 'DOWNLOAD_TIMEOUT'} -httpauth = make_builtin_addon( - 'httpauth', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', - 300, -) +httpauth = make_builtin_addon('httpauth') -httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, -) +httpcache = make_builtin_addon('httpcache', {'enabled': True}) -httpcompression = make_builtin_addon( - 'httpcompression', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', - 590, - {'enabled': True}, -) +httpcompression = make_builtin_addon('httpcompression') httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} -httpproxy = make_builtin_addon( - 'httpproxy', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', - 750, -) +httpproxy = make_builtin_addon('httpproxy') -metarefresh = make_builtin_addon( - 'metarefresh', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', - 580, - {'enabled': True}, -) +metarefresh = make_builtin_addon('metarefresh') metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} -redirect = make_builtin_addon( - 'redirect', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', - 600, - {'enabled': True}, -) +redirect = make_builtin_addon('redirect') -retry = make_builtin_addon( - 'retry', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.retry.RetryMiddleware', - 500, - {'enabled': True}, -) +retry = make_builtin_addon('retry') -robotstxt = make_builtin_addon( - 'robotstxt', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', - 100, - {'obey': True}, -) +robotstxt = make_builtin_addon('robotstxt', {'obey': True}) -stats = make_builtin_addon( - 'stats', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.stats.DownloaderStats', - 850, -) +stats = make_builtin_addon('stats') -useragent = make_builtin_addon( - 'useragent', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', - 400, -) +useragent = make_builtin_addon('useragent') useragent.config_mapping = {'user_agent': 'USER_AGENT'} @@ -204,90 +93,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon( - 'throttle', - 'EXTENSIONS', - 'scrapy.extensions.throttle.AutoThrottle', - 0, - {'enabled': True}, -) +autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) -corestats = make_builtin_addon( - 'corestats', - 'EXTENSIONS' - 'scrapy.extensions.corestats.CoreStats', - 0, -) +corestats = make_builtin_addon('corestats') -closespider = make_builtin_addon( - 'closespider', - 'EXTENSIONS' - 'scrapy.extensions.closespider.CloseSpider', - 0, -) +closespider = make_builtin_addon('closespider') -debugger = make_builtin_addon( - 'debugger', - 'EXTENSIONS' - 'scrapy.extensions.debug.Debugger', - 0, -) +debugger = make_builtin_addon('debugger') -feedexport = make_builtin_addon( - 'feedexport', - 'EXTENSIONS' - 'scrapy.extensions.feedexport.FeedExporter', - 0, -) +feedexport = make_builtin_addon('feedexport') feedexport.settings_prefix = 'FEED' -logstats = make_builtin_addon( - 'logstats', - 'EXTENSIONS' - 'scrapy.extensions.logstats.LogStats', - 0, -) +logstats = make_builtin_addon('logstats') -memdebug = make_builtin_addon( - 'memdebug', - 'EXTENSIONS' - 'scrapy.extensions.memdebug.MemoryDebugger', - 0, - {'enabled': True}, -) +memdebug = make_builtin_addon('memdebug', {'enabled': True}) -memusage = make_builtin_addon( - 'memusage', - 'EXTENSIONS' - 'scrapy.extensions.memusage.MemoryUsage', - 0, - {'enabled': True}, -) +memusage = make_builtin_addon('memusage', {'enabled': True}) -spiderstate = make_builtin_addon( - 'spiderstate', - 'EXTENSIONS' - 'scrapy.extensions.spiderstate.SpiderState', - 0, -) +spiderstate = make_builtin_addon('spiderstate') -stacktracedump = make_builtin_addon( - 'stacktracedump', - 'EXTENSIONS' - 'scrapy.extensions.debug.StackTraceDump', - 0, -) +stacktracedump = make_builtin_addon('stacktracedump') -statsmailer = make_builtin_addon( - 'statsmailer', - 'EXTENSIONS' - 'scrapy.extensions.statsmailer.StatsMailer', - 0, -) +statsmailer = make_builtin_addon('statsmailer') -telnetconsole = make_builtin_addon( - 'telnetconsole', - 'EXTENSIONS' - 'scrapy.telnet.TelnetConsole', - 0, -) +telnetconsole = make_builtin_addon('telnetconsole') diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index 607c911fb..c89876950 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -9,28 +9,11 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): def test_make_builtin_addon(self): - httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, - ) + httpcache = make_builtin_addon('httpcache', {'enabled': True}) self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.component_type, 'DOWNLOADER_MIDDLEWARES') - self.assertEqual(httpcache.component, 'scrapy.downloadermiddlewares.' - 'httpcache.HttpCacheMiddleware') - self.assertEqual(httpcache.component_order, 900) self.assertEqual(httpcache.default_config, {'enabled': True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, - '99.9', - ) + httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') self.assertEqual(httpcache.version, '99.9') def test_defaultheaders_export_config(self): From b7b00fb95669196167a5e5f76b1f3b8e2feb7b84 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 01:18:41 +0100 Subject: [PATCH 017/211] PEP8ify add-ons and tests --- scrapy/addons/__init__.py | 6 ++++-- tests/test_addons/__init__.py | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 48de99c1a..169c79eac 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -323,6 +323,7 @@ class AddonManager(Mapping): """ # Collect all active add-ons and the components they provide ws = WorkingSet('') + def add_dist(project_name, version, **kwargs): if project_name in ws.entry_keys.get('scrapy', []): raise ImportError("Component {} provided by multiple add-ons" @@ -354,8 +355,9 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr)) + "Add-ons {} require or modify missing component {}" + "".format(required[reqstr]+modified[reqstr], reqstr) + ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index b135dd04d..32ee25ad8 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -24,6 +24,7 @@ class AddonTest(unittest.TestCase): def setUp(self): self.rawaddon = Addon() + class AddonWithAttributes(Addon): name = 'Test' version = '1.0' @@ -132,7 +133,7 @@ class AddonManagerTest(unittest.TestCase): def test_add_verifies(self): brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') + 'tests.test_addons.addons.BrokenAddon') self.assertRaises(zope.interface.exceptions.BrokenImplementation, self.manager.add, brokenaddon) @@ -145,11 +146,13 @@ class AddonManagerTest(unittest.TestCase): def test_remove(self): manager = AddonManager() + def test_gets_removed(removearg): manager.add(addonmod) self.assertIn('AddonModule', manager) manager.remove(removearg) self.assertNotIn('AddonModule', manager) + test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') @@ -158,8 +161,7 @@ class AddonManagerTest(unittest.TestCase): self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon( - 'tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) @@ -194,8 +196,7 @@ class AddonManagerTest(unittest.TestCase): getattr(manager, func)(*args, **kwargs) six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], - ['key']) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') # XXX: Check module equality, see above self.assertEqual(manager['AddonModule'].name, addonmod.name) @@ -284,17 +285,17 @@ class AddonManagerTest(unittest.TestCase): ua_first.assert_called_once_with(manager.configs['FirstAddon'], manager) ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + manager) manager.update_settings(settings) us_first.assert_called_once_with(manager.configs['FirstAddon'], settings) us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + settings) manager.check_configuration(crawler) cc_first.assert_called_once_with(manager.configs['FirstAddon'], crawler) cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -321,6 +322,7 @@ class AddonManagerTest(unittest.TestCase): class FirstAddon(addons.GoodAddon): name = 'FirstAddon' + def update_addons(self, config, addons): addons.add(AddedAddon()) From 33dfb3e167f7d31e2573ab39c2b80bf7728c2b00 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 11:22:24 +0100 Subject: [PATCH 018/211] Remove unused project path util function --- scrapy/utils/project.py | 12 ------------ tests/test_utils_project.py | 27 --------------------------- 2 files changed, 39 deletions(-) delete mode 100644 tests/test_utils_project.py diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a1266c879..a15a0d90f 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -71,15 +71,3 @@ def get_project_settings(): settings.setdict(env_overrides, priority='project') return settings - -def get_project_path(): - """Return the Python path of the current project. - - This fails when the settings module does not live in the project's root. - """ - if not inside_project(): - raise NotConfigured("Not inside a project") - settings_module_path = os.environ.get(ENVVAR) - if not settings_module_path: - raise NotConfigured("Unable to locate project's python path") - return settings_module_path.rsplit('.', 1)[0] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py deleted file mode 100644 index cea4d9950..000000000 --- a/tests/test_utils_project.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -from tests import mock -import unittest - -from scrapy.exceptions import NotConfigured -from scrapy.utils.project import get_project_path, inside_project - - -class UtilsProjectTestCase(unittest.TestCase): - - @mock.patch('scrapy.utils.project.inside_project', return_value=True) - def test_get_project_path(self, mock_ip): - def _test(settingsmod, expected): - with mock.patch.dict('os.environ', - {'SCRAPY_SETTINGS_MODULE': settingsmod}): - self.assertEqual(get_project_path(), expected) - _test('project.settings', 'project') - _test('project.othername', 'project') - _test('nested.project.settings', 'nested.project') - - with mock.patch.dict('os.environ', {}, clear=True): - self.assertRaises(NotConfigured, get_project_path) - - mock_ip.return_value = False - with mock.patch.dict('os.environ', - {'SCRAPY_SETTINGS_MODULE': 'some.settings'}): - self.assertRaises(NotConfigured, get_project_path) From a65fc0db7d7d728bc8abbb37eff358870ce7a9b5 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 12 Nov 2015 18:37:26 +0100 Subject: [PATCH 019/211] Drop support for providing file paths as add-ons --- scrapy/utils/misc.py | 24 --------------------- tests/test_addons/__init__.py | 26 ++++++----------------- tests/test_utils_misc/__init__.py | 19 +---------------- tests/test_utils_misc/testpkg/__init__.py | 1 - tests/test_utils_misc/testpkg/submod.py | 1 - 5 files changed, 8 insertions(+), 63 deletions(-) delete mode 100644 tests/test_utils_misc/testpkg/__init__.py delete mode 100644 tests/test_utils_misc/testpkg/submod.py diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 9461d93e9..e55edd63e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,8 +1,5 @@ """Helper functions which doesn't fit anywhere else""" -import itertools -import os.path import re -import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -72,10 +69,6 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - try: - return get_module_from_filepath(path) - except ImportError: - pass raise NameError("Could not load '%s'" % path) @@ -101,23 +94,6 @@ def walk_modules(path): return mods -def get_module_from_filepath(path): - """Load and return a python module/package from a file path""" - path = path.rstrip("/") - if path.endswith('.py'): - path = path.rsplit('.py', 1)[0] - basefolder, modname = os.path.split(path) - # XXX: There are other ways to import modules from a full path which don't - # need to modify PYTHONPATH, see - # https://stackoverflow.com/questions/67631/ - # These methods differ between py2 and py3, and apparently the - # py3 method was deprecated in Python 3.4 - sys.path.insert(0, basefolder) - mod = import_module(modname) - sys.path.pop(0) - return mod - - def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 32ee25ad8..a4e278fa5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -108,8 +108,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') - def setUp(self): self.manager = AddonManager() @@ -156,7 +154,6 @@ class AddonManagerTest(unittest.TestCase): test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') - test_gets_removed(self.ADDONMODPATH) self.assertRaises(KeyError, manager.remove, 'nonexistent') self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) @@ -164,18 +161,12 @@ class AddonManagerTest(unittest.TestCase): goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) - loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) - # XXX: The module is in fact imported twice under different names into - # sys.modules, is there a good assertion for module equality? - self.assertEqual(loaded_addonmod.name, addonmod.name) + loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") + self.assertIs(loaded_addonmod, addonmod) - # Does not provide interface, but has _addon attribute pointing to - # GoodAddon instance addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') - goodaddon = self.manager.get_addon(addonspath) - # XXX: Again, the imported class and addons.GoodAddon are different - # since they are imported twice. How to use isInstance? - self.assertEqual(goodaddon.name, addons.GoodAddon.name) + goodaddon = self.manager.get_addon("tests.test_addons.addons") + self.assertIsInstance(goodaddon, addons.GoodAddon) self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') @@ -198,21 +189,18 @@ class AddonManagerTest(unittest.TestCase): self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertEqual(manager['AddonModule'], addonmod) self.assertIn('key', manager.configs['AddonModule']) self.assertEqual(manager.configs['AddonModule']['key'], 'val1') addonsdict = { - self.ADDONMODPATH: { - 'key': 'val1', - }, + "tests.test_addons.addonmod": {'key': 'val1'}, 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, } _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('ADDONS', {self.ADDONMODPATH: 0, + settings.set('ADDONS', {"tests.test_addons.addonmod": 0, 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index f33562b7d..8ea8786d7 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from scrapy.item import Item, Field from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, - walk_modules, get_module_from_filepath) + walk_modules) __doctests__ = ['scrapy.utils.misc'] @@ -21,9 +21,6 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_module_or_object(self): testmod = load_module_or_object(__name__ + '.testmod') self.assertTrue(hasattr(testmod, 'TESTVAR')) - testmod = load_module_or_object( - os.path.join(os.path.dirname(__file__), 'testmod.py')) - self.assertTrue(hasattr(testmod, 'TESTVAR')) obj = load_object('scrapy.utils.misc.load_object') self.assertIs(obj, load_object) @@ -67,20 +64,6 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) - def test_get_module_from_filepath(self): - testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') - testmod = get_module_from_filepath(testmodpath) - self.assertTrue(hasattr(testmod, 'TESTVAR')) - - testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') - testpkg = get_module_from_filepath(testpkgpath) - self.assertTrue(hasattr(testpkg, 'TESTVAR2')) - # Check submodule access - import testpkg.submod - self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) - self.assertIs(testpkg.submod.TESTVAR3, - load_object(testpkg.__name__ + ".submod.TESTVAR3")) - def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py deleted file mode 100644 index 12cc2f6d9..000000000 --- a/tests/test_utils_misc/testpkg/__init__.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py deleted file mode 100644 index 8a07e3592..000000000 --- a/tests/test_utils_misc/testpkg/submod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR3 = True From 52d0df5f989903d46e6de4878e1d6a0e87a2c803 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 12 May 2021 13:08:08 -0300 Subject: [PATCH 020/211] CaseInsensitiveDict (deprecate CaselessDict) --- scrapy/http/headers.py | 13 ++++--- scrapy/pipelines/files.py | 4 +- scrapy/utils/datatypes.py | 46 +++++++++++++++++++++++ tests/test_utils_datatypes.py | 71 +++++++++++++++++++++++++---------- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1a2b99b0a..dfbcf8361 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,5 +1,6 @@ from w3lib.http import headers_dict_to_raw -from scrapy.utils.datatypes import CaselessDict + +from scrapy.utils.datatypes import CaseInsensitiveDict, CaselessDict from scrapy.utils.python import to_unicode @@ -76,13 +77,13 @@ class Headers(CaselessDict): return headers_dict_to_raw(self) def to_unicode_dict(self): - """ Return headers as a CaselessDict with unicode keys + """ Return headers as a CaseInsensitiveDict with unicode keys and unicode values. Multiple values are joined with ','. """ - return CaselessDict( - (to_unicode(key, encoding=self.encoding), - to_unicode(b','.join(value), encoding=self.encoding)) - for key, value in self.items()) + return CaseInsensitiveDict( + (to_unicode(key, encoding=self.encoding), to_unicode(b','.join(value), encoding=self.encoding)) + for key, value in self.items() + ) def __copy__(self): return self.__class__(self) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 13ecd4e6c..2f1a25dfc 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -23,7 +23,7 @@ from scrapy.http import Request from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.boto import is_botocore_available -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import md5sum @@ -143,7 +143,7 @@ class S3FilesStore: """ Convert headers to botocore keyword agruments. """ # This is required while we need to support both boto and botocore. - mapping = CaselessDict({ + mapping = CaseInsensitiveDict({ 'Content-Type': 'ContentType', 'Cache-Control': 'CacheControl', 'Content-Disposition': 'ContentDisposition', diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e31284a7f..ca6089e0f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,14 +6,30 @@ This module must not depend on any module outside the Standard Library. """ import collections +import warnings import weakref from collections.abc import Mapping +from typing import Any, AnyStr + +from scrapy.exceptions import ScrapyDeprecationWarning class CaselessDict(dict): __slots__ = () + def __new__(cls, *args, **kwargs): + from scrapy.http.headers import Headers + + if issubclass(cls, CaselessDict) and not issubclass(cls, Headers): + warnings.warn( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return super().__new__(cls, *args, **kwargs) + def __init__(self, seq=None): super().__init__() if seq: @@ -63,6 +79,36 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) +class CaseInsensitiveDict(collections.UserDict): + """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. + + It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. + """ + + def __getitem__(self, key: AnyStr) -> Any: + return super().__getitem__(self.normkey(key)) + + def __setitem__(self, key: AnyStr, value: Any) -> None: + super().__setitem__(self.normkey(key), self.normvalue(value)) + + def __delitem__(self, key: AnyStr) -> None: + super().__delitem__(self.normkey(key)) + + def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + return super().__contains__(self.normkey(key)) + + def normkey(self, key: AnyStr) -> AnyStr: + """Method to normalize dictionary key access""" + return key.lower() + + def normvalue(self, value: Any) -> Any: + """Method to normalize values prior to be set""" + return value + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {super().__repr__()}>" + + class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index e4bccf30e..c033cd537 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,26 +1,34 @@ import copy import unittest +import warnings from collections.abc import Mapping, MutableMapping +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request -from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude +from scrapy.utils.datatypes import ( + CaseInsensitiveDict, + CaselessDict, + LocalCache, + LocalWeakReferencedCache, + SequenceExclude, +) from scrapy.utils.python import garbage_collect __doctests__ = ['scrapy.utils.datatypes'] -class CaselessDictTest(unittest.TestCase): +class CaseInsensitiveDictMixin: def test_init_dict(self): seq = {'red': 1, 'black': 3} - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) def test_init_pair_sequence(self): seq = (('red', 1), ('black', 3)) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) @@ -39,7 +47,7 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) @@ -64,12 +72,12 @@ class CaselessDictTest(unittest.TestCase): return len(self._d) seq = MyMutableMapping(red=1, black=3) - d = CaselessDict(seq) + d = self.dict_class(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) def test_caseless(self): - d = CaselessDict() + d = self.dict_class() d['key_Lower'] = 1 self.assertEqual(d['KEy_loWer'], 1) self.assertEqual(d.get('KEy_loWer'), 1) @@ -79,19 +87,19 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get('key_Lower'), 3) def test_delete(self): - d = CaselessDict({'key_lower': 1}) + d = self.dict_class({'key_lower': 1}) del d['key_LOWER'] self.assertRaises(KeyError, d.__getitem__, 'key_LOWER') self.assertRaises(KeyError, d.__getitem__, 'key_lower') def test_getdefault(self): - d = CaselessDict() + d = self.dict_class() self.assertEqual(d.get('c', 5), 5) d['c'] = 10 self.assertEqual(d.get('c', 5), 10) def test_setdefault(self): - d = CaselessDict({'a': 1, 'b': 2}) + d = self.dict_class({'a': 1, 'b': 2}) r = d.setdefault('A', 5) self.assertEqual(r, 1) @@ -104,15 +112,15 @@ class CaselessDictTest(unittest.TestCase): def test_fromkeys(self): keys = ('a', 'b') - d = CaselessDict.fromkeys(keys) + d = self.dict_class.fromkeys(keys) self.assertEqual(d['A'], None) self.assertEqual(d['B'], None) - d = CaselessDict.fromkeys(keys, 1) + d = self.dict_class.fromkeys(keys, 1) self.assertEqual(d['A'], 1) self.assertEqual(d['B'], 1) - instance = CaselessDict() + instance = self.dict_class() d = instance.fromkeys(keys) self.assertEqual(d['A'], None) self.assertEqual(d['B'], None) @@ -122,18 +130,19 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d['B'], 1) def test_contains(self): - d = CaselessDict() + d = self.dict_class() d['a'] = 1 assert 'a' in d + assert 'A' in d def test_pop(self): - d = CaselessDict() + d = self.dict_class() d['a'] = 1 self.assertEqual(d.pop('A'), 1) self.assertRaises(KeyError, d.pop, 'A') def test_normkey(self): - class MyDict(CaselessDict): + class MyDict(self.dict_class): def normkey(self, key): return key.title() @@ -142,7 +151,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(list(d.keys()), ['Key-One']) def test_normvalue(self): - class MyDict(CaselessDict): + class MyDict(self.dict_class): def normvalue(self, value): if value is not None: return value + 1 @@ -171,11 +180,35 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d.get('key'), 2) def test_copy(self): - h1 = CaselessDict({'header1': 'value'}) + h1 = self.dict_class({'header1': 'value'}) h2 = copy.copy(h1) self.assertEqual(h1, h2) self.assertEqual(h1.get('header1'), h2.get('header1')) - assert isinstance(h2, CaselessDict) + assert isinstance(h2, self.dict_class) + + +class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaseInsensitiveDict + + def test_repr(self): + d = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d), "") + + +class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): + dict_class = CaselessDict + + def test_deprecation_message(self): + with warnings.catch_warnings(record=True) as caught: + self.dict_class({"foo": "bar"}) + + self.assertEqual(len(caught), 1) + self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) + self.assertEqual( + "scrapy.utils.datatypes.CaselessDict is deprecated," + " please use scrapy.utils.datatypes.CaseInsensitiveDict instead", + str(caught[0].message), + ) class SequenceExcludeTest(unittest.TestCase): From bbeed6ae8fd9aed3651b104e4cc3e56495e1b1b9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Aug 2021 14:09:30 -0300 Subject: [PATCH 021/211] CaseInsensitiveDict: preserve original keys (only lookups are key-insensitive) --- scrapy/utils/datatypes.py | 36 ++++++++++++++++++++++------------- tests/test_utils_datatypes.py | 13 +++++++++++-- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index ca6089e0f..1d56811f0 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -79,35 +79,45 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict): +class CaseInsensitiveDict(collections.UserDict,): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. """ + def __init__(self, *args, **kwargs) -> None: + self._keys: dict = {} + super().__init__(*args, **kwargs) + def __getitem__(self, key: AnyStr) -> Any: - return super().__getitem__(self.normkey(key)) + normalized_key = self.normkey(key) + return super().__getitem__(self._keys[normalized_key.lower()]) def __setitem__(self, key: AnyStr, value: Any) -> None: - super().__setitem__(self.normkey(key), self.normvalue(value)) + normalized_key = self.normkey(key) + if normalized_key.lower() in self._keys: + del self[self._keys[normalized_key.lower()]] + super().__setitem__(normalized_key, self.normvalue(value)) + self._keys[normalized_key.lower()] = normalized_key def __delitem__(self, key: AnyStr) -> None: - super().__delitem__(self.normkey(key)) + normalized_key = self.normkey(key) + stored_key = self._keys.pop(normalized_key.lower()) + super().__delitem__(stored_key) def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] - return super().__contains__(self.normkey(key)) - - def normkey(self, key: AnyStr) -> AnyStr: - """Method to normalize dictionary key access""" - return key.lower() - - def normvalue(self, value: Any) -> Any: - """Method to normalize values prior to be set""" - return value + normalized_key = self.normkey(key) + return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" + def normkey(self, key: AnyStr) -> AnyStr: + return key + + def normvalue(self, value: Any) -> Any: + return value + class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index c033cd537..5faaabe81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,4 +1,5 @@ import copy +from typing import Iterator import unittest import warnings from collections.abc import Mapping, MutableMapping @@ -191,8 +192,16 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): dict_class = CaseInsensitiveDict def test_repr(self): - d = self.dict_class({"foo": "bar"}) - self.assertEqual(repr(d), "") + d1 = self.dict_class({"foo": "bar"}) + self.assertEqual(repr(d1), "") + d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + self.assertEqual(repr(d2), "") + + def test_iter(self): + d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"}) + iterkeys = iter(d) + self.assertIsInstance(iterkeys, Iterator) + self.assertEqual(list(iterkeys), ["AsDf", "FoO"]) class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase): From 10ebf6384ed58253c237224d523e602b1f3c2224 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Aug 2021 14:12:55 -0300 Subject: [PATCH 022/211] Remove unnecessary comma --- scrapy/utils/datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 1d56811f0..6eeabe1ee 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -79,7 +79,7 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict,): +class CaseInsensitiveDict(collections.UserDict): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. From 1c031b8a8dd719e6011ee29889bc8181cdbc9a9b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 12 May 2022 13:10:08 -0300 Subject: [PATCH 023/211] Underscore CaseInsensitiveDict normkey/normvalue --- scrapy/utils/datatypes.py | 19 +++++++++---------- tests/test_utils_datatypes.py | 10 +++++++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index f45e1c9b8..807a95504 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -80,9 +80,8 @@ class CaselessDict(dict): class CaseInsensitiveDict(collections.UserDict): - """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. - - It also allows overriding key and value normalization by defining custom `normkey` and `normvalue` methods. + """A dict-like structure that accepts strings or bytes + as keys and allows case-insensitive lookups. """ def __init__(self, *args, **kwargs) -> None: @@ -90,32 +89,32 @@ class CaseInsensitiveDict(collections.UserDict): super().__init__(*args, **kwargs) def __getitem__(self, key: AnyStr) -> Any: - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) return super().__getitem__(self._keys[normalized_key.lower()]) def __setitem__(self, key: AnyStr, value: Any) -> None: - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) if normalized_key.lower() in self._keys: del self[self._keys[normalized_key.lower()]] - super().__setitem__(normalized_key, self.normvalue(value)) + super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key def __delitem__(self, key: AnyStr) -> None: - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) stored_key = self._keys.pop(normalized_key.lower()) super().__delitem__(stored_key) def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] - normalized_key = self.normkey(key) + normalized_key = self._normkey(key) return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" - def normkey(self, key: AnyStr) -> AnyStr: + def _normkey(self, key: AnyStr) -> AnyStr: return key - def normvalue(self, value: Any) -> Any: + def _normvalue(self, value: Any) -> Any: return value diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 5faaabe81..0a724f237 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,8 +1,8 @@ import copy -from typing import Iterator import unittest import warnings from collections.abc import Mapping, MutableMapping +from typing import Iterator from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -144,19 +144,23 @@ class CaseInsensitiveDictMixin: def test_normkey(self): class MyDict(self.dict_class): - def normkey(self, key): + def _normkey(self, key): return key.title() + normkey = _normkey # deprecated CaselessDict class + d = MyDict() d['key-one'] = 2 self.assertEqual(list(d.keys()), ['Key-One']) def test_normvalue(self): class MyDict(self.dict_class): - def normvalue(self, value): + def _normvalue(self, value): if value is not None: return value + 1 + normvalue = _normvalue # deprecated CaselessDict class + d = MyDict({'key': 1}) self.assertEqual(d['key'], 2) self.assertEqual(d.get('key'), 2) From 2c65066ad9e293630da2c594af06ad483abe800d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 27 May 2022 19:56:42 -0300 Subject: [PATCH 024/211] Avoid exceptions on copy --- scrapy/utils/datatypes.py | 7 +++++-- tests/test_utils_datatypes.py | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 807a95504..fd5ac3b08 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -94,8 +94,11 @@ class CaseInsensitiveDict(collections.UserDict): def __setitem__(self, key: AnyStr, value: Any) -> None: normalized_key = self._normkey(key) - if normalized_key.lower() in self._keys: - del self[self._keys[normalized_key.lower()]] + try: + lower_key = self._keys[normalized_key.lower()] + del self[lower_key] + except KeyError: + pass super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 0a724f237..36df9006f 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -187,9 +187,15 @@ class CaseInsensitiveDictMixin: def test_copy(self): h1 = self.dict_class({'header1': 'value'}) h2 = copy.copy(h1) + assert isinstance(h2, self.dict_class) self.assertEqual(h1, h2) self.assertEqual(h1.get('header1'), h2.get('header1')) - assert isinstance(h2, self.dict_class) + self.assertEqual(h1.get('header1'), h2.get('HEADER1')) + h3 = h1.copy() + assert isinstance(h3, self.dict_class) + self.assertEqual(h1, h3) + self.assertEqual(h1.get('header1'), h3.get('header1')) + self.assertEqual(h1.get('header1'), h3.get('HEADER1')) class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase): From a34b929a40c12933f75db4665b71348444a5d603 Mon Sep 17 00:00:00 2001 From: srki24 Date: Fri, 4 Nov 2022 18:00:17 +0100 Subject: [PATCH 025/211] issues/5043 Detaching the stream --- scrapy/exporters.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 76cbe4d4b..243ec4fe1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -247,6 +247,12 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def finish_exporting(self): + # Detaching stream in order to avoid file closing. + # The file will be closed with slot.storage.store + # https://github.com/scrapy/scrapy/issues/5043 + self.stream.detach() + def _build_row(self, values): for s in values: try: From 2f2bcb006d349eeeed10018362c780496be96550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 2 Feb 2023 05:55:59 +0100 Subject: [PATCH 026/211] Test stream detaching in CsvItemExporter --- scrapy/exporters.py | 5 +---- tests/test_exporters.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 243ec4fe1..42105690c 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -248,10 +248,7 @@ class CsvItemExporter(BaseItemExporter): self.csv_writer.writerow(values) def finish_exporting(self): - # Detaching stream in order to avoid file closing. - # The file will be closed with slot.storage.store - # https://github.com/scrapy/scrapy/issues/5043 - self.stream.detach() + self.stream.detach() # Avoid closing the wrapped file. def _build_row(self, values): for s in values: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 69ac928c3..bec8d2267 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -85,6 +85,10 @@ class BaseItemExporterTest(unittest.TestCase): if self.ie.__class__ is not BaseItemExporter: raise self.ie.finish_exporting() + # Delete the item exporter object, so that if it causes the output + # file handle be closed, which should not be the case, follow-up + # interactions with the output file handle will surface the issue. + del self.ie self._check_output() def test_export_item(self): @@ -230,6 +234,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i1) ie.export_item(i2) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. f.seek(0) self.assertEqual(self.item_class(**pickle.load(f)), i1) self.assertEqual(self.item_class(**pickle.load(f)), i2) @@ -241,6 +246,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertEqual(pickle.loads(fp.getvalue()), item) @@ -267,6 +273,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. fp.seek(0) self.assertEqual(marshal.load(fp), item) @@ -299,6 +306,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual(fp.getvalue(), expected) def test_header_export_all(self): @@ -330,6 +338,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual(output.getvalue(), b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') @@ -414,6 +423,7 @@ class XmlItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): @@ -520,6 +530,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) @@ -534,6 +545,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item['time'] = str(item['time']) self.assertEqual(exported, item) @@ -561,6 +573,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) @@ -577,6 +590,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) @@ -588,6 +602,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) @@ -597,6 +612,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item['time'] = str(item['time']) self.assertEqual(exported, [item]) From 426f3ebb7b368084f6e77ccf8a121c85c7913049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 2 Feb 2023 05:58:32 +0100 Subject: [PATCH 027/211] =?UTF-8?q?Fix=20typo:=20causes=20it=20be=20closed?= =?UTF-8?q?=20=E2=86=92=20causes=20it=20to=20be=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 34475b05d..95ff5a93c 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -92,7 +92,7 @@ class BaseItemExporterTest(unittest.TestCase): raise self.ie.finish_exporting() # Delete the item exporter object, so that if it causes the output - # file handle be closed, which should not be the case, follow-up + # file handle to be closed, which should not be the case, follow-up # interactions with the output file handle will surface the issue. del self.ie self._check_output() From 60bf56b715e443951125a10ff91ad1699270a82c Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:15:24 +0100 Subject: [PATCH 028/211] Add boto3 availability util method --- scrapy/utils/boto.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 085ee7d25..7b18b6bcf 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -8,3 +8,12 @@ def is_botocore_available(): return True except ImportError: return False + + +def is_boto3_available(): + try: + import boto3 # noqa: F401 + + return True + except ImportError: + return False From 01ad49515d31e3053949d20e923dc61a87147eab Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:18:28 +0100 Subject: [PATCH 029/211] Use boto3 session and client --- scrapy/extensions/feedexport.py | 37 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cd26b5778..34eb8e4b4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -173,16 +173,37 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - import botocore.session + if is_boto3_available(): + import boto3.session + session = boto3.session.Session() + + self.s3_client = session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + ) + else: + warnings.warn( + "Botocore usage is deprecated for S3FeedStorage, " + "please use boto3 to avoid problems", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + import botocore.session + session = botocore.get_session() session = botocore.session.get_session() - self.s3_client = session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - ) + self.s3_client = session.create_client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + ) + if feed_options and feed_options.get("overwrite", True) is False: logger.warning( "S3 does not support appending to files. To " From c1a8baa1fa0c6210e5c1351ab5a4063b8dacaa7b Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:20:01 +0100 Subject: [PATCH 030/211] Add forgotten import --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 34eb8e4b4..0e05337bc 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -22,6 +22,7 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available +from scrapy.utils.boto import is_boto3_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -195,7 +196,6 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session session = botocore.get_session() - session = botocore.session.get_session() self.s3_client = session.create_client( "s3", aws_access_key_id=self.access_key, From 59ba3c4e4cabd661810e0ac1b9963040d2e42f31 Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 16:29:06 +0100 Subject: [PATCH 031/211] Use boto3's `upload_fileobj` --- scrapy/extensions/feedexport.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0e05337bc..601fde7eb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -174,7 +174,9 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - if is_boto3_available(): + self._using_boto3 = is_boto3_available() + + if self._using_boto3: import boto3.session session = boto3.session.Session() @@ -187,8 +189,8 @@ class S3FeedStorage(BlockingFeedStorage): ) else: warnings.warn( - "Botocore usage is deprecated for S3FeedStorage, " - "please use boto3 to avoid problems", + "`botocore` usage has been deprecated for S3 feed " + "export, please use `boto3` to avoid problems", category=ScrapyDeprecationWarning, stacklevel=2, ) @@ -227,9 +229,14 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs - ) + if self._using_boto3: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + ) + else: + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs + ) file.close() From 29c2477f0a8365d0476fb8c07f391fa109f4615a Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 16:40:05 +0100 Subject: [PATCH 032/211] Document the need to install boto3 --- docs/topics/feed-exports.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..8aa3e3be4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -101,12 +101,12 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-fs` - :ref:`topics-feed-storage-ftp` -- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-s3` (requires boto3_) - :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if the botocore_ +not available. For example, the S3 backend is only available if at least the botocore_ library is installed. @@ -193,7 +193,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `botocore`_ >= 1.4.87 +- Required external libraries: `boto3`_ >= 1.26.70, will fall back to botocore_ if unavailable The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -779,6 +779,7 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ +.. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ From f45a7d3f3c90ad27b2b13bd1ee038ce2fe9c02f7 Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 17:07:55 +0100 Subject: [PATCH 033/211] Remove `stacklevel` from warning --- scrapy/extensions/feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 601fde7eb..e27e6f915 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -192,7 +192,6 @@ class S3FeedStorage(BlockingFeedStorage): "`botocore` usage has been deprecated for S3 feed " "export, please use `boto3` to avoid problems", category=ScrapyDeprecationWarning, - stacklevel=2, ) import botocore.session From eb0cca471d20f00a1cc0980b84fe6266772faaa3 Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Feb 2023 16:53:49 +0100 Subject: [PATCH 034/211] Apply pre-commit changes --- scrapy/extensions/feedexport.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e27e6f915..36107a62b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,8 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.boto import is_botocore_available -from scrapy.utils.boto import is_boto3_available +from scrapy.utils.boto import is_boto3_available, is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -178,6 +177,7 @@ class S3FeedStorage(BlockingFeedStorage): if self._using_boto3: import boto3.session + session = boto3.session.Session() self.s3_client = session.client( @@ -195,6 +195,7 @@ class S3FeedStorage(BlockingFeedStorage): ) import botocore.session + session = botocore.get_session() self.s3_client = session.create_client( From 45b9dbae40de00fc7d4498e871538c074c64be8f Mon Sep 17 00:00:00 2001 From: jazzthief Date: Wed, 22 Feb 2023 13:28:34 +0100 Subject: [PATCH 035/211] Fix a typo --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 36107a62b..c4ec410e3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -196,7 +196,7 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session - session = botocore.get_session() + session = botocore.session.get_session() self.s3_client = session.create_client( "s3", From d311779887d5c8a34c8062eadd3155cb0ef4dc72 Mon Sep 17 00:00:00 2001 From: kenshi kikuchi Date: Wed, 8 Mar 2023 16:24:09 +0900 Subject: [PATCH 036/211] Fix FeedExporter + Fix FeedExporter not to export empty file + Change default value of FEED_STORE_EMPTY --- scrapy/extensions/feedexport.py | 90 +++++++++++++++++------------ scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 31 +++++----- 3 files changed, 70 insertions(+), 53 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cd26b5778..8a60bc528 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -274,8 +274,6 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: def __init__( self, - file, - exporter, storage, uri, format, @@ -283,9 +281,14 @@ class _FeedSlot: batch_id, uri_template, filter, + feed_options, + spider, + exporters, + settings, + crawler, ): - self.file = file - self.exporter = exporter + self.file = None + self.exporter = None self.storage = storage # feed params self.batch_id = batch_id @@ -294,15 +297,44 @@ class _FeedSlot: self.uri_template = uri_template self.uri = uri self.filter = filter + # exporter params + self.feed_options = feed_options + self.spider = spider + self.exporters = exporters + self.settings = settings + self.crawler = crawler # flags self.itemcount = 0 self._exporting = False + self._fileloaded = False def start_exporting(self): + if not self._fileloaded: + self.file = self.storage.open(self.spider) + if "postprocessing" in self.feed_options: + self.file = PostProcessingManager( + self.feed_options["postprocessing"], self.file, self.feed_options + ) + self.exporter = self._get_exporter( + file=self.file, + format=self.feed_options["format"], + fields_to_export=self.feed_options["fields"], + encoding=self.feed_options["encoding"], + indent=self.feed_options["indent"], + **self.feed_options["item_export_kwargs"], + ) + self._fileloaded = True + if not self._exporting: self.exporter.start_exporting() self._exporting = True + def _get_instance(self, objcls, *args, **kwargs): + return create_instance(objcls, self.settings, self.crawler, *args, **kwargs) + + def _get_exporter(self, file, format, *args, **kwargs): + return self._get_instance(self.exporters[format], file, *args, **kwargs) + def finish_exporting(self): if self._exporting: self.exporter.finish_exporting() @@ -379,15 +411,22 @@ class FeedExporter: deferred_list = [] for slot in self.slots: d = self._close_slot(slot, spider) - deferred_list.append(d) + if d: + deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None def _close_slot(self, slot, spider): - slot.finish_exporting() - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) + if slot.itemcount: + # Nomal case + slot.finish_exporting() + elif slot.store_empty and slot.batch_id == 1: + # Need Store Empty + slot.start_exporting() + slot.finish_exporting() + else: + # In this case, the file is not stored, so no processing is required. + return None + logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" d = defer.maybeDeferred(slot.storage.store, slot.file) @@ -423,23 +462,7 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - file = storage.open(spider) - if "postprocessing" in feed_options: - file = PostProcessingManager( - feed_options["postprocessing"], file, feed_options - ) - - exporter = self._get_exporter( - file=file, - format=feed_options["format"], - fields_to_export=feed_options["fields"], - encoding=feed_options["encoding"], - indent=feed_options["indent"], - **feed_options["item_export_kwargs"], - ) slot = _FeedSlot( - file=file, - exporter=exporter, storage=storage, uri=uri, format=feed_options["format"], @@ -447,9 +470,12 @@ class FeedExporter: batch_id=batch_id, uri_template=uri_template, filter=self.filters[uri_template], + feed_options=feed_options, + spider=spider, + exporters=self.exporters, + settings=self.settings, + crawler=getattr(self, "crawler", None), ) - if slot.store_empty: - slot.start_exporting() return slot def item_scraped(self, item, spider): @@ -533,14 +559,6 @@ class FeedExporter: else: logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme}) - def _get_instance(self, objcls, *args, **kwargs): - return create_instance( - objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs - ) - - def _get_exporter(self, file, format, *args, **kwargs): - return self._get_instance(self.exporters[format], file, *args, **kwargs) - def _get_storage(self, uri, feed_options): """Fork of create_instance specific to feed storage classes diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 260ec1701..ea63d35c5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -141,7 +141,7 @@ EXTENSIONS_BASE = { FEED_TEMPDIR = None FEEDS = {} FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_STORE_EMPTY = False +FEED_STORE_EMPTY = True FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None FEED_STORAGES = {} diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eafe1b334..acdc39870 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -725,10 +725,9 @@ class FeedExportTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not Path(file_path).exists(): - continue - - content[feed_options["format"]] = Path(file_path).read_bytes() + content[feed_options["format"]] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) finally: for file_path in FEEDS.keys(): @@ -945,9 +944,10 @@ class FeedExportTest(FeedExportTestBase): "FEEDS": { self._random_temp_filename(): {"format": fmt}, }, + "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) - self.assertEqual(b"", data[fmt]) + self.assertEqual(None, data[fmt]) @defer.inlineCallbacks def test_start_finish_exporting_items(self): @@ -1057,7 +1057,6 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {"format": "csv"}, }, "FEED_STORAGES": {"file": LogOnStoreFileStorage}, - "FEED_STORE_EMPTY": False, } with LogCapture() as log: @@ -1680,10 +1679,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not Path(file_path).exists(): - continue - - content[str(file_path)] = Path(file_path).read_bytes() + content[str(file_path)] = ( + Path(file_path).read_bytes() if Path(file_path).exists() else None + ) finally: for file_path in FEEDS.keys(): @@ -2184,6 +2182,9 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = Path(path).parent + if not dir_name.exists(): + content[feed["format"]] = [] + continue for file in sorted(dir_name.iterdir()): content[feed["format"]].append(file.read_bytes()) finally: @@ -2367,10 +2368,11 @@ class BatchDeliveriesTest(FeedExportTestBase): / self._file_mark: {"format": fmt}, }, "FEED_EXPORT_BATCH_ITEM_COUNT": 1, + "FEED_STORE_EMPTY": False, } data = yield self.exported_no_data(settings) data = dict(data) - self.assertEqual(b"", data[fmt][0]) + self.assertEqual(0, len(data[fmt])) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -2484,9 +2486,6 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) - @pytest.mark.skipif( - sys.platform == "win32", reason="Odd behaviour on file creation/output" - ) @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2508,7 +2507,7 @@ class BatchDeliveriesTest(FeedExportTestBase): "FEED_EXPORT_BATCH_ITEM_COUNT": 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items), len([_ for _ in data["json"] if _])) + self.assertEqual(len(items), len(data["json"])) @defer.inlineCallbacks def test_stats_batch_file_success(self): @@ -2595,7 +2594,7 @@ class BatchDeliveriesTest(FeedExportTestBase): crawler = get_crawler(TestSpider, settings) yield crawler.crawl() - self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) + self.assertEqual(len(CustomS3FeedStorage.stubs), len(items)) for stub in CustomS3FeedStorage.stubs[:-1]: stub.assert_no_pending_responses() From 05893e17966be9037efae8be9ded494579b28b60 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 8 Mar 2023 02:59:47 -0600 Subject: [PATCH 037/211] docs: Spider.update_settings --- docs/topics/spiders.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 788bd7678..e5a539fe7 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -145,6 +145,42 @@ scrapy.Spider :param kwargs: keyword arguments passed to the :meth:`__init__` method :type kwargs: dict + .. method:: update_settings(cls, settings) + + The ``update_settings()`` method is used to modify the spider's settings + and can be called during initialization of a spider instance. + + It takes a ``Settings`` object as a parameter and adds or updates the spider's + configuration values. This method is a class method, meaning that it is + called on the Spider class and allows all instances of the Spider to share + the same configuration. + + To create class hierarchies for spiders, it is recommended to use the ``custom_settings`` + attribute instead of ``update_settings()``, as it allows for default settings to be + defined and automatically inherited by subclasses. + + For example, suppose a MySpider needs update FEEDS: + + .. code-block:: python + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + custom_feed = { + "/home/user/documents/items.json": { + "format": "json", + "indent": 4, + } + } + + @classmethod + def update_settings(cls, settings): + settings.setdefault("FEEDS", {}).update(cls.custom_feed) + super().update_settings(settings) + + + .. method:: start_requests() This method must return an iterable with the first Requests to crawl for From 1d862d083104405ba432ddf0d741a304381b5608 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 8 Mar 2023 03:26:38 -0600 Subject: [PATCH 038/211] fix: remove line breaks --- docs/topics/spiders.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e5a539fe7..d501466de 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -179,8 +179,6 @@ scrapy.Spider settings.setdefault("FEEDS", {}).update(cls.custom_feed) super().update_settings(settings) - - .. method:: start_requests() This method must return an iterable with the first Requests to crawl for From 96d51c3afa979587412bf3be71351056d2ef885a Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 8 Mar 2023 04:21:21 -0600 Subject: [PATCH 039/211] docs: update --- docs/topics/spiders.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d501466de..22bbf2ce4 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -145,23 +145,24 @@ scrapy.Spider :param kwargs: keyword arguments passed to the :meth:`__init__` method :type kwargs: dict - .. method:: update_settings(cls, settings) + .. classmethod:: update_settings(settings) The ``update_settings()`` method is used to modify the spider's settings and can be called during initialization of a spider instance. - It takes a ``Settings`` object as a parameter and adds or updates the spider's - configuration values. This method is a class method, meaning that it is - called on the Spider class and allows all instances of the Spider to share - the same configuration. + It takes a :class:`~scrapy.settings.Settings` object as a parameter and + adds or updates the spider's configuration values. This method is a class method, + meaning that it is called on the :class:`~scrapy.Spider` class and allows all instances + of the spider to share the same configuration. - To create class hierarchies for spiders, it is recommended to use the ``custom_settings`` + To create class hierarchies for spiders, it is recommended to use the :attr:`custom_settings` attribute instead of ``update_settings()``, as it allows for default settings to be defined and automatically inherited by subclasses. - For example, suppose a MySpider needs update FEEDS: + For example, suppose a spider needs update :setting:`FEEDS`: .. code-block:: python + import scrapy From 39dbfa1d8276e1c8abb6aedc51f644ee27264f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Mar 2023 15:31:39 +0100 Subject: [PATCH 040/211] Minimize test reliance on S3; do not install botocore on the default test environments --- tests/test_feedexport.py | 23 ++++++----------------- tests/test_pipeline_files.py | 12 ++++++++---- tox.ini | 4 +--- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eafe1b334..19ca311c3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -235,8 +235,10 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - def test_parse_credentials(self): + def setUp(self): skip_if_no_boto() + + def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", "AWS_SECRET_ACCESS_KEY": "settings_secret", @@ -272,8 +274,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - skip_if_no_boto() - settings = { "AWS_ACCESS_KEY_ID": "access_key", "AWS_SECRET_ACCESS_KEY": "secret_key", @@ -392,7 +392,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): - skip_if_no_boto() storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -408,7 +407,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): - skip_if_no_boto() storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -888,15 +886,10 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_stats_multiple_file(self): settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", "FEEDS": { printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", }, - "s3://bucket/key/foo.csv": { - "format": "csv", - }, "stdout:": { "format": "xml", }, @@ -908,18 +901,12 @@ class FeedExportTest(FeedExportTestBase): self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) - self.assertIn( - "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats() - ) self.assertIn( "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1 - ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 ) @@ -2535,7 +2522,6 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() - bucket = "mybucket" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2707,6 +2693,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): maxDiff = None + def setUp(self): + skip_if_no_boto() + def test_init(self): settings_dict = { "FEED_URI": "file:///tmp/foobar", diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 9701e5d4e..e0bcfcfea 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -225,12 +225,16 @@ class FilesPipelineTestCase(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: + def setUp(self): + self.tempdir = mkdtemp() + + def tearDown(self): + rmtree(self.tempdir) + def test_item_fields_default(self): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) - pipeline = FilesPipeline.from_settings( - Settings({"FILES_STORE": "s3://example/files/"}) - ) + pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {"url": url})] @@ -245,7 +249,7 @@ class FilesPipelineTestCaseFieldsMixin: pipeline = FilesPipeline.from_settings( Settings( { - "FILES_STORE": "s3://example/files/", + "FILES_STORE": self.tempdir, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", } diff --git a/tox.ini b/tox.ini index 5a9d9cf29..5c2f583d9 100644 --- a/tox.ini +++ b/tox.ini @@ -18,8 +18,6 @@ deps = mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' - # Extras - botocore>=1.4.87 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -121,7 +119,7 @@ setenv = basepython = python3 deps = {[testenv]deps} - boto + botocore>=1.4.87 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. From 590955fac8de5d1f951b4ecb724eb2ea4f212653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Mar 2023 16:03:44 +0100 Subject: [PATCH 041/211] Provide separate test environments for botocore and boto3 extras; add extra-deps-pinned and remote extras from pinned --- .github/workflows/tests-ubuntu.yml | 9 ++++++ tox.ini | 44 ++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 8fcf90a18..96b26a1f8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -37,10 +37,19 @@ jobs: - python-version: pypy3.7 env: TOXENV: pypy3-pinned + - python-version: 3.7.13 + env: + TOXENV: extra-deps-pinned + - python-version: 3.7.13 + env: + TOXENV: botocore-pinned - python-version: "3.11" env: TOXENV: extra-deps + - python-version: "3.11" + env: + TOXENV: botocore steps: - uses: actions/checkout@v3 diff --git a/tox.ini b/tox.ini index 5c2f583d9..f94d7f751 100644 --- a/tox.ini +++ b/tox.ini @@ -88,11 +88,6 @@ deps = # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment - - # Extras - botocore==1.4.87 - google-cloud-storage==1.29.0 - Pillow==7.1.0 setenv = _SCRAPY_PINNED=true install_command = @@ -119,14 +114,26 @@ setenv = basepython = python3 deps = {[testenv]deps} - botocore>=1.4.87 + boto3 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 robotexclusionrulesparser - Pillow>=4.0.0 - Twisted[http2]>=17.9.0 + Pillow + Twisted[http2] + +[testenv:extra-deps-pinned] +basepython = python3.7 +deps = + {[pinned]deps} + boto3==1.0.0 + google-cloud-storage==1.29.0 + Pillow==7.1.0 + robotexclusionrulesparser==1.6.2 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:asyncio] commands = @@ -185,3 +192,24 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + + +# Run S3 tests with botocore installed but without boto3. + +[testenv:botocore] +deps = + {[testenv]deps} + botocore>=1.4.87 +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[testenv:botocore-pinned] +basepython = python3.7 +deps = + {[pinned]deps} + botocore==1.4.87 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} From 7e7b41c6b32a639395cf3183e98fa8a8a78afd98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Mar 2023 16:38:13 +0100 Subject: [PATCH 042/211] Fix test expectations for boto3 --- scrapy/extensions/feedexport.py | 3 +- tests/test_feedexport.py | 72 +++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c4ec410e3..4f0a946de 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -228,12 +228,13 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - kwargs = {"ACL": self.acl} if self.acl else {} if self._using_boto3: + kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs ) else: + kwargs = {"ACL": self.acl} if self.acl else {} self.s3_client.put_object( Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 19ca311c3..2e350df65 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -44,6 +44,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, _FeedSlot, + is_boto3_available, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -285,30 +286,39 @@ class S3FeedStorageTest(unittest.TestCase): verifyObject(IFeedStorage, storage) file = mock.MagicMock() - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) + if is_boto3_available(): + storage.s3_client = mock.MagicMock() yield storage.store(file) - - stub.assert_no_pending_responses() self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], + storage.s3_client.upload_fileobj.call_args, + mock.call(Bucket=bucket, Key=key, Fileobj=file), ) + else: + from botocore.stub import Stubber + + with Stubber(storage.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "Body": file, + "Bucket": bucket, + "Key": key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + mock.call.close(), + ], + ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -391,7 +401,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.endpoint_url, "https://example.com") @defer.inlineCallbacks - def test_store_botocore_without_acl(self): + def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -403,10 +413,18 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1]) + if is_boto3_available(): + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + else: + acl = storage.s3_client.put_object.call_args[1].get("ACL") + self.assertIsNone(acl) @defer.inlineCallbacks - def test_store_botocore_with_acl(self): + def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -416,9 +434,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertEqual( - storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl" - ) + if is_boto3_available(): + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + else: + acl = storage.s3_client.put_object.call_args[1]["ACL"] + self.assertEqual(acl, "custom-acl") def test_overwrite_default(self): with LogCapture() as log: From c8ed793257d952a425ea1e55def4e1c3b3ca8b68 Mon Sep 17 00:00:00 2001 From: kenshi kikuchi Date: Thu, 16 Mar 2023 17:16:14 +0900 Subject: [PATCH 043/211] Fix test_export_no_items_multiple_feeds --- tests/test_feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index acdc39870..8ab546efd 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1057,13 +1057,13 @@ class FeedExportTest(FeedExportTestBase): self._random_temp_filename(): {"format": "csv"}, }, "FEED_STORAGES": {"file": LogOnStoreFileStorage}, + "FEED_STORE_EMPTY": False, } with LogCapture() as log: yield self.exported_no_data(settings) - print(log) - self.assertEqual(str(log).count("Storage.store is called"), 3) + self.assertEqual(str(log).count("Storage.store is called"), 0) @defer.inlineCallbacks def test_export_multiple_item_classes(self): From 50801c7207e6f964c312b19c9fe0bcc2c6514064 Mon Sep 17 00:00:00 2001 From: kenshi kikuchi Date: Thu, 16 Mar 2023 17:17:20 +0900 Subject: [PATCH 044/211] Fix Docs --- docs/topics/feed-exports.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..93d68d49d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -552,9 +552,10 @@ to ``.json`` or ``.xml``. FEED_STORE_EMPTY ---------------- -Default: ``False`` +Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). +If False and there is no items, no new files are created and existing files are not modified. .. setting:: FEED_STORAGES From 6ab49e954f25d491df7986065d270bb0068c7c89 Mon Sep 17 00:00:00 2001 From: namelessGonbai <43787036+namelessGonbai@users.noreply.github.com> Date: Thu, 16 Mar 2023 18:03:06 +0900 Subject: [PATCH 045/211] Update docs/topics/feed-exports.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/feed-exports.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 93d68d49d..2a80daa46 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -555,7 +555,9 @@ FEED_STORE_EMPTY Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). -If False and there is no items, no new files are created and existing files are not modified. +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option +` is enabled. .. setting:: FEED_STORAGES From a17d996da2dad6d250dd93da34b5b63f3d63239d Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Mar 2023 17:20:22 +0100 Subject: [PATCH 046/211] Change `boto3` version to 1.20.0 for `extra-deps-pinned` env --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f94d7f751..80fc28735 100644 --- a/tox.ini +++ b/tox.ini @@ -127,7 +127,7 @@ deps = basepython = python3.7 deps = {[pinned]deps} - boto3==1.0.0 + boto3==1.20.0 google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 From 4ebc08ef1042cafd16c6a7eb20e3a4dcf43e3c97 Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Mar 2023 17:24:11 +0100 Subject: [PATCH 047/211] Switch from `is_boto3_available()` to `IS_BOTO3_AVAILABLE` var --- scrapy/extensions/feedexport.py | 14 ++++++++++---- scrapy/utils/boto.py | 9 --------- tests/test_feedexport.py | 8 ++++---- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4f0a946de..83849ca13 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,7 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.boto import is_boto3_available, is_botocore_available +from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +try: + import boto3 # noqa: F401 + + IS_BOTO3_AVAILABLE = True +except ImportError: + IS_BOTO3_AVAILABLE = False + def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): argument_names = get_func_args(builder) @@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - self._using_boto3 = is_boto3_available() - if self._using_boto3: + if IS_BOTO3_AVAILABLE: import boto3.session session = boto3.session.Session() @@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - if self._using_boto3: + if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 7b18b6bcf..085ee7d25 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -8,12 +8,3 @@ def is_botocore_available(): return True except ImportError: return False - - -def is_boto3_available(): - try: - import boto3 # noqa: F401 - - return True - except ImportError: - return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2e350df65..7df3e6dd3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -35,6 +35,7 @@ import scrapy from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( + IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FileFeedStorage, @@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, _FeedSlot, - is_boto3_available, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase): file = mock.MagicMock() - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: storage.s3_client = mock.MagicMock() yield storage.store(file) self.assertEqual( @@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: acl = ( storage.s3_client.upload_fileobj.call_args[1] .get("ExtraArgs", {}) @@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] else: acl = storage.s3_client.put_object.call_args[1]["ACL"] From cb67bc17b72a1ae619c89cc26d6f526dd2a26338 Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Mar 2023 17:25:05 +0100 Subject: [PATCH 048/211] Remove `botocore` from docs --- docs/topics/feed-exports.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 8aa3e3be4..5eea6aaf9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -106,7 +106,7 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if at least the botocore_ +not available. For example, the S3 backend is only available if the boto3_ library is installed. @@ -193,7 +193,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `boto3`_ >= 1.26.70, will fall back to botocore_ if unavailable +- Required external libraries: `boto3`_ >= 1.20.0 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -780,6 +780,5 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto3: https://github.com/boto/boto3 -.. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ From 495372648c533fc66196cafd4991dbfd403c7df8 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Thu, 16 Mar 2023 23:14:57 -0600 Subject: [PATCH 049/211] fix: docs update_settings() --- docs/topics/spiders.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 22bbf2ce4..796db2dc8 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -148,16 +148,16 @@ scrapy.Spider .. classmethod:: update_settings(settings) The ``update_settings()`` method is used to modify the spider's settings - and can be called during initialization of a spider instance. + and is called during initialization of a spider instance. It takes a :class:`~scrapy.settings.Settings` object as a parameter and - adds or updates the spider's configuration values. This method is a class method, + can add or updates the spider's configuration values. This method is a class method, meaning that it is called on the :class:`~scrapy.Spider` class and allows all instances of the spider to share the same configuration. - To create class hierarchies for spiders, it is recommended to use the :attr:`custom_settings` - attribute instead of ``update_settings()``, as it allows for default settings to be - defined and automatically inherited by subclasses. + One of the main advantages of ``update_settings()`` is that it allows + you to dynamically add, remove or change settings based on spider arguments + or other external factors. For example, suppose a spider needs update :setting:`FEEDS`: From a1fc37cbff9645116bbf6fa63eaf9df59aac7c0a Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Fri, 17 Mar 2023 12:13:05 -0600 Subject: [PATCH 050/211] Update docs/topics/spiders.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 796db2dc8..dc417614d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -159,7 +159,7 @@ scrapy.Spider you to dynamically add, remove or change settings based on spider arguments or other external factors. - For example, suppose a spider needs update :setting:`FEEDS`: + For example, suppose a spider needs to modify :setting:`FEEDS`: .. code-block:: python From 24f28c415caadcf20f4a564ebd6bd52bea9b61ad Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 17 Mar 2023 12:16:08 -0600 Subject: [PATCH 051/211] fix: docs update_settings() --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index dc417614d..64c2a3ae0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -156,7 +156,7 @@ scrapy.Spider of the spider to share the same configuration. One of the main advantages of ``update_settings()`` is that it allows - you to dynamically add, remove or change settings based on spider arguments + you to dynamically add, remove or change settings based on other settings or other external factors. For example, suppose a spider needs to modify :setting:`FEEDS`: From 44cdaa442bf25e360c8acf757623116a5eda5bba Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Fri, 17 Mar 2023 13:19:03 -0600 Subject: [PATCH 052/211] Update docs/topics/spiders.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 64c2a3ae0..0f6c2b1ba 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -151,7 +151,7 @@ scrapy.Spider and is called during initialization of a spider instance. It takes a :class:`~scrapy.settings.Settings` object as a parameter and - can add or updates the spider's configuration values. This method is a class method, + can add or update the spider's configuration values. This method is a class method, meaning that it is called on the :class:`~scrapy.Spider` class and allows all instances of the spider to share the same configuration. From 67bfb304cdeb19a0b72b0a09542582f718cf07d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 21 Apr 2023 18:52:58 +0400 Subject: [PATCH 053/211] Release notes for the current master. --- docs/news.rst | 105 ++++++++++++++++++++++++++++++++++++++- docs/topics/settings.rst | 2 +- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 9b9eeac71..6cf366449 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,108 @@ Release notes ============= +.. _release-2.9.0: + +Scrapy 2.9.0 (YYYY-MM-DD) +------------------------- + +Highlights: + +- Per-domain request settings. +- Compatibility with new cryptography_ and new parsel_. +- TBD + +New features +~~~~~~~~~~~~ + +- Settings correponding to :setting:`DOWNLOAD_DELAY`, + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis + via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + +- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a + curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) + +- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be + :class:`pathlib.Path` instances. (:issue:`5801`) + +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The ``scrapy parse`` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The ``scrapy genspider`` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + +Bug fixes +~~~~~~~~~ + +- Fixed an error when using feed postprocessing with S3 storage. + (:issue:`5500`, :issue:`5581`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method. + (:issue:`5811`, :issue:`5821`) + +- Fixed an error when using cryptography_ 40.0.0+ and + :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled. + (:issue:`5857`, :issue:`5858`) + +- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline` + for files on Google Cloud Storage are no longer Base64-encoded. + (:issue:`5874`, :issue:`5891`) + +- Fixed an error breaking user handling of send failures in + :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + +Documentation +~~~~~~~~~~~~~ + +- Expanded contributing docs. (:issue:`5109`, :issue:`5851`) + +- Added blacken-docs_ to pre-commit and reformatted the docs with it. + (:issue:`5813`, :issue:`5816`) + +- Fixed a JS issue. (:issue:`5875`, :issue:`5877`) + +- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`) + +- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`, + :issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) + +- Tests for most of the examples in the docs are now run as a part of CI, + found problems were fixed. (:issue:`5816`, :issue:`5826`) + +- Removed usage of deprecated Python classes. (:issue:`5849`) + +- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`) + +- Fixed a random failure of the ``test_feedexport.test_batch_path_differ`` + test. (:issue:`5855`, :issue:`5898`) + +- Updated docstrings to match output produced by parsel_ 1.8.1 so that they + don't cause test failures. (:issue:`5902`) + +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`) + +.. _blacken-docs: https://github.com/adamchainz/blacken-docs + .. _release-2.8.0: Scrapy 2.8.0 (2023-02-02) @@ -4207,8 +4309,6 @@ Relocations + Note: telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595) -.. _parsel: https://github.com/scrapy/parsel - Bugfixes ~~~~~~~~ @@ -5638,6 +5738,7 @@ First release of Scrapy. .. _LevelDB: https://github.com/google/leveldb .. _lxml: https://lxml.de/ .. _marshal: https://docs.python.org/2/library/marshal.html +.. _parsel: https://github.com/scrapy/parsel .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4412b5c1c..3e06d84f9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,7 +783,7 @@ DOWNLOAD_SLOTS Default: ``{}`` -Allows to define concurrency/delay parameters on per slot(domain) basis: +Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python From d1d6465ef4ea8987efb08e2f9abfd65b36719e04 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 17:19:01 +0400 Subject: [PATCH 054/211] Address feedback. --- docs/news.rst | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 6cf366449..5f189760d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,7 +10,7 @@ Scrapy 2.9.0 (YYYY-MM-DD) Highlights: -- Per-domain request settings. +- Per-domain download settings. - Compatibility with new cryptography_ and new parsel_. - TBD @@ -19,7 +19,7 @@ New features - Settings correponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) - Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a @@ -28,26 +28,6 @@ New features - Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be :class:`pathlib.Path` instances. (:issue:`5801`) -- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed - string values for the curl ``--data-raw`` argument, which are produced by - browsers for data that includes certain symbols. (:issue:`5899`, - :issue:`5901`) - -- The ``scrapy parse`` command now also works with async generator callbacks. - (:issue:`5819`, :issue:`5824`) - -- The ``scrapy genspider`` command now properly works with HTTPS URLs. - (:issue:`3553`, :issue:`5808`) - -- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) - -- :class:`LinkExtractor ` - now skips certain malformed URLs instead of raising an exception. - (:issue:`5881`) - -- :func:`scrapy.utils.python.get_func_args` now supports more types of - callables. (:issue:`5872`, :issue:`5885`) - Bug fixes ~~~~~~~~~ @@ -65,6 +45,26 @@ Bug fixes for files on Google Cloud Storage are no longer Base64-encoded. (:issue:`5874`, :issue:`5891`) +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The :command:`parse` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The :command:`genspider` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + - Fixed an error breaking user handling of send failures in :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) From 636559f1cc652e839ef42522e7168e3ff9d77921 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 17:55:07 +0400 Subject: [PATCH 055/211] Add newer changes. --- docs/news.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 5f189760d..cbbb376e5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,8 @@ Highlights: - Per-domain download settings. - Compatibility with new cryptography_ and new parsel_. -- TBD +- JMESPath selectors from the new parsel_. +- Bug fixes. New features ~~~~~~~~~~~~ @@ -22,6 +23,12 @@ New features :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) +- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors + available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) + +- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` + signals. (:issue:`5876`) + - Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) @@ -31,6 +38,8 @@ New features Bug fixes ~~~~~~~~~ +- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`) + - Fixed an error when using feed postprocessing with S3 storage. (:issue:`5500`, :issue:`5581`) @@ -65,6 +74,9 @@ Bug fixes - :func:`scrapy.utils.python.get_func_args` now supports more types of callables. (:issue:`5872`, :issue:`5885`) +- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers. + (:issue:`5914`, :issue:`5917`) + - Fixed an error breaking user handling of send failures in :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) @@ -89,7 +101,7 @@ Quality assurance - Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) - Tests for most of the examples in the docs are now run as a part of CI, - found problems were fixed. (:issue:`5816`, :issue:`5826`) + found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`) - Removed usage of deprecated Python classes. (:issue:`5849`) @@ -99,9 +111,10 @@ Quality assurance test. (:issue:`5855`, :issue:`5898`) - Updated docstrings to match output produced by parsel_ 1.8.1 so that they - don't cause test failures. (:issue:`5902`) + don't cause test failures. (:issue:`5902`, :issue:`5919`) -- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`) +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`, + :issue:`5908`) .. _blacken-docs: https://github.com/adamchainz/blacken-docs From 4596a58a1333b8174b09336ab74c32c2e14c40f4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 16 Apr 2023 01:12:21 +0400 Subject: [PATCH 056/211] Typing for smaller scrapy/utils/ modules. --- scrapy/utils/boto.py | 2 +- scrapy/utils/decorators.py | 16 +++++++++------- scrapy/utils/serialize.py | 3 ++- scrapy/utils/versions.py | 3 ++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 085ee7d25..53cfeddd0 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,7 @@ """Boto/botocore helpers""" -def is_botocore_available(): +def is_botocore_available() -> bool: try: import botocore # noqa: F401 diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 4e684645b..04186559f 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,19 +1,21 @@ import warnings from functools import wraps +from typing import Any, Callable from twisted.internet import defer, threads +from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning -def deprecated(use_instead=None): +def deprecated(use_instead: Any = None) -> Callable: """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def deco(func): + def deco(func: Callable) -> Callable: @wraps(func) - def wrapped(*args, **kwargs): + def wrapped(*args: Any, **kwargs: Any) -> Any: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -28,23 +30,23 @@ def deprecated(use_instead=None): return deco -def defers(func): +def defers(func: Callable) -> Callable[..., Deferred]: """Decorator to make sure a function always returns a deferred""" @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return defer.maybeDeferred(func, *a, **kw) return wrapped -def inthread(func): +def inthread(func: Callable) -> Callable[..., Deferred]: """Decorator to call a function in a thread and return a deferred with the result """ @wraps(func) - def wrapped(*a, **kw): + def wrapped(*a: Any, **kw: Any) -> Deferred: return threads.deferToThread(func, *a, **kw) return wrapped diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 414658944..3b4f67f00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -1,6 +1,7 @@ import datetime import decimal import json +from typing import Any from itemadapter import ItemAdapter, is_item from twisted.internet import defer @@ -12,7 +13,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S" - def default(self, o): + def default(self, o: Any) -> Any: if isinstance(o, set): return list(o) if isinstance(o, datetime.datetime): diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index b0737d3d5..9b637bdb0 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -1,5 +1,6 @@ import platform import sys +from typing import List, Tuple import cryptography import cssselect @@ -12,7 +13,7 @@ import scrapy from scrapy.utils.ssl import get_openssl_version -def scrapy_components_versions(): +def scrapy_components_versions() -> List[Tuple[str, str]]: lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) From e0dbc83bd269e256e8247525e4d5a1d07658b635 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 16 Apr 2023 01:30:04 +0400 Subject: [PATCH 057/211] More typing for scrapy/utils/request.py. --- scrapy/utils/request.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 6d8be991d..6c7f3b345 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -6,7 +6,18 @@ scrapy.http.Request objects import hashlib import json import warnings -from typing import Dict, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Iterable, + List, + Optional, + Tuple, + Type, + Union, +) from urllib.parse import urlunparse from weakref import WeakKeyDictionary @@ -19,11 +30,16 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode +if TYPE_CHECKING: + from scrapy.crawler import Crawler + _deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]" _deprecated_fingerprint_cache = WeakKeyDictionary() -def _serialize_headers(headers, request): +def _serialize_headers( + headers: Iterable[bytes], request: Request +) -> Generator[bytes, Any, None]: for header in headers: if header in request.headers: yield header @@ -139,7 +155,7 @@ def request_fingerprint( return cache[cache_key] -def _request_fingerprint_as_bytes(*args, **kwargs): +def _request_fingerprint_as_bytes(*args: Any, **kwargs: Any) -> bytes: with warnings.catch_warnings(): warnings.simplefilter("ignore") return bytes.fromhex(request_fingerprint(*args, **kwargs)) @@ -231,7 +247,7 @@ class RequestFingerprinter: def from_crawler(cls, crawler): return cls(crawler) - def __init__(self, crawler=None): + def __init__(self, crawler: Optional["Crawler"] = None): if crawler: implementation = crawler.settings.get( "REQUEST_FINGERPRINTER_IMPLEMENTATION" @@ -265,7 +281,7 @@ class RequestFingerprinter: f"and '2.7'." ) - def fingerprint(self, request: Request): + def fingerprint(self, request: Request) -> bytes: return self._fingerprint(request) @@ -311,7 +327,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: If a spider is given, it will try to resolve the callbacks looking at the spider for methods with the same name. """ - request_cls = load_object(d["_class"]) if "_class" in d else Request + request_cls: Type[Request] = load_object(d["_class"]) if "_class" in d else Request kwargs = {key: value for key, value in d.items() if key in request_cls.attributes} if d.get("callback") and spider: kwargs["callback"] = _get_method(spider, d["callback"]) @@ -320,7 +336,7 @@ def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request: return request_cls(**kwargs) -def _get_method(obj, name): +def _get_method(obj: Any, name: Any) -> Any: """Helper function for request_from_dict""" name = str(name) try: From f64a7dedca6d4bf33f86c633a0381a8017eb79f7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 23 Apr 2023 22:56:27 +0400 Subject: [PATCH 058/211] Add typing to scrapy/utils/url.py. --- scrapy/utils/url.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 833aa3e20..22b4197f9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,6 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import re +from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this @@ -15,8 +16,14 @@ from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode +if TYPE_CHECKING: + from scrapy import Spider -def url_is_from_any_domain(url, domains): + +UrlT = Union[str, bytes, ParseResult] + + +def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc.lower() if not host: @@ -25,29 +32,29 @@ def url_is_from_any_domain(url, domains): return any((host == d) or (host.endswith(f".{d}")) for d in domains) -def url_is_from_spider(url, spider): +def url_is_from_spider(url: UrlT, spider: Type["Spider"]) -> bool: """Return True if the url belongs to the given spider""" return url_is_from_any_domain( url, [spider.name] + list(getattr(spider, "allowed_domains", [])) ) -def url_has_any_extension(url, extensions): +def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: """Return True if the url ends with one of the extensions provided""" lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) -def parse_url(url, encoding=None): +def parse_url(url: UrlT, encoding: Optional[str] = None) -> ParseResult: """Return urlparsed url from the given argument (which could be an already parsed url) """ if isinstance(url, ParseResult): return url - return urlparse(to_unicode(url, encoding)) + return cast(ParseResult, urlparse(to_unicode(url, encoding))) -def escape_ajax(url): +def escape_ajax(url: str) -> str: """ Return the crawlable url according to: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started @@ -76,7 +83,7 @@ def escape_ajax(url): return add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:]) -def add_http_if_no_scheme(url): +def add_http_if_no_scheme(url: str) -> str: """Add http as the default scheme if it is missing from the url.""" match = re.match(r"^\w+://", url, flags=re.I) if not match: @@ -87,7 +94,7 @@ def add_http_if_no_scheme(url): return url -def _is_posix_path(string): +def _is_posix_path(string: str) -> bool: return bool( re.match( r""" @@ -109,7 +116,7 @@ def _is_posix_path(string): ) -def _is_windows_path(string): +def _is_windows_path(string: str) -> bool: return bool( re.match( r""" @@ -125,11 +132,11 @@ def _is_windows_path(string): ) -def _is_filesystem_path(string): +def _is_filesystem_path(string: str) -> bool: return _is_posix_path(string) or _is_windows_path(string) -def guess_scheme(url): +def guess_scheme(url: str) -> str: """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" if _is_filesystem_path(url): @@ -138,12 +145,12 @@ def guess_scheme(url): def strip_url( - url, - strip_credentials=True, - strip_default_port=True, - origin_only=False, - strip_fragment=True, -): + url: str, + strip_credentials: bool = True, + strip_default_port: bool = True, + origin_only: bool = False, + strip_fragment: bool = True, +) -> str: """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" From 7347d021457866ade62a6ed8a0839766e91032e4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 01:12:52 +0400 Subject: [PATCH 059/211] Add typing to scrapy/utils/datatypes.py. --- scrapy/resolver.py | 8 +++++--- scrapy/utils/datatypes.py | 28 ++++++++++++++++------------ scrapy/utils/misc.py | 12 +++++++----- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6cbe01cbf..e2e8beff4 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,3 +1,5 @@ +from typing import Any + from twisted.internet import defer from twisted.internet.base import ThreadedResolver from twisted.internet.interfaces import ( @@ -11,7 +13,7 @@ from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache # TODO: cache misses -dnscache = LocalCache(10000) +dnscache: LocalCache[str, Any] = LocalCache(10000) @implementer(IResolverSimple) @@ -36,7 +38,7 @@ class CachingThreadedResolver(ThreadedResolver): def install_on_reactor(self): self.reactor.installResolver(self) - def getHostByName(self, name, timeout=None): + def getHostByName(self, name: str, timeout=None): if name in dnscache: return defer.succeed(dnscache[name]) # in Twisted<=16.6, getHostByName() is always called with @@ -110,7 +112,7 @@ class CachingHostnameResolver: def resolveHostName( self, resolutionReceiver, - hostName, + hostName: str, portNumber=0, addressTypes=None, transportSemantics="TCP", diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index fa57a4f26..599b201ea 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,6 +8,10 @@ This module must not depend on any module outside the Standard Library. import collections import weakref from collections.abc import Mapping +from typing import Any, Optional, Sequence, TypeVar + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") class CaselessDict(dict): @@ -64,24 +68,24 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class LocalCache(collections.OrderedDict): +class LocalCache(collections.OrderedDict[_KT, _VT]): """Dictionary with a finite number of keys. Older items expires first. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.limit = limit + self.limit: Optional[int] = limit - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: if self.limit: while len(self) >= self.limit: self.popitem(last=False) super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. @@ -93,17 +97,17 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): it cannot be instantiated with an initial dictionary. """ - def __init__(self, limit=None): + def __init__(self, limit: Optional[int] = None): super().__init__() - self.data = LocalCache(limit=limit) + self.data: LocalCache = LocalCache(limit=limit) - def __setitem__(self, key, value): + def __setitem__(self, key: _KT, value: _VT) -> None: try: super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching - def __getitem__(self, key): + def __getitem__(self, key: _KT) -> Optional[_VT]: # type: ignore[override] try: return super().__getitem__(key) except (TypeError, KeyError): @@ -113,8 +117,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): class SequenceExclude: """Object to test if an item is NOT within some sequence.""" - def __init__(self, seq): - self.seq = seq + def __init__(self, seq: Sequence): + self.seq: Sequence = seq - def __contains__(self, item): + def __contains__(self, item: Any) -> bool: return item not in self.seq diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..ea3f934c7 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -214,16 +214,18 @@ def walk_callable(node): yield node -_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache: LocalWeakReferencedCache[ + Callable, bool +] = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable): +def is_generator_with_return_value(callable: Callable) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise """ if callable in _generator_callbacks_cache: - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) def returns_none(return_node): value = return_node.value @@ -248,10 +250,10 @@ def is_generator_with_return_value(callable): for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): _generator_callbacks_cache[callable] = True - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) _generator_callbacks_cache[callable] = False - return _generator_callbacks_cache[callable] + return bool(_generator_callbacks_cache[callable]) def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None: From ea299dfd7ce8766c2c9430fe8b297fddad45adee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 01:21:00 +0400 Subject: [PATCH 060/211] Add typing to scrapy/utils/misc.py. --- scrapy/utils/misc.py | 46 +++++++++++++++++++++++++++++------------- scrapy/utils/python.py | 4 ++-- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ea3f934c7..defc8663d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -10,7 +10,21 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import TYPE_CHECKING, Any, Callable, Union +from types import ModuleType +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Deque, + Generator, + Iterable, + List, + Optional, + Pattern, + Union, + cast, +) from w3lib.html import replace_entities @@ -26,7 +40,7 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -def arg_to_iter(arg): +def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -35,7 +49,7 @@ def arg_to_iter(arg): if arg is None: return [] if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return arg + return cast(Iterable[Any], arg) return [arg] @@ -72,7 +86,7 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def walk_modules(path): +def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. @@ -80,7 +94,7 @@ def walk_modules(path): For example: walk_modules('scrapy.utils') """ - mods = [] + mods: List[ModuleType] = [] mod = import_module(path) mods.append(mod) if hasattr(mod, "__path__"): @@ -94,7 +108,9 @@ def walk_modules(path): return mods -def extract_regex(regex, text, encoding="utf-8"): +def extract_regex( + regex: Union[str, Pattern], text: str, encoding: str = "utf-8" +) -> List[str]: """Extract a list of unicode strings from the given text/encoding using the following policies: * if the regex contains a named group called "extract" that will be returned @@ -111,9 +127,11 @@ def extract_regex(regex, text, encoding="utf-8"): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group("extract")] # named group + # named group + strings = [regex.search(text).group("extract")] # type: ignore[union-attr] except Exception: - strings = regex.findall(text) # full regex or numbered groups + # full regex or numbered groups + strings = regex.findall(text) strings = flatten(strings) if isinstance(text, str): @@ -123,7 +141,7 @@ def extract_regex(regex, text, encoding="utf-8"): ] -def md5sum(file): +def md5sum(file: IO) -> str: """Calculate the md5 checksum of a file-like object without reading its whole content in memory. @@ -140,7 +158,7 @@ def md5sum(file): return m.hexdigest() -def rel_has_nofollow(rel): +def rel_has_nofollow(rel: Optional[str]) -> bool: """Return True if link rel attribute has nofollow type""" return rel is not None and "nofollow" in rel.replace(",", " ").split() @@ -181,7 +199,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): @contextmanager -def set_environ(**kwargs): +def set_environ(**kwargs: str) -> Generator[None, Any, None]: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -198,11 +216,11 @@ def set_environ(**kwargs): os.environ[k] = v -def walk_callable(node): +def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: """Similar to ``ast.walk``, but walks only function body and skips nested functions defined within the node. """ - todo = deque([node]) + todo: Deque[ast.AST] = deque([node]) walked_func_def = False while todo: node = todo.popleft() @@ -227,7 +245,7 @@ def is_generator_with_return_value(callable: Callable) -> bool: if callable in _generator_callbacks_cache: return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node): + def returns_none(return_node: ast.Return) -> bool: value = return_node.value return ( value is None or isinstance(value, ast.NameConstant) and value.value is None diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..ae8feaf7d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -22,7 +22,7 @@ from typing import ( from scrapy.utils.asyncgen import as_async_generator -def flatten(x): +def flatten(x: Iterable) -> list: """flatten(sequence) -> list Returns a single, flat list which contains all elements retrieved @@ -42,7 +42,7 @@ def flatten(x): return list(iflatten(x)) -def iflatten(x): +def iflatten(x: Iterable) -> Iterable: """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" From 43ee483a0dc6c7883ba9f219ac8b4fd95647b83d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 01:34:54 +0400 Subject: [PATCH 061/211] Add typing to scrapy/utils/reactor.py. --- scrapy/utils/reactor.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index f1b9239e6..ad3d1d8bc 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,7 +1,8 @@ import asyncio import sys +from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence +from typing import Any, Callable, Dict, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error @@ -57,7 +58,7 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) -def set_asyncio_event_loop_policy(): +def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. This should only be used to install the reactor. @@ -65,7 +66,7 @@ def set_asyncio_event_loop_policy(): _get_asyncio_event_loop_policy() -def get_asyncio_event_loop_policy(): +def get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: warn( "Call to deprecated function " "scrapy.utils.reactor.get_asyncio_event_loop_policy().\n" @@ -81,7 +82,7 @@ def get_asyncio_event_loop_policy(): return _get_asyncio_event_loop_policy() -def _get_asyncio_event_loop_policy(): +def _get_asyncio_event_loop_policy() -> AbstractEventLoopPolicy: policy = asyncio.get_event_loop_policy() if ( sys.version_info >= (3, 8) @@ -93,7 +94,7 @@ def _get_asyncio_event_loop_policy(): return policy -def install_reactor(reactor_path, event_loop_path=None): +def install_reactor(reactor_path: str, event_loop_path: Optional[str] = None) -> None: """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import path if the asyncio reactor is enabled""" @@ -111,14 +112,14 @@ def install_reactor(reactor_path, event_loop_path=None): installer() -def _get_asyncio_event_loop(): +def _get_asyncio_event_loop() -> AbstractEventLoop: return set_asyncio_event_loop(None) -def set_asyncio_event_loop(event_loop_path): +def set_asyncio_event_loop(event_loop_path: Optional[str]) -> AbstractEventLoop: """Sets and returns the event loop with specified import path.""" if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) + event_loop_class: Type[AbstractEventLoop] = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: @@ -146,7 +147,7 @@ def set_asyncio_event_loop(event_loop_path): return event_loop -def verify_installed_reactor(reactor_path): +def verify_installed_reactor(reactor_path: str) -> None: """Raises :exc:`Exception` if the installed :mod:`~twisted.internet.reactor` does not match the specified import path.""" @@ -162,7 +163,7 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) -def verify_installed_asyncio_event_loop(loop_path): +def verify_installed_asyncio_event_loop(loop_path: str) -> None: from twisted.internet import reactor loop_class = load_object(loop_path) @@ -181,7 +182,7 @@ def verify_installed_asyncio_event_loop(loop_path): ) -def is_asyncio_reactor_installed(): +def is_asyncio_reactor_installed() -> bool: from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) From 4da86915109f77066c306130ecd122a17430191d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 02:02:33 +0400 Subject: [PATCH 062/211] Add typing to scrapy/utils/gz.py and remove dead code. --- scrapy/utils/gz.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..98ca510ed 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,15 +1,18 @@ import struct from gzip import GzipFile from io import BytesIO +from typing import List + +from scrapy.http import Response -def gunzip(data): +def gunzip(data: bytes) -> bytes: """Gunzip the given data and return as much data as possible. This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output_list = [] + output_list: List[bytes] = [] chunk = b"." while chunk: try: @@ -18,17 +21,13 @@ def gunzip(data): except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output_list is empty and f.extrabuf - # contains the whole page content - if output_list or getattr(f, "extrabuf", None): - try: - output_list.append(f.extrabuf[-f.extrasize :]) - finally: - break + # some pages are quite small so output_list is empty + if output_list: + break else: raise return b"".join(output_list) -def gzip_magic_number(response): +def gzip_magic_number(response: Response) -> bool: return response.body[:3] == b"\x1f\x8b\x08" From d400f1ac0664768d04985f0a00bc6d47a54957fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 18:35:24 +0400 Subject: [PATCH 063/211] Add more typing to scrapy/utils/python.py. --- scrapy/utils/python.py | 58 ++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index ae8feaf7d..0b5dc324f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import collections.abc import gc import inspect import re @@ -12,9 +13,17 @@ from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, + Callable, + Dict, + Generator, Iterable, + Iterator, + List, Mapping, Optional, + Pattern, + Tuple, Union, overload, ) @@ -78,7 +87,7 @@ def is_listlike(x: Any) -> bool: return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) -def unique(list_, key=lambda x: x): +def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list: """efficient function to uniquify a list preserving item order""" seen = set() result = [] @@ -124,7 +133,9 @@ def to_bytes( return text.encode(encoding, errors) -def re_rsearch(pattern, text, chunk_size=1024): +def re_rsearch( + pattern: Union[str, Pattern], text: str, chunk_size: int = 1024 +) -> Optional[Tuple[int, int]]: """ This function does a reverse search in a text using a regular expression given in the attribute 'pattern'. @@ -138,7 +149,7 @@ def re_rsearch(pattern, text, chunk_size=1024): the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter(): + def _chunk_iter() -> Generator[Tuple[str, int], Any, None]: offset = len(text) while True: offset -= chunk_size * 1024 @@ -158,14 +169,14 @@ def re_rsearch(pattern, text, chunk_size=1024): return None -def memoizemethod_noargs(method): +def memoizemethod_noargs(method: Callable) -> Callable: """Decorator to cache the result of a method (without arguments) using a weak reference to its object """ - cache = weakref.WeakKeyDictionary() + cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary() @wraps(method) - def new_method(self, *args, **kwargs): + def new_method(self: Any, *args: Any, **kwargs: Any) -> Any: if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] @@ -187,12 +198,12 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func, stripself=False): +def get_func_args(func: Callable, stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") - args = [] + args: List[str] = [] try: sig = inspect.signature(func) except ValueError: @@ -217,7 +228,7 @@ def get_func_args(func, stripself=False): return args -def get_spec(func): +def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -246,7 +257,7 @@ def get_spec(func): else: raise TypeError(f"{type(func)} is not callable") - defaults = spec.defaults or [] + defaults: Tuple[Any, ...] = spec.defaults or () firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] @@ -254,7 +265,9 @@ def get_spec(func): return args, kwargs -def equal_attributes(obj1, obj2, attributes): +def equal_attributes( + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] +) -> bool: """Compare two objects attributes""" # not attributes given return False by default if not attributes: @@ -282,19 +295,20 @@ def without_none_values(iterable: Iterable) -> Iterable: ... -def without_none_values(iterable): +def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: """Return a copy of ``iterable`` with all ``None`` entries removed. If ``iterable`` is a mapping, return a dictionary where all pairs that have value ``None`` have been removed. """ - try: + if isinstance(iterable, collections.abc.Mapping): return {k: v for k, v in iterable.items() if v is not None} - except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + else: + # the iterable __init__ must take another iterable + return type(iterable)(v for v in iterable if v is not None) # type: ignore[call-arg] -def global_object_name(obj): +def global_object_name(obj: Any) -> str: """ Return full name of a global object. @@ -307,14 +321,14 @@ def global_object_name(obj): if hasattr(sys, "pypy_version_info"): - def garbage_collect(): + def garbage_collect() -> None: # Collecting weakreferences can take two collections on PyPy. gc.collect() gc.collect() else: - def garbage_collect(): + def garbage_collect() -> None: gc.collect() @@ -329,10 +343,10 @@ class MutableChain(Iterable): def extend(self, *iterables: Iterable) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) - def __iter__(self): + def __iter__(self) -> Iterator: return self - def __next__(self): + def __next__(self) -> Any: return next(self.data) @@ -353,8 +367,8 @@ class MutableAsyncChain(AsyncIterable): def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: self.data = _async_chain(self.data, _async_chain(*iterables)) - def __aiter__(self): + def __aiter__(self) -> AsyncIterator: return self - async def __anext__(self): + async def __anext__(self) -> Any: return await self.data.__anext__() From 9661a5c4913e4be314572e12a49c215e7631db88 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 18:50:23 +0400 Subject: [PATCH 064/211] Add typing to scrapy/utils/deprecate.py. --- scrapy/utils/deprecate.py | 59 ++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index f4d6e0451..ab2719bb3 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,12 +2,12 @@ import inspect import warnings -from typing import List, Tuple +from typing import Any, List, Optional, Tuple, Type, overload from scrapy.exceptions import ScrapyDeprecationWarning -def attribute(obj, oldattr, newattr, version="0.12"): +def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> None: cname = obj.__class__.__name__ warnings.warn( f"{cname}.{oldattr} attribute is deprecated and will be no longer supported " @@ -18,16 +18,16 @@ def attribute(obj, oldattr, newattr, version="0.12"): def create_deprecated_class( - name, - new_class, - clsdict=None, - warn_category=ScrapyDeprecationWarning, - warn_once=True, - old_class_path=None, - new_class_path=None, - subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.", - instance_warn_message="{cls} is deprecated, instantiate {new} instead.", -): + name: str, + new_class: type, + clsdict: Optional[dict[str, Any]] = None, + warn_category: Type[Warning] = ScrapyDeprecationWarning, + warn_once: bool = True, + old_class_path: Optional[str] = None, + new_class_path: Optional[str] = None, + subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.", + instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.", +) -> type: """ Return a "deprecated" class that causes its subclasses to issue a warning. Subclasses of ``new_class`` are considered subclasses of this class. @@ -53,17 +53,20 @@ def create_deprecated_class( OldName. """ - class DeprecatedClass(new_class.__class__): - deprecated_class = None - warned_on_subclass = False + # https://github.com/python/mypy/issues/4177 + class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + deprecated_class: Optional[type] = None + warned_on_subclass: bool = False - def __new__(metacls, name, bases, clsdict_): + def __new__( + metacls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any] + ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name, bases, clsdict_): + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): @@ -81,10 +84,10 @@ def create_deprecated_class( # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details - def __instancecheck__(cls, inst): + def __instancecheck__(cls, inst: Any) -> bool: return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__)) - def __subclasscheck__(cls, sub): + def __subclasscheck__(cls, sub: type) -> bool: if cls is not DeprecatedClass.deprecated_class: # we should do the magic only if second `issubclass` argument # is the deprecated class itself - subclasses of the @@ -98,7 +101,7 @@ def create_deprecated_class( mro = getattr(sub, "__mro__", ()) return any(c in {cls, new_class} for c in mro) - def __call__(cls, *args, **kwargs): + def __call__(cls, *args: Any, **kwargs: Any) -> Any: old = DeprecatedClass.deprecated_class if cls is old: msg = instance_warn_message.format( @@ -125,7 +128,7 @@ def create_deprecated_class( return deprecated_cls -def _clspath(cls, forced=None): +def _clspath(cls: type, forced: Optional[str] = None) -> str: if forced is not None: return forced return f"{cls.__module__}.{cls.__name__}" @@ -134,7 +137,17 @@ def _clspath(cls, forced=None): DEPRECATION_RULES: List[Tuple[str, str]] = [] -def update_classpath(path): +@overload +def update_classpath(path: str) -> str: + ... + + +@overload +def update_classpath(path: Any) -> Any: + ... + + +def update_classpath(path: Any) -> Any: """Update a deprecated path from an object with its new location""" for prefix, replacement in DEPRECATION_RULES: if isinstance(path, str) and path.startswith(prefix): @@ -147,7 +160,7 @@ def update_classpath(path): return path -def method_is_overridden(subclass, base_class, method_name): +def method_is_overridden(subclass: type, base_class: type, method_name: str) -> bool: """ Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. From b8277f4cab7941989402a8339634a91dcd3500c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:08:46 +0400 Subject: [PATCH 065/211] Add more typing to scrapy/utils/defer.py. --- scrapy/utils/defer.py | 57 ++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d25ebbdf4..307707bf5 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -9,13 +9,16 @@ from typing import ( Any, AsyncGenerator, AsyncIterable, + AsyncIterator, Callable, Coroutine, + Dict, Generator, Iterable, Iterator, List, Optional, + Tuple, Union, cast, ) @@ -44,7 +47,7 @@ def defer_fail(_failure: Failure) -> Deferred: return d -def defer_succeed(result) -> Deferred: +def defer_succeed(result: Any) -> Deferred: """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -58,7 +61,7 @@ def defer_succeed(result) -> Deferred: return d -def defer_result(result) -> Deferred: +def defer_result(result: Any) -> Deferred: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -66,7 +69,7 @@ def defer_result(result) -> Deferred: return defer_succeed(result) -def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: +def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred: """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop """ @@ -84,7 +87,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: def parallel( - iterable: Iterable, count: int, callable: Callable, *args, **named + iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any ) -> Deferred: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -146,14 +149,14 @@ class _AsyncCooperatorAdapter(Iterator): self, aiterable: AsyncIterable, callable: Callable, - *callable_args, - **callable_kwargs + *callable_args: Any, + **callable_kwargs: Any, ): - self.aiterator = aiterable.__aiter__() - self.callable = callable - self.callable_args = callable_args - self.callable_kwargs = callable_kwargs - self.finished = False + self.aiterator: AsyncIterator = aiterable.__aiter__() + self.callable: Callable = callable + self.callable_args: Tuple[Any, ...] = callable_args + self.callable_kwargs: Dict[str, Any] = callable_kwargs + self.finished: bool = False self.waiting_deferreds: List[Deferred] = [] self.anext_deferred: Optional[Deferred] = None @@ -201,7 +204,11 @@ class _AsyncCooperatorAdapter(Iterator): def parallel_async( - async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named + async_iterable: AsyncIterable, + count: int, + callable: Callable, + *args: Any, + **named: Any, ) -> Deferred: """Like parallel but for async iterators""" coop = Cooperator() @@ -210,7 +217,9 @@ def parallel_async( return dl -def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_chain( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred built by chaining the given callbacks""" d: Deferred = Deferred() for x in callbacks: @@ -220,7 +229,11 @@ def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: def process_chain_both( - callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw + callbacks: Iterable[Callable], + errbacks: Iterable[Callable], + input: Any, + *a: Any, + **kw: Any, ) -> Deferred: """Return a Deferred built by chaining the given callbacks and errbacks""" d: Deferred = Deferred() @@ -240,7 +253,9 @@ def process_chain_both( return d -def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred: +def process_parallel( + callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any +) -> Deferred: """Return a Deferred with the output of all successful calls to the given callbacks """ @@ -250,7 +265,9 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred return d -def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: +def iter_errback( + iterable: Iterable, errback: Callable, *a: Any, **kw: Any +) -> Generator: """Wraps an iterable calling an errback if an error is caught while iterating it. """ @@ -265,7 +282,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: async def aiter_errback( - aiterable: AsyncIterable, errback: Callable, *a, **kw + aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any ) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() @@ -280,7 +297,7 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o) -> Any: +def deferred_from_coro(o: Any) -> Any: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o @@ -303,13 +320,13 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable: """ @wraps(coro_f) - def f(*coro_args, **coro_kwargs): + def f(*coro_args: Any, **coro_kwargs: Any) -> Any: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f -def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: +def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) From c04b9ba19de85154c7bfec536f584f31456e44b3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:10:42 +0400 Subject: [PATCH 066/211] Add typing to scrapy/utils/template.py. --- scrapy/utils/template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 1499aeb3d..6b22f3bfa 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -4,10 +4,10 @@ import re import string from os import PathLike from pathlib import Path -from typing import Union +from typing import Any, Union -def render_templatefile(path: Union[str, PathLike], **kwargs): +def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") @@ -24,7 +24,7 @@ def render_templatefile(path: Union[str, PathLike], **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r"[^a-zA-Z\d]") -def string_camelcase(string): +def string_camelcase(string: str) -> str: """Convert a word to its CamelCase version and remove invalid chars >>> string_camelcase('lost-pound') From 36507ddb7b55d6fc4bfdd19286d82057ef3fb5cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:16:57 +0400 Subject: [PATCH 067/211] Add typing to scrapy/utils/engine.py. --- scrapy/utils/engine.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 8e3ec2c37..a5f2a8c6e 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -2,9 +2,13 @@ # used in global tests code from time import time # noqa: F401 +from typing import TYPE_CHECKING, Any, List, Tuple + +if TYPE_CHECKING: + from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine): +def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -23,7 +27,7 @@ def get_engine_status(engine): "engine.scraper.slot.needs_backout()", ] - checks = [] + checks: List[Tuple[str, Any]] = [] for test in tests: try: checks += [(test, eval(test))] @@ -33,7 +37,7 @@ def get_engine_status(engine): return checks -def format_engine_status(engine=None): +def format_engine_status(engine: "ExecutionEngine") -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -43,5 +47,5 @@ def format_engine_status(engine=None): return s -def print_engine_status(engine): +def print_engine_status(engine: "ExecutionEngine") -> None: print(format_engine_status(engine)) From f38cea9c8c5410e0553c666f090fb150a68ae591 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 19:22:57 +0400 Subject: [PATCH 068/211] Add typing to scrapy/utils/display.py. --- scrapy/utils/display.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 77c32b002..596cf89e4 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,17 +6,18 @@ import ctypes import platform import sys from pprint import pformat as pformat_ +from typing import Any from packaging.version import Version as parse_version -def _enable_windows_terminal_processing(): +def _enable_windows_terminal_processing() -> bool: # https://stackoverflow.com/a/36760881 - kernel32 = ctypes.windll.kernel32 + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)) -def _tty_supports_color(): +def _tty_supports_color() -> bool: if sys.platform != "win32": return True @@ -28,7 +29,7 @@ def _tty_supports_color(): return _enable_windows_terminal_processing() -def _colorize(text, colorize=True): +def _colorize(text: str, colorize: bool = True) -> str: if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: @@ -42,9 +43,9 @@ def _colorize(text, colorize=True): return highlight(text, PythonLexer(), TerminalFormatter()) -def pformat(obj, *args, **kwargs): +def pformat(obj: Any, *args: Any, **kwargs: Any) -> str: return _colorize(pformat_(obj), kwargs.pop("colorize", True)) -def pprint(obj, *args, **kwargs): +def pprint(obj: Any, *args: Any, **kwargs: Any) -> None: print(pformat(obj, *args, **kwargs)) From 54fa04aa0a47314f121ff5a7627e7e47d4d2c013 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 20:32:34 +0400 Subject: [PATCH 069/211] Add typing to scrapy/utils/test.py, fix a FTP test. --- scrapy/crawler.py | 18 +++++++------ scrapy/spiderloader.py | 11 +++++--- scrapy/utils/test.py | 49 ++++++++++++++++++++++++------------ tests/test_pipeline_files.py | 41 +++++++++++++++--------------- 4 files changed, 71 insertions(+), 48 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 256f6e2c5..9631c73d6 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,11 +4,13 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement +from scrapy.spiderloader import SpiderLoader + try: # zope >= 5.0 only supports MultipleInvalid from zope.interface.exceptions import MultipleInvalid @@ -171,7 +173,7 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings): + def _get_spider_loader(settings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") loader_cls = load_object(cls_path) @@ -190,13 +192,13 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False @property @@ -252,7 +254,9 @@ class CrawlerRunner: return d.addBoth(_done) - def create_crawler(self, crawler_or_spidercls): + def create_crawler( + self, crawler_or_spidercls: Union[Type[Spider], str, Crawler] + ) -> Crawler: """ Return a :class:`~scrapy.crawler.Crawler` object. @@ -272,7 +276,7 @@ class CrawlerRunner: return crawler_or_spidercls return self._create_crawler(crawler_or_spidercls) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 02a451a2b..ea5a26e77 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,10 +1,13 @@ import traceback import warnings from collections import defaultdict +from typing import DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer +from scrapy import Spider from scrapy.interfaces import ISpiderLoader +from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes @@ -16,11 +19,11 @@ class SpiderLoader: in a Scrapy project. """ - def __init__(self, settings): + def __init__(self, settings: BaseSettings): self.spider_modules = settings.getlist("SPIDER_MODULES") self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") - self._spiders = {} - self._found = defaultdict(list) + self._spiders: Dict[str, Type[Spider]] = {} + self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() def _check_name_duplicates(self): @@ -68,7 +71,7 @@ class SpiderLoader: def from_settings(cls, settings): return cls(settings) - def load(self, spider_name): + def load(self, spider_name: str) -> Type[Spider]: """ Return the Spider class for the given spider name. If the spider name is not found, raise a KeyError. diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 58576903a..97de8d25a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,24 +7,30 @@ import os from importlib import import_module from pathlib import Path from posixpath import split -from unittest import mock +from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type +from unittest import TestCase, mock +from twisted.internet.defer import Deferred from twisted.trial.unittest import SkipTest +from scrapy import Spider +from scrapy.crawler import Crawler from scrapy.utils.boto import is_botocore_available -def assert_gcs_environ(): +def assert_gcs_environ() -> None: if "GCS_PROJECT_ID" not in os.environ: raise SkipTest("GCS_PROJECT_ID not found") -def skip_if_no_boto(): +def skip_if_no_boto() -> None: if not is_botocore_available(): raise SkipTest("missing botocore library") -def get_gcs_content_and_delete(bucket, path): +def get_gcs_content_and_delete( + bucket: Any, path: str +) -> Tuple[bytes, List[Dict[str, str]], Any]: from google.cloud import storage client = storage.Client(project=os.environ.get("GCS_PROJECT_ID")) @@ -37,8 +43,13 @@ def get_gcs_content_and_delete(bucket, path): def get_ftp_content_and_delete( - path, host, port, username, password, use_active_mode=False -): + path: str, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, +) -> bytes: from ftplib import FTP ftp = FTP() @@ -46,19 +57,23 @@ def get_ftp_content_and_delete( ftp.login(username, password) if use_active_mode: ftp.set_pasv(False) - ftp_data = [] + ftp_data: List[bytes] = [] - def buffer_data(data): + def buffer_data(data: bytes) -> None: ftp_data.append(data) ftp.retrbinary(f"RETR {path}", buffer_data) dirname, filename = split(path) ftp.cwd(dirname) ftp.delete(filename) - return "".join(ftp_data) + return b"".join(ftp_data) -def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): +def get_crawler( + spidercls: Optional[Type[Spider]] = None, + settings_dict: Optional[Dict[str, Any]] = None, + prevent_warnings: bool = True, +) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -82,7 +97,7 @@ def get_pythonpath() -> str: return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get("PYTHONPATH", "") -def get_testenv(): +def get_testenv() -> Dict[str, str]: """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. """ @@ -91,21 +106,23 @@ def get_testenv(): return env -def assert_samelines(testcase, text1, text2, msg=None): +def assert_samelines( + testcase: TestCase, text1: str, text2: str, msg: Optional[str] = None +) -> None: """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) -def get_from_asyncio_queue(value): - q = asyncio.Queue() +def get_from_asyncio_queue(value: Any) -> Coroutine: + q: asyncio.Queue = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter -def mock_google_cloud_storage(): +def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. """ @@ -122,7 +139,7 @@ def mock_google_cloud_storage(): return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url): +def get_web_client_agent_req(url: str) -> Deferred: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c80666586..859ad6f9c 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -32,6 +32,7 @@ from scrapy.utils.test import ( get_gcs_content_and_delete, skip_if_no_boto, ) +from tests.mockserver import MockFTPServer from .test_pipeline_media import _mocked_download_func @@ -639,31 +640,29 @@ class TestGCSFilesStore(unittest.TestCase): class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): - uri = os.environ.get("FTP_TEST_FILE_URI") - if not uri: - raise unittest.SkipTest("No FTP URI available for testing") data = b"TestFTPFilesStore: \xe2\x98\x83" buf = BytesIO(data) meta = {"foo": "bar"} path = "full/filename" - store = FTPFilesStore(uri) - empty_dict = yield store.stat_file(path, info=None) - self.assertEqual(empty_dict, {}) - yield store.persist_file(path, buf, info=None, meta=meta, headers=None) - stat = yield store.stat_file(path, info=None) - self.assertIn("last_modified", stat) - self.assertIn("checksum", stat) - self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") - path = f"{store.basedir}/{path}" - content = get_ftp_content_and_delete( - path, - store.host, - store.port, - store.username, - store.password, - store.USE_ACTIVE_MODE, - ) - self.assertEqual(data.decode(), content) + with MockFTPServer() as ftp_server: + store = FTPFilesStore(ftp_server.url("/")) + empty_dict = yield store.stat_file(path, info=None) + self.assertEqual(empty_dict, {}) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + stat = yield store.stat_file(path, info=None) + self.assertIn("last_modified", stat) + self.assertIn("checksum", stat) + self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6") + path = f"{store.basedir}/{path}" + content = get_ftp_content_and_delete( + path, + store.host, + store.port, + store.username, + store.password, + store.USE_ACTIVE_MODE, + ) + self.assertEqual(data, content) class ItemWithFiles(Item): From 048812ba350d9f7fe6831d4c6f0d60d222b7f131 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 22:40:29 +0400 Subject: [PATCH 070/211] Bump types-* versions. --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index af8f1f57a..a1e956bf4 100644 --- a/tox.ini +++ b/tox.ini @@ -40,10 +40,10 @@ deps = mypy==1.2.0 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==9.4.0.19 - types-Pygments==2.14.0.7 - types-pyOpenSSL==23.1.0.1 - types-setuptools==67.6.0.7 + types-Pillow==9.5.0.2 + types-Pygments==2.15.0.0 + types-pyOpenSSL==23.1.0.2 + types-setuptools==67.7.0.1 commands = mypy --show-error-codes {posargs: scrapy tests} From 0ec79e316619c1c98b0a1dd4fb0edea6e6de803d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 23:01:27 +0400 Subject: [PATCH 071/211] Fix compatibility with Python 3.8. --- scrapy/utils/deprecate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ab2719bb3..ea577c44a 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,7 +2,7 @@ import inspect import warnings -from typing import Any, List, Optional, Tuple, Type, overload +from typing import Any, Dict, List, Optional, Tuple, Type, overload from scrapy.exceptions import ScrapyDeprecationWarning @@ -20,7 +20,7 @@ def attribute(obj: Any, oldattr: str, newattr: str, version: str = "0.12") -> No def create_deprecated_class( name: str, new_class: type, - clsdict: Optional[dict[str, Any]] = None, + clsdict: Optional[Dict[str, Any]] = None, warn_category: Type[Warning] = ScrapyDeprecationWarning, warn_once: bool = True, old_class_path: Optional[str] = None, @@ -59,14 +59,14 @@ def create_deprecated_class( warned_on_subclass: bool = False def __new__( - metacls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any] + metacls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any] ) -> type: cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls - def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: dict[str, Any]): + def __init__(cls, name: str, bases: Tuple[type, ...], clsdict_: Dict[str, Any]): meta = cls.__class__ old = meta.deprecated_class if old in bases and not (warn_once and meta.warned_on_subclass): From e03c6bb70a8915f997771472ef05efc0c3ad6bd6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 7 May 2023 23:03:35 +0400 Subject: [PATCH 072/211] Fix pylint issues. --- scrapy/crawler.py | 3 +-- scrapy/utils/gz.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9631c73d6..69ff07bb7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -9,8 +9,6 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement -from scrapy.spiderloader import SpiderLoader - try: # zope >= 5.0 only supports MultipleInvalid from zope.interface.exceptions import MultipleInvalid @@ -27,6 +25,7 @@ from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 98ca510ed..c0eb77e07 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -24,8 +24,7 @@ def gunzip(data: bytes) -> bytes: # some pages are quite small so output_list is empty if output_list: break - else: - raise + raise return b"".join(output_list) From caa66fa15ae5d353434e5bb1d0c1cc2bdf857b6c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 May 2023 13:27:01 +0400 Subject: [PATCH 073/211] Mention deprecating _FeedSlot. --- docs/news.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cbbb376e5..4e198c893 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -15,6 +15,13 @@ Highlights: - JMESPath selectors from the new parsel_. - Bug fixes. +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to + :class:`scrapy.extensions.feedexport.FeedSlot` and the old name is + deprecated. (:issue:`5876`) + New features ~~~~~~~~~~~~ From 52c072640aa61884de05214cb1bdda07c2a87bef Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 May 2023 14:30:06 +0400 Subject: [PATCH 074/211] =?UTF-8?q?Bump=20version:=202.8.0=20=E2=86=92=202?= =?UTF-8?q?.9.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4cfba674d..a00b7cfb3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.8.0 +current_version = 2.9.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 4e198c893..c7ad11862 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.9.0: -Scrapy 2.9.0 (YYYY-MM-DD) +Scrapy 2.9.0 (2023-05-08) ------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 834f26295..c8e38b614 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.8.0 +2.9.0 From c327a92e971411e50e49dded772a73b293f9f0a9 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 18:04:18 +0500 Subject: [PATCH 075/211] add additional requests examples. --- docs/topics/asyncio.rst | 47 +++++++++++++++++++++++++++++++++++++++++ scrapy/utils/defer.py | 10 +++++---- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..7aa83f505 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,6 +98,53 @@ Futures. Scrapy provides two helpers for this: into your own code. +Async additional requests +===================== + +The spider below shows a single use-case of scraping page and gathering price from a separate url:: + + + class SingleRequestSpider(scrapy.Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + 'h1': response.css('h1').get(), + 'price': additional_response.css('#price').get(), + } + + +Spider with gathering batch requests:: + + class BatchRequestsSpider(scrapy.Spider): + name = "batch" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + scrapy.Request("https://example.com/price1"), + scrapy.Request("https://example.com/price2"), + ] + coroutines = [] + for r in additional_requests: + deffered = self.crawler.engine.download(r) + coroutines.append(maybe_deferred_to_future(deffered)) + + responses = await asyncio.gather( + *coroutines, return_exceptions=True + ) + yield { + 'h1': response.css('h1::text').get(), + 'price': responses[0].css('.price_color::text').get(), + 'price2': responses[1].css('.price_color::text').get(), + } + + + .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d25ebbdf4..a46274fef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - additional_response = await deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await deferred_to_future(deferred) """ return d.asFuture(_get_asyncio_event_loop()) @@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - extra_response = await maybe_deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) """ if not is_asyncio_reactor_installed(): return d From a75231a1ecd9a08b31ea3c1d5b59e457ac85ccf2 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 18:57:43 +0500 Subject: [PATCH 076/211] fix underline. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7aa83f505..d46527cfc 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this: Async additional requests -===================== +========================= The spider below shows a single use-case of scraping page and gathering price from a separate url:: From d32c6782347c97086e804da0da01f45622743198 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 19:02:34 +0500 Subject: [PATCH 077/211] Update description. --- docs/topics/asyncio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index d46527cfc..d439e0ab8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -101,7 +101,7 @@ Futures. Scrapy provides two helpers for this: Async additional requests ========================= -The spider below shows a single use-case of scraping page and gathering price from a separate url:: +The spider below shows a single use-case of scraping a page and gathering a price from a separate URL:: class SingleRequestSpider(scrapy.Spider): @@ -118,7 +118,7 @@ The spider below shows a single use-case of scraping page and gathering price fr } -Spider with gathering batch requests:: +The spider gathering batch requests:: class BatchRequestsSpider(scrapy.Spider): name = "batch" From 99b0ece165ff27b70a6d7375a86bcc67da111df7 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 20:27:46 +0500 Subject: [PATCH 078/211] remove extra line. --- docs/topics/asyncio.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index d439e0ab8..0dab0ac5e 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -144,7 +144,6 @@ The spider gathering batch requests:: } - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement From 6998e1c905ef6e5fa737b32ac0b6e7f1b7701c14 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 10 May 2023 14:21:18 +0400 Subject: [PATCH 079/211] Fix typing-related issued on Python < 3.9. --- scrapy/utils/datatypes.py | 7 +++---- scrapy/utils/misc.py | 4 +--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 599b201ea..0f6bdc5ab 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -5,10 +5,9 @@ Python Standard Library. This module must not depend on any module outside the Standard Library. """ -import collections import weakref from collections.abc import Mapping -from typing import Any, Optional, Sequence, TypeVar +from typing import Any, Optional, OrderedDict, Sequence, TypeVar _KT = TypeVar("_KT") _VT = TypeVar("_VT") @@ -68,7 +67,7 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class LocalCache(collections.OrderedDict[_KT, _VT]): +class LocalCache(OrderedDict[_KT, _VT]): """Dictionary with a finite number of keys. Older items expires first. @@ -85,7 +84,7 @@ class LocalCache(collections.OrderedDict[_KT, _VT]): super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT]): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index defc8663d..70187ba74 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -232,9 +232,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: yield node -_generator_callbacks_cache: LocalWeakReferencedCache[ - Callable, bool -] = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) def is_generator_with_return_value(callable: Callable) -> bool: From b1f4017788877ba2139f8621ecb5e821c62c111d Mon Sep 17 00:00:00 2001 From: bulat Date: Wed, 10 May 2023 15:34:58 +0500 Subject: [PATCH 080/211] Refactor batch sample. --- docs/topics/asyncio.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 0dab0ac5e..dc83148f5 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -126,8 +126,8 @@ The spider gathering batch requests:: async def parse(self, response, **kwargs): additional_requests = [ - scrapy.Request("https://example.com/price1"), - scrapy.Request("https://example.com/price2"), + scrapy.Request("https://example.com/price"), + scrapy.Request("https://example.com/color"), ] coroutines = [] for r in additional_requests: @@ -139,8 +139,8 @@ The spider gathering batch requests:: ) yield { 'h1': response.css('h1::text').get(), - 'price': responses[0].css('.price_color::text').get(), - 'price2': responses[1].css('.price_color::text').get(), + 'price': responses[0].css('.price::text').get(), + 'color': responses[1].css('color::text').get(), } From 87d10161cd413353eba2abf8ebdc2a8656927c43 Mon Sep 17 00:00:00 2001 From: bulat Date: Wed, 10 May 2023 15:35:48 +0500 Subject: [PATCH 081/211] Add selector as class. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index dc83148f5..f00ba0ff8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -140,7 +140,7 @@ The spider gathering batch requests:: yield { 'h1': response.css('h1::text').get(), 'price': responses[0].css('.price::text').get(), - 'color': responses[1].css('color::text').get(), + 'color': responses[1].css('.color::text').get(), } From 57f3140daaa0166f924fcca42d3f3d3ef178bf92 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Wed, 10 May 2023 18:31:54 +0500 Subject: [PATCH 082/211] Update docs/topics/asyncio.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index f00ba0ff8..f9efef108 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -101,8 +101,10 @@ Futures. Scrapy provides two helpers for this: Async additional requests ========================= -The spider below shows a single use-case of scraping a page and gathering a price from a separate URL:: +The spider below shows how to send a request and await its response all from +within a spider callback: +.. code-block:: python class SingleRequestSpider(scrapy.Spider): name = "single" From 26374e21f81eb53f5a21e1cc68a65e54fbb62cb6 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Wed, 10 May 2023 18:32:36 +0500 Subject: [PATCH 083/211] Update docs/topics/asyncio.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index f9efef108..7fe78585a 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -119,8 +119,9 @@ within a spider callback: 'price': additional_response.css('#price').get(), } +You can also send multiple requests in parallel: -The spider gathering batch requests:: +.. code-block:: python class BatchRequestsSpider(scrapy.Spider): name = "batch" From 85103b493289011161731e4fafec4320cd85e0af Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Thu, 11 May 2023 12:53:43 +0500 Subject: [PATCH 084/211] add proper example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7fe78585a..eeec76157 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -123,6 +123,8 @@ You can also send multiple requests in parallel: .. code-block:: python + from twisted.internet.defer import DeferredList + class BatchRequestsSpider(scrapy.Spider): name = "batch" start_urls = ["https://example.com/product"] @@ -132,14 +134,11 @@ You can also send multiple requests in parallel: scrapy.Request("https://example.com/price"), scrapy.Request("https://example.com/color"), ] - coroutines = [] + deferreds = [] for r in additional_requests: - deffered = self.crawler.engine.download(r) - coroutines.append(maybe_deferred_to_future(deffered)) - - responses = await asyncio.gather( - *coroutines, return_exceptions=True - ) + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) yield { 'h1': response.css('h1::text').get(), 'price': responses[0].css('.price::text').get(), From 6194db133518b07b92bfdae35332c69e95f5c415 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 12:54:01 +0500 Subject: [PATCH 085/211] Update title. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7fe78585a..1b670aaab 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,7 +98,7 @@ Futures. Scrapy provides two helpers for this: into your own code. -Async additional requests +Inline requests ========================= The spider below shows how to send a request and await its response all from From fc2d1b217130ebf605636049ea773196a50303a0 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 12:56:29 +0500 Subject: [PATCH 086/211] make example reachable. --- docs/topics/asyncio.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 12bf548df..fcf44c0cb 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,8 +125,8 @@ You can also send multiple requests in parallel: from twisted.internet.defer import DeferredList - class BatchRequestsSpider(scrapy.Spider): - name = "batch" + class MultipleRequestsSpider(scrapy.Spider): + name = "multiple" start_urls = ["https://example.com/product"] async def parse(self, response, **kwargs): @@ -141,8 +141,8 @@ You can also send multiple requests in parallel: responses = await maybe_deferred_to_future(DeferredList(deferreds)) yield { 'h1': response.css('h1::text').get(), - 'price': responses[0].css('.price::text').get(), - 'color': responses[1].css('.color::text').get(), + 'price': responses[0][1].css('.price::text').get(), + 'price2': responses[1][1].css('.color::text').get(), } From b62c1263de4b026e8529416d5dede1795d47f7ad Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:15:30 +0500 Subject: [PATCH 087/211] add import to the example. --- docs/topics/asyncio.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index fcf44c0cb..2ad784335 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -106,6 +106,8 @@ within a spider callback: .. code-block:: python + from scrapy.utils.defer import maybe_deferred_to_future + class SingleRequestSpider(scrapy.Spider): name = "single" start_urls = ["https://example.org/product"] From 4878cc7ef04ae40279d80fec5d5234d17569ce11 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:19:40 +0500 Subject: [PATCH 088/211] Add proper imports. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 2ad784335..a3f45a84b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,7 +125,7 @@ You can also send multiple requests in parallel: .. code-block:: python - from twisted.internet.defer import DeferredList + from scrapy.utils.defer import DeferredList class MultipleRequestsSpider(scrapy.Spider): name = "multiple" From 8de2064ba33d6e0b8e0a22a6b5f6928a35eb44b7 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:22:33 +0500 Subject: [PATCH 089/211] add import. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index a3f45a84b..5e0063be0 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,7 +125,7 @@ You can also send multiple requests in parallel: .. code-block:: python - from scrapy.utils.defer import DeferredList + from scrapy.utils.defer import DeferredList, maybe_deferred_to_future class MultipleRequestsSpider(scrapy.Spider): name = "multiple" From 5adada5d19e9e275462330b070b746305115112e Mon Sep 17 00:00:00 2001 From: isabela_catanante Date: Fri, 12 May 2023 12:55:24 +0200 Subject: [PATCH 090/211] Improve the overwrite feed option documentation --- docs/topics/feed-exports.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..b4ac93b1d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -209,6 +215,12 @@ You can also define a custom ACL and custom endpoint for exported feeds using th - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -236,6 +248,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python @@ -488,6 +506,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported `_) + - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported) + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) .. versionadded:: 2.4.0 From e4cf8fc121fc89d70949a9159bfe67cbd0429e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 May 2023 18:51:58 +0200 Subject: [PATCH 091/211] Update asyncio.rst --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 5e0063be0..efb93c844 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this: Inline requests -========================= +=============== The spider below shows how to send a request and await its response all from within a spider callback: From d362699fa3855c7fd6e11204ccd8668128e38a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 16 May 2023 13:39:02 +0200 Subject: [PATCH 092/211] Move inline request examples to the coroutines documentation --- docs/topics/asyncio.rst | 50 --------------------------------- docs/topics/coroutines.rst | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index efb93c844..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this: into your own code. -Inline requests -=============== - -The spider below shows how to send a request and await its response all from -within a spider callback: - -.. code-block:: python - - from scrapy.utils.defer import maybe_deferred_to_future - - class SingleRequestSpider(scrapy.Spider): - name = "single" - start_urls = ["https://example.org/product"] - - async def parse(self, response, **kwargs): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) - yield { - 'h1': response.css('h1').get(), - 'price': additional_response.css('#price').get(), - } - -You can also send multiple requests in parallel: - -.. code-block:: python - - from scrapy.utils.defer import DeferredList, maybe_deferred_to_future - - class MultipleRequestsSpider(scrapy.Spider): - name = "multiple" - start_urls = ["https://example.com/product"] - - async def parse(self, response, **kwargs): - additional_requests = [ - scrapy.Request("https://example.com/price"), - scrapy.Request("https://example.com/color"), - ] - deferreds = [] - for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) - yield { - 'h1': response.css('h1::text').get(), - 'price': responses[0][1].css('.price::text').get(), - 'price2': responses[1][1].css('.color::text').get(), - } - - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3916bd295..a65bab3ca 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -134,6 +134,63 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs +.. _inline-requests: + +Inline requests +=============== + +The spider below shows how to send a request and await its response all from +within a spider callback: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + + + class SingleRequestSpider(Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = Request("https://example.org/price") + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + "h1": response.css("h1").get(), + "price": additional_response.css("#price").get(), + } + +You can also send multiple requests in parallel: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + from twisted.internet.defer import DeferredList + + + class MultipleRequestsSpider(Spider): + name = "multiple" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + Request("https://example.com/price"), + Request("https://example.com/color"), + ] + deferreds = [] + for r in additional_requests: + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) + yield { + "h1": response.css("h1::text").get(), + "price": responses[0][1].css(".price::text").get(), + "price2": responses[1][1].css(".color::text").get(), + } + + .. _sync-async-spider-middleware: Mixing synchronous and asynchronous spider middlewares From 49839d6071832aab23093c34fa6ceb961fcdf9d0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 4 Jun 2023 19:59:58 +0400 Subject: [PATCH 093/211] Don't rely on get_testenv() for running mockserver. --- tests/mockserver.py | 19 +++++++++++++++---- tests/test_crawler.py | 11 ++++++++--- tests/test_proxy_connect.py | 3 --- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index eb4f8334e..647b0682e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,11 +1,13 @@ import argparse import json +import os import random import sys from pathlib import Path from shutil import rmtree from subprocess import PIPE, Popen from tempfile import mkdtemp +from typing import Dict from urllib.parse import urlencode from OpenSSL import SSL @@ -20,7 +22,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -32,6 +33,16 @@ def getarg(request, name, default=None, type=None): return default +def get_mockserver_env() -> Dict[str, str]: + """Return a OS environment dict suitable to run mockserver processes.""" + + tests_path = Path(__file__).parent.parent + pythonpath = str(tests_path) + os.pathsep + os.environ.get("PYTHONPATH", "") + env = os.environ.copy() + env["PYTHONPATH"] = pythonpath + return env + + # most of the following resources are copied from twisted.web.test.test_webclient class ForeverTakingResource(resource.Resource): """ @@ -264,7 +275,7 @@ class MockServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) http_address = self.proc.stdout.readline().strip().decode("ascii") https_address = self.proc.stdout.readline().strip().decode("ascii") @@ -308,7 +319,7 @@ class MockDNSServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.mockserver", "-t", "dns"], stdout=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) self.host = "127.0.0.1" self.port = int( @@ -331,7 +342,7 @@ class MockFTPServer: self.proc = Popen( [sys.executable, "-u", "-m", "tests.ftpserver", "-d", str(self.path)], stderr=PIPE, - env=get_testenv(), + env=get_mockserver_env(), ) for line in self.proc.stderr: if b"starting FTP server" in line: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 706bfbaa9..ecb9c9b62 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,4 +1,5 @@ import logging +import os import platform import subprocess import sys @@ -23,8 +24,8 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.misc import load_object from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler, get_testenv -from tests.mockserver import MockServer +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer, get_mockserver_env class BaseCrawlerTest(unittest.TestCase): @@ -289,12 +290,16 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: script_dir: Path + cwd = os.getcwd() def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args) p = subprocess.Popen( - args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE + args, + env=get_mockserver_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) stdout, stderr = p.communicate() return stderr.decode("utf-8") diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index c05f4da91..dc0a82086 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -21,8 +21,6 @@ class MitmProxy: auth_pass = "scrapy" def start(self): - from scrapy.utils.test import get_testenv - script = """ import sys from mitmproxy.tools.main import mitmdump @@ -46,7 +44,6 @@ sys.exit(mitmdump()) "--ssl-insecure", ], stdout=PIPE, - env=get_testenv(), ) line = self.proc.stdout.readline().decode("utf-8") host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1) From 493ea435384d4b5891129cbcf61d07b09a00ce7f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 12 Jun 2023 21:50:29 +0400 Subject: [PATCH 094/211] Improve finding tests.test_cmdline.settings. --- tests/test_cmdline/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 15833cd19..25ded143c 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,4 +1,5 @@ import json +import os import pstats import shutil import sys @@ -14,6 +15,8 @@ from scrapy.utils.test import get_testenv class CmdlineTest(unittest.TestCase): def setUp(self): self.env = get_testenv() + tests_path = Path(__file__).parent.parent + self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent) self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings" def _execute(self, *new_args, **kwargs): From 3f92882be4b35cf476f0c7c284e4fe1ba498e873 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Jun 2023 19:13:58 +0400 Subject: [PATCH 095/211] Fix a wrong merge. --- scrapy/extensions/feedexport.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index de3ed093c..7e93bc366 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -492,20 +492,6 @@ class FeedExporter: :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri, feed_options) - file = storage.open(spider) - if "postprocessing" in feed_options: - file = PostProcessingManager( - feed_options["postprocessing"], file, feed_options - ) - - exporter = self._get_exporter( - file=file, - format=feed_options["format"], - fields_to_export=feed_options["fields"], - encoding=feed_options["encoding"], - indent=feed_options["indent"], - **feed_options["item_export_kwargs"], - ) slot = FeedSlot( storage=storage, uri=uri, From 0adbd210acc9d1af75e42df2b17ee28e3fecdfc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 13 Jun 2023 19:34:26 +0400 Subject: [PATCH 096/211] Fix extra-deps-pinned tests. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 8a6693a49..5f8bf85f2 100644 --- a/tox.ini +++ b/tox.ini @@ -139,6 +139,7 @@ deps = install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:asyncio] commands = From 075ad6f196e7936edfd75b57d10b7313da7fe4f4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:34:18 +0400 Subject: [PATCH 097/211] Test, linter etc. fixes. --- docs/topics/addons.rst | 24 ++- scrapy/addons/__init__.py | 106 +++++----- scrapy/addons/builtins.py | 135 +++++++----- scrapy/utils/misc.py | 2 +- tests/test_addons/__init__.py | 329 ++++++++++++++--------------- tests/test_addons/addonmod.py | 2 + tests/test_addons/addons.py | 11 +- tests/test_addons/test_builtins.py | 18 +- tests/test_crawl.py | 2 +- tests/test_middleware.py | 4 +- 10 files changed, 329 insertions(+), 304 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 4dab15a2a..ba6e839a5 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (see Provide add-on interface through a module: -.. No idea why just using '::' doesn't work for this one .. code-block:: python - name = 'AddonModule' - version = '1.0' + name = "AddonModule" + version = "1.0" + class MyPipeline(object): - # ... + ... + class MyDownloaderMiddleware(object): - # ... + ... + def update_settings(config, settings): settings.set( - 'ITEM_PIPELINES', + "ITEM_PIPELINES", {MyPipeline(): 200}, - priority='addon', - } + priority="addon", + ) settings.set( - 'DOWNLOADER_MIDDLEWARES', + "DOWNLOADER_MIDDLEWARES", {MyDownloaderMiddleware(): 800}, - priority='addon', - } + priority="addon", + ) Forward to other add-ons depending on Python version:: diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 169c79eac..aad9ef789 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,10 +1,10 @@ -from collections import defaultdict, Mapping, OrderedDict -from inspect import isclass -import six import warnings +from collections import OrderedDict, defaultdict +from collections.abc import Mapping +from inspect import isclass -from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface +from pkg_resources import Distribution, Requirement, WorkingSet from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None """``dict`` of settings that will be exported via :meth:`export_basics`.""" @@ -83,8 +82,8 @@ class Addon(object): else: # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp - v = config.get('order', self.component_order) - settings[self.component_type].update({k: v}, 'addon') + v = config.get("order", self.component_order) + settings[self.component_type].update({k: v}, "addon") def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. @@ -95,8 +94,8 @@ class Addon(object): :param settings: Settings object into which to expose the basic settings :type settings: :class:`~scrapy.settings.Settings` """ - for setting, value in six.iteritems(self.basic_settings or {}): - settings.set(setting, value, 'addon') + for setting, value in (self.basic_settings or {}).items(): + settings.set(setting, value, "addon") def export_config(self, config, settings): """Export the add-on configuration, all keys in caps and with @@ -121,14 +120,13 @@ class Addon(object): prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v - for k, v in six.iteritems(self.config_mapping or {})} - for key, val in six.iteritems(conf): + conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} + for key, val in conf.items(): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + '_' + key).upper() - settings.set(key, val, 'addon') + key = (prefix + "_" + key).upper() + settings.set(key, val, "addon") def update_settings(self, config, settings): """Export both the basic settings and the add-on configuration. I.e., @@ -210,11 +208,11 @@ class AddonManager(Mapping): verifyObject(IAddon, addon) name = addon.name if name in self: - raise ValueError("Addon '{}' already loaded".format(name)) + raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} if name in self._disable_on_add: - self.configs[name]['_enabled'] = False + self.configs[name]["_enabled"] = False self._disable_on_add.remove(name) def remove(self, addon): @@ -229,7 +227,7 @@ class AddonManager(Mapping): """ if addon in self: del self[addon] - elif hasattr(addon, 'name') and addon.name in self: + elif hasattr(addon, "name") and addon.name in self: del self[addon.name] else: try: @@ -250,14 +248,14 @@ class AddonManager(Mapping): :param path: Python or file path to an add-on :type path: ``str`` """ - if isinstance(path, six.string_types): + if isinstance(path, str): try: obj = load_module_or_object(path) except NameError: - raise NameError("Could not find add-on '%s'" % path) + raise NameError(f"Could not find add-on '{path}'") else: obj = path - if hasattr(obj, '_addon'): + if hasattr(obj, "_addon"): obj = AddonManager.get_addon(obj._addon) return obj @@ -284,7 +282,7 @@ class AddonManager(Mapping): and values correspond to their configuration :type addonsdict: ``dict`` """ - for addonpath, addoncfg in six.iteritems(addonsdict): + for addonpath, addoncfg in addonsdict.items(): self.add(addonpath, addoncfg) def load_settings(self, settings): @@ -299,7 +297,7 @@ class AddonManager(Mapping): which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings['ADDONS']) + paths = build_component_list(settings["ADDONS"]) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): @@ -322,20 +320,23 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet('') + ws = WorkingSet("") def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get('scrapy', []): - raise ImportError("Component {} provided by multiple add-ons" - "".format(project_name)) + if project_name in ws.entry_keys.get("scrapy", []): + raise ImportError( + f"Component {project_name} provided by multiple add-ons" + ) else: - dist = Distribution(project_name=project_name, version=version, - **kwargs) - ws.add(dist, entry='scrapy') + dist = Distribution( + project_name=project_name, version=version, **kwargs + ) + ws.add(dist, entry="scrapy") + for name in self: ver = self[name].version add_dist(name, ver) - for provides_name in getattr(self[name], 'provides', []): + for provides_name in getattr(self[name], "provides", []): add_dist(provides_name, ver) # Collect all required and modified components @@ -345,8 +346,9 @@ class AddonManager(Mapping): for entry in getattr(self[name], attribute_name, []): attrs[entry].append(name) return attrs - modified = compile_attribute_dict('modifies') - required = compile_attribute_dict('requires') + + modified = compile_attribute_dict("modifies") + required = compile_attribute_dict("requires") req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: @@ -355,15 +357,16 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr) + f"Add-ons {required[reqstr] + modified[reqstr]} require" + f" or modify missing component {reqstr}" ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: - warnings.warn("Component '{}', required by add-ons {}, is modified " - "by add-ons {}".format(conflict, required[conflict], - modified[conflict])) + warnings.warn( + f"Component '{conflict}', required by add-ons {required[conflict]}," + f" is modified by add-ons {modified[conflict]}" + ) def disable(self, addon): """Disable an add-on, i.e. prevent its callbacks from being called. @@ -375,7 +378,7 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = False + self.configs[addon]["_enabled"] = False else: self._disable_on_add.append(addon) @@ -389,23 +392,23 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = True + self.configs[addon]["_enabled"] = True elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " - "enabled") + raise ValueError("Add-ons need to be added before they can be " "enabled") @property def disabled(self): """Names of disabled add-ons""" - return ([a for a in self if not self.configs[a].get('_enabled', True)] + - self._disable_on_add) + return [ + a for a in self if not self.configs[a].get("_enabled", True) + ] + self._disable_on_add @property def enabled(self): """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get('_enabled', True)] + return [a for a in self if self.configs[a].get("_enabled", True)] def _call_if_exists(self, obj, cbname, *args, **kwargs): if obj is None: @@ -418,9 +421,10 @@ class AddonManager(Mapping): cb(*args, **kwargs) def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get('_enabled', True): - self._call_if_exists(self[addonname], cbname, - self.configs[addonname], *args, **kwargs) + if self.configs[addonname].get("_enabled", True): + self._call_if_exists( + self[addonname], cbname, self.configs[addonname], *args, **kwargs + ) def update_addons(self): """Call ``update_addons()`` of all held add-ons. @@ -432,7 +436,7 @@ class AddonManager(Mapping): while called_addons != set(self): for name in set(self).difference(called_addons): called_addons.add(name) - self._call_addon(name, 'update_addons', self) + self._call_addon(name, "update_addons", self) def update_settings(self, settings): """Call ``update_settings()`` of all held add-ons. @@ -442,7 +446,7 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, 'update_settings', settings) + self._call_addon(name, "update_settings", settings) def check_configuration(self, crawler): """Call ``check_configuration()`` of all held add-ons. @@ -451,7 +455,7 @@ class AddonManager(Mapping): :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, 'check_configuration', crawler) + self._call_addon(name, "check_configuration", crawler) -from scrapy.addons.builtins import * +from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index 9babdeb6f..ea3afbf99 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -1,23 +1,44 @@ import scrapy from scrapy.addons import Addon -__all__ = ['make_builtin_addon', - - 'depth', 'httperror', 'offsite', 'referer', 'urllength', - - 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', - 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', - 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', - 'stats', 'useragent', - - 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', - 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', - 'statsmailer', 'telnetconsole', - ] +__all__ = [ + "make_builtin_addon", + "depth", + "httperror", + "offsite", + "referer", + "urllength", + "ajaxcrawl", + "chunked", + "cookies", + "defaultheaders", + "downloadtimeout", + "httpauth", + "httpcache", + "httpcompression", + "httpproxy", + "metarefresh", + "redirect", + "retry", + "robotstxt", + "stats", + "useragent", + "autothrottle", + "corestats", + "closespider", + "debugger", + "feedexport", + "logstats", + "memdebug", + "memusage", + "spiderstate", + "stacktracedump", + "statsmailer", + "telnetconsole", +] -def make_builtin_addon(addon_name, addon_default_config=None, - addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ @@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None, # SPIDER MIDDLEWARES -depth = make_builtin_addon('depth') +depth = make_builtin_addon("depth") -httperror = make_builtin_addon('httperror') +httperror = make_builtin_addon("httperror") -offsite = make_builtin_addon('offsite') +offsite = make_builtin_addon("offsite") -referer = make_builtin_addon('referer') +referer = make_builtin_addon("referer") -urllength = make_builtin_addon('urllength') +urllength = make_builtin_addon("urllength") # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) +ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) -chunked = make_builtin_addon('chunked') +chunked = make_builtin_addon("chunked") + +cookies = make_builtin_addon("cookies") + +defaultheaders = make_builtin_addon("defaultheaders") -cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} conf.update(config) - settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') + settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") + + defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon('downloadtimeout') -downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', - 'download_timeout': 'DOWNLOAD_TIMEOUT'} +downloadtimeout = make_builtin_addon("downloadtimeout") +downloadtimeout.config_mapping = { + "timeout": "DOWNLOAD_TIMEOUT", + "download_timeout": "DOWNLOAD_TIMEOUT", +} -httpauth = make_builtin_addon('httpauth') +httpauth = make_builtin_addon("httpauth") -httpcache = make_builtin_addon('httpcache', {'enabled': True}) +httpcache = make_builtin_addon("httpcache", {"enabled": True}) -httpcompression = make_builtin_addon('httpcompression') -httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} +httpcompression = make_builtin_addon("httpcompression") +httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} -httpproxy = make_builtin_addon('httpproxy') +httpproxy = make_builtin_addon("httpproxy") -metarefresh = make_builtin_addon('metarefresh') -metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} +metarefresh = make_builtin_addon("metarefresh") +metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} -redirect = make_builtin_addon('redirect') +redirect = make_builtin_addon("redirect") -retry = make_builtin_addon('retry') +retry = make_builtin_addon("retry") -robotstxt = make_builtin_addon('robotstxt', {'obey': True}) +robotstxt = make_builtin_addon("robotstxt", {"obey": True}) -stats = make_builtin_addon('stats') +stats = make_builtin_addon("stats") -useragent = make_builtin_addon('useragent') -useragent.config_mapping = {'user_agent': 'USER_AGENT'} +useragent = make_builtin_addon("useragent") +useragent.config_mapping = {"user_agent": "USER_AGENT"} # ITEM PIPELINES @@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) +autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) -corestats = make_builtin_addon('corestats') +corestats = make_builtin_addon("corestats") -closespider = make_builtin_addon('closespider') +closespider = make_builtin_addon("closespider") -debugger = make_builtin_addon('debugger') +debugger = make_builtin_addon("debugger") -feedexport = make_builtin_addon('feedexport') -feedexport.settings_prefix = 'FEED' +feedexport = make_builtin_addon("feedexport") +feedexport.settings_prefix = "FEED" -logstats = make_builtin_addon('logstats') +logstats = make_builtin_addon("logstats") -memdebug = make_builtin_addon('memdebug', {'enabled': True}) +memdebug = make_builtin_addon("memdebug", {"enabled": True}) -memusage = make_builtin_addon('memusage', {'enabled': True}) +memusage = make_builtin_addon("memusage", {"enabled": True}) -spiderstate = make_builtin_addon('spiderstate') +spiderstate = make_builtin_addon("spiderstate") -stacktracedump = make_builtin_addon('stacktracedump') +stacktracedump = make_builtin_addon("stacktracedump") -statsmailer = make_builtin_addon('statsmailer') +statsmailer = make_builtin_addon("statsmailer") -telnetconsole = make_builtin_addon('telnetconsole') +telnetconsole = make_builtin_addon("telnetconsole") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4bf7e7e66..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -85,7 +85,7 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - raise NameError("Could not load '%s'" % path) + raise NameError(f"Could not load '{path}'") def walk_modules(path): diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index a4e278fa5..741dd81cf 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,145 +1,143 @@ -from collections import OrderedDict import itertools -import os.path -import six -from tests import mock import unittest import warnings +from collections import OrderedDict +from unittest import mock from pkg_resources import VersionConflict -import zope.interface +from zope.interface import directlyProvides +from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from . import addons -from . import addonmod +from . import addonmod, addons class AddonTest(unittest.TestCase): - def setUp(self): self.rawaddon = Addon() class AddonWithAttributes(Addon): - name = 'Test' - version = '1.0' + name = "Test" + version = "1.0" + self.testaddon = AddonWithAttributes() def test_interface(self): # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, 'name')) - self.assertFalse(hasattr(self.rawaddon, 'version')) - self.assertRaises(BrokenImplementation, verifyObject, IAddon, - self.rawaddon) + self.assertFalse(hasattr(self.rawaddon, "name")) + self.assertFalse(hasattr(self.rawaddon, "version")) + self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), - 'DOWNLOAD_HANDLERS': BaseSettings()}, - 'default') + settings = BaseSettings( + {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, + "default", + ) self.testaddon.component_type = None self.testaddon.export_component({}, settings) - self.assertEqual(len(settings['ITEM_PIPELINES']), 0) - self.testaddon.component_type = 'ITEM_PIPELINES' - self.testaddon.component = 'test.component' + self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) + self.testaddon.component_type = "ITEM_PIPELINES" + self.testaddon.component = "test.component" self.testaddon.export_component({}, settings) - six.assertCountEqual(self, settings['ITEM_PIPELINES'], - ['test.component']) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) self.testaddon.component_order = 313 self.testaddon.export_component({}, settings) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) - self.testaddon.component_type = 'DOWNLOAD_HANDLERS' - self.testaddon.component_key = 'http' + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) + self.testaddon.component_type = "DOWNLOAD_HANDLERS" + self.testaddon.component_key = "http" self.testaddon.export_component({}, settings) - self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], - 'test.component') + self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") def test_export_basics(self): settings = BaseSettings() - self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} self.testaddon.export_basics(settings) - self.assertEqual(settings['TESTKEY'], 313) - self.assertEqual(settings['OTHERKEY'], True) - self.assertEqual(settings.getpriority('TESTKEY'), 15) + self.assertEqual(settings["TESTKEY"], 313) + self.assertEqual(settings["OTHERKEY"], True) + self.assertEqual(settings.getpriority("TESTKEY"), 15) def test_export_config(self): settings = BaseSettings() self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} - self.testaddon.default_config = {'key': 55, 'defaultkey': 100} - self.testaddon.export_config({'key': 313, 'OTHERKEY': True, - 'mapped_KEY': 99}, settings) - self.assertEqual(settings['TEST_KEY'], 313) - self.assertEqual(settings['TEST_DEFAULTKEY'], 100) - self.assertEqual(settings['TEST_OTHERKEY'], True) - self.assertNotIn('MAPPED_key', settings) - self.assertNotIn('MAPPED_KEY', settings) - self.assertEqual(settings['MAPPING_WORKED'], 99) - self.assertEqual(settings.getpriority('TEST_KEY'), 15) + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["TEST_KEY"], 313) + self.assertEqual(settings["TEST_DEFAULTKEY"], 100) + self.assertEqual(settings["TEST_OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("TEST_KEY"), 15) - self.testaddon.settings_prefix = 'PREF' - self.testaddon.export_config({'newkey': 99}, settings) - self.assertEqual(settings['PREF_NEWKEY'], 99) + self.testaddon.settings_prefix = "PREF" + self.testaddon.export_config({"newkey": 99}, settings) + self.assertEqual(settings["PREF_NEWKEY"], 99) - with mock.patch.object(settings, 'set') as mock_set: + with mock.patch.object(settings, "set") as mock_set: self.testaddon.settings_prefix = False - self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.testaddon.export_config({"thirdnewkey": 99}, settings) self.assertEqual(mock_set.call_count, 0) def test_update_settings(self): settings = BaseSettings() - settings.set('TEST_KEY1', 'default', priority='default') - settings.set('TEST_KEY2', 'project', priority='project') + settings.set("TEST_KEY1", "default", priority="default") + settings.set("TEST_KEY2", "project", priority="project") self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} - addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings['OTHERTEST_KEY'], 'addon') - self.assertEqual(settings['TEST_KEY1'], 'addon') - self.assertEqual(settings['TEST_KEY2'], 'project') - self.assertEqual(settings['TEST_KEY3'], 'addon') + self.assertEqual(settings["OTHERTEST_KEY"], "addon") + self.assertEqual(settings["TEST_KEY1"], "addon") + self.assertEqual(settings["TEST_KEY2"], "project") + self.assertEqual(settings["TEST_KEY3"], "addon") class AddonManagerTest(unittest.TestCase): - def setUp(self): self.manager = AddonManager() def test_add(self): manager = AddonManager() - manager.add(addonmod, {'key': 'val1'}) - manager.add('tests.test_addons.addons.GoodAddon') - six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + manager.add(addonmod, {"key": "val1"}) + manager.add("tests.test_addons.addons.GoodAddon") + self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["AddonModule"], ["key"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") self.assertRaises(ValueError, manager.add, addonmod) def test_add_dont_instantiate_providing_classes(self): class ProviderGoodAddon(addons.GoodAddon): pass - zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + + directlyProvides(ProviderGoodAddon, IAddon) manager = AddonManager() manager.add(ProviderGoodAddon) - self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + self.assertIs(manager["GoodAddon"], ProviderGoodAddon) def test_add_verifies(self): - brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') - self.assertRaises(zope.interface.exceptions.BrokenImplementation, - self.manager.add, - brokenaddon) + brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") + self.assertRaises( + BrokenImplementation, + self.manager.add, + brokenaddon, + ) def test_add_adds_missing_interface_declaration(self): class GoodAddonWithoutDeclaration(object): - name = 'GoodAddonWithoutDeclaration' - version = '1.0' + name = "GoodAddonWithoutDeclaration" + version = "1.0" + self.manager.add(GoodAddonWithoutDeclaration) def test_remove(self): @@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase): def test_gets_removed(removearg): manager.add(addonmod) - self.assertIn('AddonModule', manager) + self.assertIn("AddonModule", manager) manager.remove(removearg) - self.assertNotIn('AddonModule', manager) + self.assertNotIn("AddonModule", manager) - test_gets_removed('AddonModule') + test_gets_removed("AddonModule") test_gets_removed(addonmod) - test_gets_removed('tests.test_addons.addonmod') - self.assertRaises(KeyError, manager.remove, 'nonexistent') + test_gets_removed("tests.test_addons.addonmod") + self.assertRaises(KeyError, manager.remove, "nonexistent") self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") self.assertIs(loaded_addonmod, addonmod) - addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') goodaddon = self.manager.get_addon("tests.test_addons.addons") self.assertIsInstance(goodaddon, addons.GoodAddon) - self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") def test_get_addon_forward(self): class SomeCls(object): - _addon = 'tests.test_addons.addons.GoodAddon' + _addon = "tests.test_addons.addons.GoodAddon" + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) def test_get_addon_nested(self): - x = addons.GoodAddon('outer') - x._addon = addons.GoodAddon('middle') - x._addon._addon = addons.GoodAddon('inner') + x = addons.GoodAddon("outer") + x._addon = addons.GoodAddon("middle") + x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() getattr(manager, func)(*args, **kwargs) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - self.assertEqual(manager['AddonModule'], addonmod) - self.assertIn('key', manager.configs['AddonModule']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") addonsdict = { - "tests.test_addons.addonmod": {'key': 'val1'}, - 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, - } - _test_load_method('load_dict', addonsdict) + "tests.test_addons.addonmod": {"key": "val1"}, + "tests.test_addons.addons.GoodAddon": {"key": "val2"}, + } + _test_load_method("load_dict", addonsdict) settings = BaseSettings() - settings.set('ADDONS', {"tests.test_addons.addonmod": 0, - 'tests.test_addons.addons.GoodAddon': 0}) - settings.set('ADDONMODULE', {'key': 'val1'}) - settings.set('GOODADDON', {'key': 'val2'}) - _test_load_method('load_settings', settings) + settings.set( + "ADDONS", + {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, + ) + settings.set("ADDONMODULE", {"key": "val1"}) + settings.set("GOODADDON", {"key": "val2"}) + _test_load_method("load_settings", settings) def test_load_dict_load_settings_order(self): def _test_load_method(expected_order, func, *args, **kwargs): @@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, 'load_dict', addonsdict) - settings = BaseSettings({ - 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} - }) - _test_load_method(expected_order, 'load_settings', settings) + _test_load_method(expected_order, "load_dict", addonsdict) + settings = BaseSettings( + {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + ) + _test_load_method(expected_order, "load_settings", settings) def test_enabled_disabled(self): manager = AddonManager() - manager.add(addons.GoodAddon('FirstAddon')) - manager.add(addons.GoodAddon('SecondAddon')) - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.add(addons.GoodAddon("FirstAddon")) + manager.add(addons.GoodAddon("SecondAddon")) + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) - manager.disable('FirstAddon') - self.assertEqual(manager.enabled, ['SecondAddon']) - self.assertEqual(manager.disabled, ['FirstAddon']) - manager.enable('FirstAddon') - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.disable("FirstAddon") + self.assertEqual(manager.enabled, ["SecondAddon"]) + self.assertEqual(manager.disabled, ["FirstAddon"]) + manager.enable("FirstAddon") + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) def test_enable_before_add(self): manager = AddonManager() - self.assertRaises(ValueError, manager.enable, 'FirstAddon') - manager.disable('FirstAddon') - manager.enable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertIn('FirstAddon', manager.enabled) + self.assertRaises(ValueError, manager.enable, "FirstAddon") + manager.disable("FirstAddon") + manager.enable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertIn("FirstAddon", manager.enabled) def test_disable_before_add(self): manager = AddonManager() - manager.disable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertEqual(manager.disabled, ['FirstAddon']) + manager.disable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertEqual(manager.disabled, ["FirstAddon"]) def test_callbacks(self): - first_addon = addons.GoodAddon('FirstAddon') - second_addon = addons.GoodAddon('SecondAddon') + first_addon = addons.GoodAddon("FirstAddon") + second_addon = addons.GoodAddon("SecondAddon") manager = AddonManager() - manager.add(first_addon, {'test': 'first'}) - manager.add(second_addon, {'test': 'second'}) + manager.add(first_addon, {"test": "first"}) + manager.add(second_addon, {"test": "second"}) crawler = mock.create_autospec(Crawler) settings = BaseSettings() - with mock.patch.object(first_addon, 'update_addons') as ua_first, \ - mock.patch.object(second_addon, 'update_addons') as ua_second, \ - mock.patch.object(first_addon, 'update_settings') as us_first, \ - mock.patch.object(second_addon, 'update_settings') as us_second, \ - mock.patch.object(first_addon, 'check_configuration') as cc_first, \ - mock.patch.object(second_addon, 'check_configuration') as cc_second: + with mock.patch.object( + first_addon, "update_addons" + ) as ua_first, mock.patch.object( + second_addon, "update_addons" + ) as ua_second, mock.patch.object( + first_addon, "update_settings" + ) as us_first, mock.patch.object( + second_addon, "update_settings" + ) as us_second, mock.patch.object( + first_addon, "check_configuration" + ) as cc_first, mock.patch.object( + second_addon, "check_configuration" + ) as cc_second: manager.update_addons() - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs['FirstAddon'], - settings) - us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) + us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs['FirstAddon'], - crawler) - cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) + cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase): us_first.reset_mock() us_second.reset_mock() - manager.disable('FirstAddon') + manager.disable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 0) - manager.enable('FirstAddon') + manager.enable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) @@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase): # This will become relevant when we let spiders implement the add-on # interface and should be replaced with a test where # AddonManager.spidercls = None then. - manager._call_if_exists(None, 'irrelevant') + manager._call_if_exists(None, "irrelevant") def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): - name = 'AddedAddon' + name = "AddedAddon" class FirstAddon(addons.GoodAddon): - name = 'FirstAddon' + name = "FirstAddon" def update_addons(self, config, addons): addons.add(AddedAddon()) manager = AddonManager() first_addon = FirstAddon() - with mock.patch.object(first_addon, 'update_addons', - wraps=first_addon.update_addons) as ua_first, \ - mock.patch.object(AddedAddon, 'update_addons') as ua_added: - manager.add(first_addon, {'non-empty': 'dict'}) + with mock.patch.object( + first_addon, "update_addons", wraps=first_addon.update_addons + ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: + manager.add(first_addon, {"non-empty": "dict"}) manager.update_addons() - six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_added.assert_called_once_with(manager.configs['AddedAddon'], - manager) + self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) def test_check_dependency_clashes_attributes(self): provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ('test', ) + provides.provides = ("test",) provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ('test', ) + provides2.provides = ("test",) requires = addons.GoodAddon("RequiresAddon") - requires.requires = ('test', ) + requires.requires = ("test",) requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ('ProvidesAddon', ) + requires_name.requires = ("ProvidesAddon",) requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ('test>=2.0', ) + requires_newer.requires = ("test>=2.0",) modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ('test', ) + modifies.modifies = ("test",) def check_with(*addons): manager = AddonManager() diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index 8ecf4b81d..c59f6737f 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -9,8 +9,10 @@ FROM = "test_addons.addonmod" name = "AddonModule" version = "1.0" + def update_settings(config, settings): pass + def check_configuration(config, crawler): pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index f3442b192..4adb9fe8f 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,18 +1,16 @@ import zope.interface -from scrapy.addons import Addon from scrapy.interfaces import IAddon class Addon(object): - FROM = 'test_addons.addons' + FROM = "test_addons.addons" @zope.interface.declarations.implementer(IAddon) class GoodAddon(object): - - name = 'GoodAddon' - version = '1.0' + name = "GoodAddon" + version = "1.0" def __init__(self, name=None, version=None): if name is not None: @@ -32,8 +30,7 @@ class GoodAddon(object): @zope.interface.declarations.implementer(IAddon) class BrokenAddon(object): - - name = 'BrokenAddon' + name = "BrokenAddon" # No version diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index c89876950..1050cbbed 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -7,19 +7,17 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon('httpcache', {'enabled': True}) - self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.default_config, {'enabled': True}) + httpcache = make_builtin_addon("httpcache", {"enabled": True}) + self.assertEqual(httpcache.name, "httpcache") + self.assertEqual(httpcache.default_config, {"enabled": True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') - self.assertEqual(httpcache.version, '99.9') + httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") + self.assertEqual(httpcache.version, "99.9") def test_defaultheaders_export_config(self): settings = Settings() dh = scrapy.addons.defaultheaders() - dh.export_config({'X-Test-Header': 'val'}, settings) - self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) - self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], - 'val') + dh.export_config({"X-Test-Header": "val"}, settings) + self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) + self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index e353f80cb..1e97863b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -419,7 +419,7 @@ with multiples lines addonmgr = AddonManager() addonmgr.add(FailedCheckAddon()) - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr # Doesn't work in 'precise' test environment: # with self.assertRaises(ValueError): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 7ddf952f7..fd57d846e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase): def test_instances_from_settings(self): settings = Settings() - myM3 = M3() class InstanceTestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2, myM3] + return ["tests.test_middleware.M1", M2] mwman = InstanceTestMiddlewareManager.from_settings(settings) self.assertIsInstance(mwman.middlewares[0], M1) self.assertIsInstance(mwman.middlewares[1], M2) - self.assertIs(mwman.middlewares[2], myM3) From 55ac26228b853aec6d543bfaf3ab85ebbf5ab002 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:50:43 +0400 Subject: [PATCH 098/211] Remove builtin addons. --- scrapy/{addons/__init__.py => addons.py} | 3 - scrapy/addons/builtins.py | 146 ----------------------- sep/sep-021.rst | 25 ---- tests/test_addons/test_builtins.py | 23 ---- 4 files changed, 197 deletions(-) rename scrapy/{addons/__init__.py => addons.py} (99%) delete mode 100644 scrapy/addons/builtins.py delete mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons.py similarity index 99% rename from scrapy/addons/__init__.py rename to scrapy/addons.py index aad9ef789..c9b873039 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons.py @@ -456,6 +456,3 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, "check_configuration", crawler) - - -from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py deleted file mode 100644 index ea3afbf99..000000000 --- a/scrapy/addons/builtins.py +++ /dev/null @@ -1,146 +0,0 @@ -import scrapy -from scrapy.addons import Addon - -__all__ = [ - "make_builtin_addon", - "depth", - "httperror", - "offsite", - "referer", - "urllength", - "ajaxcrawl", - "chunked", - "cookies", - "defaultheaders", - "downloadtimeout", - "httpauth", - "httpcache", - "httpcompression", - "httpproxy", - "metarefresh", - "redirect", - "retry", - "robotstxt", - "stats", - "useragent", - "autothrottle", - "corestats", - "closespider", - "debugger", - "feedexport", - "logstats", - "memdebug", - "memusage", - "spiderstate", - "stacktracedump", - "statsmailer", - "telnetconsole", -] - - -def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): - class ThisAddon(Addon): - name = addon_name - version = addon_version or scrapy.__version__ - default_config = addon_default_config or {} - - return ThisAddon - - -# XXX: Below are CLASSES that have lowercase names. This is in line with the -# original SEP-021 but violates PEP8. -# We might consider prepending all built-in addon names with scrapy_ or similar -# to reduce the chance of name clashes. - -# SPIDER MIDDLEWARES - -depth = make_builtin_addon("depth") - -httperror = make_builtin_addon("httperror") - -offsite = make_builtin_addon("offsite") - -referer = make_builtin_addon("referer") - -urllength = make_builtin_addon("urllength") - - -# DOWNLOADER MIDDLEWARES - -ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) - -chunked = make_builtin_addon("chunked") - -cookies = make_builtin_addon("cookies") - -defaultheaders = make_builtin_addon("defaultheaders") - - -# Assume every config entry is a header -def defaultheaders_export_config(self, config, settings): - conf = self.default_config or {} - conf.update(config) - settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") - - -defaultheaders.export_config = defaultheaders_export_config - -downloadtimeout = make_builtin_addon("downloadtimeout") -downloadtimeout.config_mapping = { - "timeout": "DOWNLOAD_TIMEOUT", - "download_timeout": "DOWNLOAD_TIMEOUT", -} - -httpauth = make_builtin_addon("httpauth") - -httpcache = make_builtin_addon("httpcache", {"enabled": True}) - -httpcompression = make_builtin_addon("httpcompression") -httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} - -httpproxy = make_builtin_addon("httpproxy") - -metarefresh = make_builtin_addon("metarefresh") -metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} - -redirect = make_builtin_addon("redirect") - -retry = make_builtin_addon("retry") - -robotstxt = make_builtin_addon("robotstxt", {"obey": True}) - -stats = make_builtin_addon("stats") - -useragent = make_builtin_addon("useragent") -useragent.config_mapping = {"user_agent": "USER_AGENT"} - - -# ITEM PIPELINES - - -# EXTENSIONS - -autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) - -corestats = make_builtin_addon("corestats") - -closespider = make_builtin_addon("closespider") - -debugger = make_builtin_addon("debugger") - -feedexport = make_builtin_addon("feedexport") -feedexport.settings_prefix = "FEED" - -logstats = make_builtin_addon("logstats") - -memdebug = make_builtin_addon("memdebug", {"enabled": True}) - -memusage = make_builtin_addon("memusage", {"enabled": True}) - -spiderstate = make_builtin_addon("spiderstate") - -stacktracedump = make_builtin_addon("stacktracedump") - -statsmailer = make_builtin_addon("statsmailer") - -telnetconsole = make_builtin_addon("telnetconsole") diff --git a/sep/sep-021.rst b/sep/sep-021.rst index cb1701014..47cba004c 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` submodule). If the object found has an ``_addon`` attribute, that attribute will be treated as the found add-on. This allows, for example, to change the add-on based on the Python version. - -Updating existing extensions ----------------------------- - -An ``Addon`` class is introduced that add-on developers may or may not subclass -depending on how much of the 'default functionality' they want. Naturally, it -does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` -exposes the add-on configuration into the global settings namespace with an -appropriate name, e.g. this section from ``scrapy.cfg``:: - - [httpcache] - dir = /some/dir - -would expose ``HTTPCACHE_DIR``. - -Add-on modules will be written for all built-in extensions and placed in -``scrapy.addons``. For many default Scrapy components, it will be sufficient to -create a subclass of ``Addon`` with minor or no method modifications. The -component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). - -Later, the global settings namespace could be cleaned up in a backwards --incompatible fashion by deprecating support for the global setting names, e.g. -``HTTPCACHE_DIR``, and instead instantiate the components with the add-on -configuration in ``update_settings()``. - diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py deleted file mode 100644 index 1050cbbed..000000000 --- a/tests/test_addons/test_builtins.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -import scrapy -import scrapy.addons -from scrapy.addons.builtins import make_builtin_addon -from scrapy.settings import Settings - - -class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon("httpcache", {"enabled": True}) - self.assertEqual(httpcache.name, "httpcache") - self.assertEqual(httpcache.default_config, {"enabled": True}) - self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") - self.assertEqual(httpcache.version, "99.9") - - def test_defaultheaders_export_config(self): - settings = Settings() - dh = scrapy.addons.defaultheaders() - dh.export_config({"X-Test-Header": "val"}, settings) - self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) - self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") From fdbc141b23b95f8f744712f86c28a4d4d4d8c52f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:22:51 +0400 Subject: [PATCH 099/211] Replace pkg_resources with packaging. --- scrapy/addons.py | 28 ++++++++++++---------------- tests/test_addons/__init__.py | 3 +-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index c9b873039..8f0f39889 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -2,9 +2,11 @@ import warnings from collections import OrderedDict, defaultdict from collections.abc import Mapping from inspect import isclass +from typing import Dict import zope.interface -from pkg_resources import Distribution, Requirement, WorkingSet +from packaging.requirements import Requirement +from packaging.version import Version from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -303,7 +305,7 @@ class AddonManager(Mapping): for a, c in zip(addons, configs): self.add(a, c) - def check_dependency_clashes(self): + def check_dependency_clashes(self) -> None: """Check for incompatibilities in add-on dependencies. Add-ons can provide information about their dependencies in their @@ -320,24 +322,20 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet("") + versions: Dict[str, Version] = {} - def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get("scrapy", []): + def add_version(project_name, version): + if project_name in versions: raise ImportError( f"Component {project_name} provided by multiple add-ons" ) - else: - dist = Distribution( - project_name=project_name, version=version, **kwargs - ) - ws.add(dist, entry="scrapy") + versions[project_name] = Version(version) for name in self: ver = self[name].version - add_dist(name, ver) + add_version(name, ver) for provides_name in getattr(self[name], "provides", []): - add_dist(provides_name, ver) + add_version(provides_name, ver) # Collect all required and modified components def compile_attribute_dict(attribute_name): @@ -352,10 +350,8 @@ class AddonManager(Mapping): req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: - req = Requirement.parse(reqstr) - # May raise VersionConflict. Do we want to catch it and raise - # our own exception or is it helpful enough? - if ws.find(req) is None: + req = Requirement(reqstr) + if req.name not in versions or versions[req.name] not in req.specifier: raise ImportError( f"Add-ons {required[reqstr] + modified[reqstr]} require" f" or modify missing component {reqstr}" diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 741dd81cf..451ffed0f 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -4,7 +4,6 @@ import warnings from collections import OrderedDict from unittest import mock -from pkg_resources import VersionConflict from zope.interface import directlyProvides from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject @@ -347,7 +346,7 @@ class AddonManagerTest(unittest.TestCase): self.assertRaises(ImportError, check_with, requires) self.assertRaises(ImportError, check_with, modifies) self.assertRaises(ImportError, check_with, provides, provides2) - self.assertRaises(VersionConflict, check_with, provides, requires_newer) + self.assertRaises(ImportError, check_with, provides, requires_newer) with warnings.catch_warnings(record=True) as w: check_with(provides, modifies) check_with(provides) From 7ebb8256f029b2027a3603685569378ccce73070 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:52:11 +0400 Subject: [PATCH 100/211] Some cleanup. --- docs/topics/addons.rst | 3 --- scrapy/addons.py | 6 +++--- tests/test_addons/addonmod.py | 6 ++---- tests/test_addons/addons.py | 10 +++------- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index ba6e839a5..b86058dc4 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -43,7 +43,6 @@ case with one requiring no configuration) are enabled/configured in a project's INSTALLED_ADDONS = ( 'httpcache', 'path.to.some.addon', - 'path/to/other/addon.py', ) HTTPCACHE = { @@ -73,8 +72,6 @@ dictionary keys. I.e., the configuration from above would look like this: [addon:path.to.some.addon] some_config = true - [addon:path/to/other/addon.py] - Enabling and configuring add-ons within Python code --------------------------------------------------- diff --git a/scrapy/addons.py b/scrapy/addons.py index 8f0f39889..43a36e5e5 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -156,10 +156,10 @@ class AddonManager(Mapping): addons = AddonManager() # ... load some add-ons here - print addons.enabled # prints names of all enabled add-ons - print addons['TestAddon'].version # prints version of add-on with name + print(addons.enabled) # prints names of all enabled add-ons + print(addons['TestAddon'].version) # prints version of add-on with name # 'TestAddon' - print addons.configs['TestAddon'] # prints configuration of 'TestAddon' + print(addons.configs['TestAddon']) # prints configuration of 'TestAddon' """ diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index c59f6737f..092c3c0eb 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -1,10 +1,8 @@ -import zope.interface +from zope.interface import moduleProvides from scrapy.interfaces import IAddon -zope.interface.moduleProvides(IAddon) - -FROM = "test_addons.addonmod" +moduleProvides(IAddon) name = "AddonModule" version = "1.0" diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index 4adb9fe8f..d878f37ea 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,13 +1,9 @@ -import zope.interface +from zope.interface import implementer from scrapy.interfaces import IAddon -class Addon(object): - FROM = "test_addons.addons" - - -@zope.interface.declarations.implementer(IAddon) +@implementer(IAddon) class GoodAddon(object): name = "GoodAddon" version = "1.0" @@ -28,7 +24,7 @@ class GoodAddon(object): pass -@zope.interface.declarations.implementer(IAddon) +@implementer(IAddon) class BrokenAddon(object): name = "BrokenAddon" # No version From 282fe3dd4fd4a87b27f8e9793760f877778d2914 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:52:25 +0400 Subject: [PATCH 101/211] Drop the SEP as we decided we don't want to update it. --- sep/sep-021.rst | 324 ------------------------------------------------ 1 file changed, 324 deletions(-) delete mode 100644 sep/sep-021.rst diff --git a/sep/sep-021.rst b/sep/sep-021.rst deleted file mode 100644 index 47cba004c..000000000 --- a/sep/sep-021.rst +++ /dev/null @@ -1,324 +0,0 @@ -======= =================== -SEP 21 -Title Add-ons -Author Pablo Hoffman -Created 2014-02-14 -Status Draft -======= =================== - -================ -SEP-021: Add-ons -================ - -This proposal introduces add-ons, a unified way to manage Scrapy extensions, -middlewares and pipelines. - -Scrapy currently supports many hooks and mechanisms for extending its -functionality, but no single entry point for enabling and configuring them. -Instead, the hooks are spread over: - -* Spider middlewares (``SPIDER_MIDDLEWARES``) -* Downloader middlewares (``DOWNLOADER_MIDDLEWARES``) -* Downloader handlers (``DOWNLOADER_HANDLERS``) -* Item pipelines (``ITEM_PIPELINES``) -* Feed exporters and storages (``FEED_EXPORTERS``, ``FEED_STORAGES``) -* Overridable components (``DUPEFILTER_CLASS``, ``STATS_CLASS``, - ``SCHEDULER``, ``SPIDER_MANAGER_CLASS``, ``ITEM_PROCESSOR``, etc.) -* Generic extensions (``EXTENSIONS``) -* CLI commands (``COMMANDS_MODULE``) - -This approach has several shortfalls: - -* Enabling an extension often requires modifying many settings, often in a - coordinated way, which is complex and error prone. -* Extension developers have little control over ensuring their library - dependencies and configuration requirements are met, especially since most - extensions never 'see' a fully-configured crawler before it starts running. -* The user is burdened with supervising potential interplay of extensions, - especially non-included ones, ranging from setting name clashes to mutually - excluding dependencies/configuration requirements. - -*Add-ons* search to remedy these shortcomings by enhancing Scrapy's extension -management, making it easy-to-use and transparent for users while giving more -configuration control to developers. - - -Design goals and non-goals -========================== - -Goals: - -* simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a configuration file -* backward compatibility with enabling extension the "old way" (i.e. modifying - settings directly) - -Non-goals: - -* a way to publish, distribute or discover extensions (use pypi for that) - - -User experience: managing add-ons -================================= - -Add-ons are enabled and configured either via Scrapy's settings, or (for add-ons -not bound to any project) in ``scrapy.cfg``. - -In the settings, add-ons can be enabled by adding either their name (for -built-in add-ons), their Python path, or their file path, to a -``INSTALLED_ADDONS`` setting. If necessary, each add-on can be configured by -providing a dictionary-valued setting with the uppercase add-on name. For -example, to enable and configure the built-in ``httpcache`` add-on and enable -(without configuring) two custom add-ons, one via Python path and one via file -path, add these entries to your settings module:: - - INSTALLED_ADDONS = ( - 'httpcache', - 'mymodule.filters.myfilter', - 'mymodule/filters/otherfilter.py', - ) - - HTTPCACHE = { - 'ignore_http_codes': [404, 503], - } - -In ``scrapy.cfg``, add-ons are enabled and configured with one section per -add-on. The section names correspond to the entries of ``INSTALLED_ADDONS``. -The configuration from above could look like this:: - - [addon:httpcache] - ignore_http_codes = 404,503 - - [addon:mymodule.filters.myfilter] - - [addon:mymodule/filters/otherfilter.py] - - -Developer experience: writing add-ons -===================================== - -Add-ons are (any) Python *objects* that implement Scrapy's *add-on interface*. -The interface is enforced through ``zope.interface``. This leaves the choice of -Python object up the developer. Examples: - -* for a small pipeline, the add-on interface could be implemented in the same - class that also implements the ``open/close_spider`` and ``process_item`` - callbacks -* for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module - -The absolute minimum interface consists of just two attributes: - -* ``NAME``: string with add-on name -* ``VERSION``: PEP-440 style version string - -To be any useful, an add-on should implement at least one of the following -callback methods: - -* ``update_addons()``: adds and configures other add-ons -* ``update_settings()``: sets configuration (such as default values for this - add-on and required settings for other extensions) and enables needed - components. -* ``check_configuration()``: receives the fully-initialized ``Crawler`` - instance before it starts running, performs additional dependency and - configuration requirement checks - -Additionally, an add-on may (and should, where appropriate) provide one or more -variables that can be used for automated detection of possible dependency -clashes: - -* ``REQUIRES``: list of built-in or custom components required by this add-on, - as PEP-440 strings -* ``MODIFIES``: list of components whose functionality is affected or replaced - by this add-on (a custom HTTP cache should list ``httpcache`` here) -* ``PROVIDES``: list of components provided by this add-on (e.g. ``mongodb`` - for an extension that provides generic read/write access to a MongoDB - database, releasing other components from having to provide their own - database access methods) - -update_addons() ------------------ - -Called: -~~~~~~~ - -Shortly after initialisation of the ``Crawler`` object. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``addons``: the add-on manager, providing methods to add and configure add-ons - -Purpose: -~~~~~~~~ - -* Configure and enable related add-ons, useful for 'umbrella add-ons' which - chain-load other add-ons based on the configuration - -Examples: -~~~~~~~~~ - -.. code-block:: python - - def update_addons(config, addons): - if "httpcache" not in addons.enabled: - addons.add("httpcache", {"expiration_secs": 60}) - -or: - -.. code-block:: python - - def update_addons(config, addons): - if "otheraddon" in addons.enabled: - addons.configs["otheraddon"]["some_config_name"] = True - -update_settings() ------------------ - -Called: -~~~~~~~ - -Directly after the ``update_addons()`` callback of all add-ons has been called. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``settings``: the crawler's ``Settings`` instance containing all project - settings - -Purpose: -~~~~~~~~ - -* Modify ``settings`` to enable required components -* Expose some add-on specific configuration (``config``) into the global - settings namespace (``settings``) if necessary -* Raise exception if components can not be properly configured (e.g. on missing - dependencies); Scrapy will print this exception *and exit* (making users - explicitly acknowledge that the add-on does not work by forcing them to - disable it). - -Side note: -~~~~~~~~~~ - -The ``MiddlewareManager.from_settings()`` method will receive a slight -modification to allow directly placing Python objects instead of class paths -in the middleware dict settings. This way, add-ons can place already -instantiated components into the settings. This allows keeping configuration -as local to components as possible and avoids cluttering up the global -settings namespace. Furthermore, it allows reusing components (e.g. using -two instances of the same mongodb pipeline to write to different locations). - -Examples: -~~~~~~~~~ - -:: - - def update_settings(config, settings): - # Don't care where this module is located - settings.set['DOWNLADER_MIDDLEWARES']({ - __name__ + '.downloadermw.coolmw': 900, - }) - - # Instantiate components to not expose settings into - # the global namespace - from .pipelines import MySQLPipeline - mysqlpl = MySQLPipeline(password = config['password']) - settings.set['ITEM_PIPELINES']({ - mysqlpl: 200, - }) - -or:: - - def update_settings(config, settings): - # Assuming this class also has a process_item() method - settings.set['ITEM_PIPELINES']({ - self: 200, - }) - -or:: - - def update_settings(config, settings): - try: - import boto - except ImportError: - raise RuntimeError("boto library is required") - -check_configuration() ---------------------- - -Called: -~~~~~~~ - -Shortly before the crawler starts crawling. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``crawler``: fully-initialized ``Crawler`` object, ready to start crawling - -Purpose: -~~~~~~~~ - -* Perform post-initialization checks like making sure the extension and its - dependencies were configured properly. -* Raise exception if a critical check failed; Scrapy will print this exception - *and exit* (see ``update_settings()`` purpose for rationale on this). - -Examples: -~~~~~~~~~ - -:: - - def check_configuration(config, crawler): - if 'some.other.addon' not in crawler.addons.enabled: - raise RuntimeError("Some other add-on required to use this add-on") - - -Implementation -============== - -A new core component, the *add-on manager*, is introduced to Scrapy. It -facilitates loading add-ons, gathering and providing information on them, -calling their callbacks at appropriate times, and performing basic checks for -dependency and configuration clashes. - -Layout ------- - -A new ``AddonManager`` class is introduced, providing methods to - -* add and remove add-ons, -* search for add-ons by name -* read enabled add-ons and their configurations from the settings module and - from ``settings.py``, -* enable and disable add-ons -* check for possible dependency incompatibilites by inspecting the collected - ``REQUIRES``, ``MODIFIES`` and ``PROVIDES`` add-on variables -* call the add-on callbacks - -Integration into start-up process ---------------------------------- - -The settings used to crawl are not complete until the spider-specific settings -have been loaded in ``Crawler.__init__()``. Add-on management follows this -approach and only starts loading add-ons when the crawler is initialised. - -Instantiation and the calls ``update_addons()`` and ``update_settings()`` happen -in ``Crawler.__init__()``. The final checks (i.e. the callback to -``check_configuration()``) is coded into the ``Crawler.crawl()`` method after -creating the engine. - -Finding add-ons ---------------- - -Add-on localisation is governed by the add-on paths given in -``INSTALLED_ADDONS`` (or by the section names if using ``scrapy.cfg``). If -nothing is found at the given path, it is tried again with ``addons.`` -prepended (i.e. pointing to the project's ``addons`` folder or module), then -with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` -submodule). If the object found has an ``_addon`` attribute, that attribute -will be treated as the found add-on. This allows, for example, to change the -add-on based on the Python version. From 22bd0d9a796cca980f3d2c2d951d0f3960ab8a18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:58:11 +0400 Subject: [PATCH 102/211] Fix docs build. --- docs/topics/addons.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index b86058dc4..198c48bcc 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -236,6 +236,7 @@ convenience functions by overwriting :meth:`~scrapy.addons.Addon.update_settings`. .. module:: scrapy.addons + :noindex: .. autoclass:: Addon :members: From fdfab17438647b10a7cb74b808ddf4acfd89c6ab Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:00:18 +0400 Subject: [PATCH 103/211] Fix docs for the renamed ADDONS setting. --- docs/topics/addons.rst | 18 +++++++++--------- docs/topics/settings.rst | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 198c48bcc..25d7da50b 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -20,15 +20,15 @@ only then, the add-on manager will read a list of enabled add-ons and their configurations from your settings. There are two places where you can provide the paths to add-ons you want to enable: -* the ``INSTALLED_ADDONS`` setting, and +* the ``ADDONS`` setting, and * the ``scrapy.cfg`` file. As Scrapy settings can be modified from many places, e.g. in a project's ``settings.py``, in a Spider's ``custom_settings`` attribute, or from the -command line, using the ``INSTALLED_ADDONS`` setting is the preferred way to +command line, using the ``ADDONS`` setting is the preferred way to manage add-ons. -The ``INSTALLED_ADDONS`` setting a tuple in which every item is a path to an +The ``ADDONS`` setting a tuple in which every item is a path to an add-on. The path can be both a Python or a file path. While more precise, it is not necessary to specify the full add-on Python path if it is either built into Scrapy or lives in your project's ``addons`` submodule. @@ -40,7 +40,7 @@ This is an example where an internal add-on and two third-party add-ons (in this case with one requiring no configuration) are enabled/configured in a project's ``settings.py``:: - INSTALLED_ADDONS = ( + ADDONS = ( 'httpcache', 'path.to.some.addon', ) @@ -79,7 +79,7 @@ Enabling and configuring add-ons within Python code The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is -populated, changing the ``INSTALLED_ADDONS`` setting or any of the add-on +populated, changing the ``ADDONS`` setting or any of the add-on configuration dictionary settings will have no effect. If you want to enable, disable, or configure add-ons in Python code, for example @@ -202,7 +202,7 @@ specify :pep:`440`-style information about required versions. Examples:: requires = ['otheraddon >= 2.0', 'yetanotheraddon'] The Python object or module that is pointed to by an add-on path (e.g. given in -the ``INSTALLED_ADDONS`` setting, or given to +the ``ADDONS`` setting, or given to :meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be either an add-on or another add-on path. @@ -401,7 +401,7 @@ of placing this in your ``settings.py``:: you can also use the add-on framework:: - INSTALLED_ADDONS = ( + ADDONS = ( # ..., 'httpcache', ) @@ -412,9 +412,9 @@ you can also use the add-on framework:: } Note that you *must* enable built-in addons by placing them in your -``INSTALLED_ADDONS`` setting before you can use them for configuring built-in +``ADDONS`` setting before you can use them for configuring built-in components. I.e., configuring the ``HTTPCACHE`` setting will have no effect -when ``httpcache`` is not listed in ``INSTALLED_ADDONS``. +when ``httpcache`` is not listed in ``ADDONS``. In general, the add-on names match the lowercase name of the component, with its type suffix removed (i.e. the add-on configuring the diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9e8c30efe..aa09abbd5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -201,6 +201,16 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. +.. setting:: ADDONS + +ADDONS +------ + +Default: ``()`` + +A tuple containing paths to the add-ons enabled in your project. For more +information, see :ref:`topics-addons`. + .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -964,16 +974,6 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. -.. setting:: INSTALLED_ADDONS - -INSTALLED_ADDONS ----------------- - -Default: ``()`` - -A tuple containing paths to the add-ons enabled in your project. For more -information, see :ref:`topics-addons`. - .. setting:: FEED_TEMPDIR FEED_TEMPDIR From 5e76464fbf1338bb791b4ae2ad97dd33f72dd0a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:01:41 +0400 Subject: [PATCH 104/211] Fix a merge error. --- scrapy/interfaces.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index d8e5f9866..b8aa77ced 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -17,11 +17,6 @@ class ISpiderLoader(Interface): """Return the list of spiders names that can handle the given request""" -# ISpiderManager is deprecated, don't use it! -# An alias is kept for backwards compatibility. -ISpiderManager = ISpiderLoader - - class IAddon(Interface): """Scrapy add-on""" From 815af431209686018b1bc2cbb80fd946cfe93614 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:11:14 +0400 Subject: [PATCH 105/211] Remove load_module_or_object. --- scrapy/addons.py | 6 +++--- scrapy/utils/misc.py | 16 ---------------- tests/test_utils_misc/__init__.py | 13 ++++++------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 43a36e5e5..153de66d2 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -11,7 +11,7 @@ from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_module_or_object +from scrapy.utils.misc import load_object @zope.interface.implementer(IAddon) @@ -252,8 +252,8 @@ class AddonManager(Mapping): """ if isinstance(path, str): try: - obj = load_module_or_object(path) - except NameError: + obj = load_object(path) + except (ValueError, NameError, ImportError): raise NameError(f"Could not find add-on '{path}'") else: obj = path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8577cce02..d861c9ab6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -72,22 +72,6 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def load_module_or_object(path): - """Load python module or (non-module) object from given path. - - Path can be both a Python or a file path. - """ - try: - return import_module(path) - except ImportError: - pass - try: - return load_object(path) - except (ValueError, NameError, ImportError): - pass - raise NameError(f"Could not load '{path}'") - - def walk_modules(path): """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 7932ca04c..4f6e0d02c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,7 +8,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, - load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -31,17 +30,17 @@ class UtilsMiscTestCase(unittest.TestCase): obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) + def test_load_object_module(self): + testmod = load_object(__name__ + ".testmod") + self.assertTrue(hasattr(testmod, "TESTVAR")) + obj = load_object("scrapy.utils.misc.load_object") + self.assertIs(obj, load_object) + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, "nomodule999.mod.function") self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) - def test_load_module_or_object(self): - testmod = load_module_or_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ From 7cfdca8f9b1b1f16aacae958f7a5c8824df056e7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:23:29 +0400 Subject: [PATCH 106/211] Actually run test_set_asyncio_event_loop(). --- tests/test_utils_asyncio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 01d0ee043..65e352053 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -4,6 +4,7 @@ from unittest import TestCase from pytest import mark +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -29,6 +30,8 @@ class AsyncioTest(TestCase): assert original_reactor == reactor + @mark.only_asyncio() + @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop() is asyncio.get_running_loop() + assert set_asyncio_event_loop(None) is asyncio.get_running_loop() From 27f5f3513437466d4e67df7dc3e0e57f959cc03b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:30:33 +0400 Subject: [PATCH 107/211] More quick doc fixes. --- docs/topics/addons.rst | 131 ++------------------------------------- docs/topics/settings.rst | 6 +- 2 files changed, 7 insertions(+), 130 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 25d7da50b..523f6e86e 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -40,10 +40,10 @@ This is an example where an internal add-on and two third-party add-ons (in this case with one requiring no configuration) are enabled/configured in a project's ``settings.py``:: - ADDONS = ( - 'httpcache', - 'path.to.some.addon', - ) + ADDONS = { + 'httpcache': 0, + 'path.to.some.addon': 0, + } HTTPCACHE = { 'expiration_secs': 60, @@ -385,126 +385,3 @@ Forward to other add-ons depending on Python version:: _addon = 'path.to.addon' else: _addon = 'path.to.other.addon' - - -Built-in add-on reference -========================= - -Scrapy comes with gateway add-ons that you can use to configure the built-in -middlewares and extensions. For example, to activate and configure the -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`, instead -of placing this in your ``settings.py``:: - - HTTPCACHE_ENABLED = True - HTTPCACHE_EXPIRATION_SECS = 60 - HTTPCACHE_IGNORE_HTTP_CODES = [404] - -you can also use the add-on framework:: - - ADDONS = ( - # ..., - 'httpcache', - ) - - HTTPCACHE = { - 'expiration_secs': 60, - 'ignore_http_codes': [404], - } - -Note that you *must* enable built-in addons by placing them in your -``ADDONS`` setting before you can use them for configuring built-in -components. I.e., configuring the ``HTTPCACHE`` setting will have no effect -when ``httpcache`` is not listed in ``ADDONS``. - -In general, the add-on names match the lowercase name of the component, with its -type suffix removed (i.e. the add-on configuring the -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is called -``httpcache``), and the configuration option names match the names of the -settings they map to, with the component prefix removed (i.e. -``expiration_secs`` maps to :setting:`HTTPCACHE_EXPIRATION_SECS`, as above). -The available add-ons are: - - -+--------------------------------------+--------------------------------------+ -| Add-on | Notes | -+======================================+======================================+ -| **Spider middlewares** | -+--------------------------------------+--------------------------------------+ -| depth (:class:`~scrapy.spidermi\ | | -| ddlewares.depth.DepthMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| httperror (:class:`~scrapy.spid\ | | -| ermiddlewares.httperror.HttpErrorMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| offsite (:class:`~scrapy.spid\ | | -| ermiddlewares.offsite.OffsiteMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| referer (:class:`~scrapy.spid\ | | -| ermiddlewares.referer.RefererMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| urllength (:class:`~scrapy.spid\ | | -| ermiddlewares.urllength.UrlLengthMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| **Downloader middlewares** | -+--------------------------------------+--------------------------------------+ -| ajaxcrawl (:class:`~scrapy.download\ | | -| ermiddlewares.ajaxcrawl.AjaxCrawlMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| chunked (:class:`~scrapy.download\ | | -| ermiddlewares.chunked.ChunkedTrans\ | | -| ferMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| cookies (:class:`~scrapy.download\ | | -| ermiddlewares.cookies.CookiesMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| defaultheaders (:class:`~scrapy.down\| Every configuration entry is treated | -| loadermiddlewares.defaultheaders.Def\| as a default header. | -| aultHeadersMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| **Extensions** | -+--------------------------------------+--------------------------------------+ -| autothrottle | Installing sets | -| (:ref:`topics-autothrottle`) | :setting:`AUTOTHROTTLE_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| corestats (:class:`~scrapy.exten\ | | -| sions.corestats.CoreStats`) | | -+--------------------------------------+--------------------------------------+ -| closespider (:class:`~scrapy.exten\ | | -| sions.closespider.CloseSpider`) | | -+--------------------------------------+--------------------------------------+ -| debugger (:class:`~scrapy.exten\ | | -| sions.debug.Debugger`) | | -+--------------------------------------+--------------------------------------+ -| feedexport (:ref:`topics-feed-expor\ | | -| ts`) | | -+--------------------------------------+--------------------------------------+ -| logstats (:class:`~scrapy.exten\ | | -| sions.logstats.LogStats`) | | -+--------------------------------------+--------------------------------------+ -| memdebug (:class:`~scrapy.exten\ | Installing sets | -| sions.memdebug.MemoryDebugger`) | :setting:`MEMDEBUG_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| memusage (:class:`~scrapy.exten\ | Installing sets | -| sions.memusage.MemoryUsage`) | :setting:`MEMUSAGE_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| spiderstate (:class:`~scrapy.exten\ | | -| sions.spiderstate.SpiderState`) | | -+--------------------------------------+--------------------------------------+ -| stacktracedump (:class:`~scrapy.ext\ | | -| ensions.debug.StackTraceDump`) | | -+--------------------------------------+--------------------------------------+ -| statsmailer (:class:`~scrapy.exten\ | | -| sions.statsmailer.StatsMailer`) | | -+--------------------------------------+--------------------------------------+ -| telnetconsole (:ref:`topics-telnet\ | | -| console`) | | -+--------------------------------------+--------------------------------------+ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index aa09abbd5..143002360 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -206,10 +206,10 @@ component must be enabled in order for the setting to have any effect. ADDONS ------ -Default: ``()`` +Default: ``{}`` -A tuple containing paths to the add-ons enabled in your project. For more -information, see :ref:`topics-addons`. +A dict containing paths to the add-ons enabled in your project and their +priorities. For more information, see :ref:`topics-addons`. .. setting:: AWS_ACCESS_KEY_ID From c7f78a8305fd7c08705cc0125e0f6e99dbaa0d10 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 20:43:18 +0400 Subject: [PATCH 108/211] Revert "Remove load_module_or_object." This reverts commit 815af431209686018b1bc2cbb80fd946cfe93614. --- scrapy/addons.py | 6 +++--- scrapy/utils/misc.py | 16 ++++++++++++++++ tests/test_utils_misc/__init__.py | 13 +++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 153de66d2..43a36e5e5 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -11,7 +11,7 @@ from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) @@ -252,8 +252,8 @@ class AddonManager(Mapping): """ if isinstance(path, str): try: - obj = load_object(path) - except (ValueError, NameError, ImportError): + obj = load_module_or_object(path) + except NameError: raise NameError(f"Could not find add-on '{path}'") else: obj = path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -72,6 +72,22 @@ def load_object(path: Union[str, Callable]) -> Any: return obj +def load_module_or_object(path): + """Load python module or (non-module) object from given path. + + Path can be both a Python or a file path. + """ + try: + return import_module(path) + except ImportError: + pass + try: + return load_object(path) + except (ValueError, NameError, ImportError): + pass + raise NameError(f"Could not load '{path}'") + + def walk_modules(path): """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 4f6e0d02c..7932ca04c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,6 +8,7 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, + load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -30,17 +31,17 @@ class UtilsMiscTestCase(unittest.TestCase): obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) - def test_load_object_module(self): - testmod = load_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, "nomodule999.mod.function") self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) + def test_load_module_or_object(self): + testmod = load_module_or_object(__name__ + ".testmod") + self.assertTrue(hasattr(testmod, "TESTVAR")) + obj = load_object("scrapy.utils.misc.load_object") + self.assertIs(obj, load_object) + def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ From 2f9ebb66c32fc11eee86261d55e232b7e12f6a09 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 17:00:38 +0400 Subject: [PATCH 109/211] Remove some dead code/docs. --- docs/topics/addons.rst | 32 ++------------------------ scrapy/addons.py | 26 --------------------- tests/test_addons/__init__.py | 43 ++++++++++++----------------------- 3 files changed, 16 insertions(+), 85 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 523f6e86e..35e40ca15 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -17,16 +17,7 @@ Activating and configuring add-ons Add-ons and their configuration live in Scrapy's :class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and only then, the add-on manager will read a list of enabled add-ons and their -configurations from your settings. There are two places where you can provide -the paths to add-ons you want to enable: - -* the ``ADDONS`` setting, and -* the ``scrapy.cfg`` file. - -As Scrapy settings can be modified from many places, e.g. in a project's -``settings.py``, in a Spider's ``custom_settings`` attribute, or from the -command line, using the ``ADDONS`` setting is the preferred way to -manage add-ons. +configurations from your ``ADDONS`` setting. The ``ADDONS`` setting a tuple in which every item is a path to an add-on. The path can be both a Python or a file path. While more precise, it is @@ -54,30 +45,11 @@ case with one requiring no configuration) are enabled/configured in a project's 'some_config': True, } -It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is -a little friendlier, be aware that this file, and therefore the configuration in -it, is not bound to a particular Scrapy project. While this should not pose a -problem when you use the project on your development machine only, a common -stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``. - -In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the -dictionary keys. I.e., the configuration from above would look like this: - -.. code-block:: cfg - - [addon:httpcache] - expiration_secs = 60 - ignore_http_codes = 404,405 - - [addon:path.to.some.addon] - some_config = true - - Enabling and configuring add-ons within Python code --------------------------------------------------- The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings -and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. +*at the beginning* of Scrapy's start-up process. Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is populated, changing the ``ADDONS`` setting or any of the add-on configuration dictionary settings will have no effect. diff --git a/scrapy/addons.py b/scrapy/addons.py index 43a36e5e5..66463afdb 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -261,32 +261,6 @@ class AddonManager(Mapping): obj = AddonManager.get_addon(obj._addon) return obj - def load_dict(self, addonsdict): - """Load add-ons and configurations from given dictionary. - - Each add-on should be an entry in the dictionary, where the key - corresponds to the add-on path. The value should be a dictionary - representing the add-on configuration. - - Example add-on dictionary:: - - addonsdict = { - 'path.to.addon1': { - 'setting1': 'value', - 'setting2': 42, - }, - 'path/to/addon2.py': { - 'addon2setting': True, - }, - } - - :param addonsdict: dictionary where keys correspond to add-on paths \ - and values correspond to their configuration - :type addonsdict: ``dict`` - """ - for addonpath, addoncfg in addonsdict.items(): - self.add(addonpath, addoncfg) - def load_settings(self, settings): """Load add-ons and configurations from settings object. diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 451ffed0f..d1c17a044 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,7 +1,6 @@ import itertools import unittest import warnings -from collections import OrderedDict from unittest import mock from zope.interface import directlyProvides @@ -178,24 +177,7 @@ class AddonManagerTest(unittest.TestCase): x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) - def test_load_dict_load_settings(self): - def _test_load_method(func, *args, **kwargs): - manager = AddonManager() - getattr(manager, func)(*args, **kwargs) - self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") - self.assertEqual(manager["AddonModule"], addonmod) - self.assertIn("key", manager.configs["AddonModule"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - - addonsdict = { - "tests.test_addons.addonmod": {"key": "val1"}, - "tests.test_addons.addons.GoodAddon": {"key": "val2"}, - } - _test_load_method("load_dict", addonsdict) - + def test_load_settings(self): settings = BaseSettings() settings.set( "ADDONS", @@ -203,25 +185,28 @@ class AddonManagerTest(unittest.TestCase): ) settings.set("ADDONMODULE", {"key": "val1"}) settings.set("GOODADDON", {"key": "val2"}) - _test_load_method("load_settings", settings) - - def test_load_dict_load_settings_order(self): - def _test_load_method(expected_order, func, *args, **kwargs): - manager = AddonManager() - getattr(manager, func)(*args, **kwargs) - self.assertEqual(list(manager.keys()), expected_order) + manager = AddonManager() + manager.load_settings(settings) + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") + def test_load_settings_order(self): # Get three addons named 0, 1, 2 addonlist = [addons.GoodAddon(str(x)) for x in range(3)] # Test both methods for every possible mutation for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] - addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, "load_dict", addonsdict) settings = BaseSettings( {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} ) - _test_load_method(expected_order, "load_settings", settings) + manager = AddonManager() + manager.load_settings(settings) + self.assertEqual(list(manager.keys()), expected_order) def test_enabled_disabled(self): manager = AddonManager() From 0258c87dab6d7336263ff53c7747f6400d8cd527 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 18:40:59 +0400 Subject: [PATCH 110/211] Add typing for Crawler.addons. --- scrapy/crawler.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5e7499f9c..fdcc6354d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -57,7 +57,7 @@ class Crawler: spidercls: Type[Spider], settings: Union[None, dict, Settings] = None, init_reactor: bool = False, - addons=None, + addons: Optional[AddonManager] = None, ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons = addons if addons is not None else AddonManager() + self.addons: AddonManager = addons if addons is not None else AddonManager() self.addons.load_settings(self.settings) self.addons.update_addons() self.addons.check_dependency_clashes() @@ -199,14 +199,14 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None, addons=None): + def __init__(self, settings=None, addons: Optional[AddonManager] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.addons = addons + self.addons: Optional[AddonManager] = addons self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False @property @@ -285,7 +285,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, self.addons) + return Crawler(spidercls, self.settings, addons=self.addons) def stop(self): """ @@ -331,7 +331,12 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True, addons=None): + def __init__( + self, + settings=None, + install_root_handler: bool = True, + addons: Optional[AddonManager] = None, + ): super().__init__(settings, addons) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) From f582246d7b0a70c9ace5642002c30040f7182d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 18:49:08 +0400 Subject: [PATCH 111/211] More doc fixes. --- docs/index.rst | 2 +- docs/topics/addons.rst | 33 +++++++++++---------------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ace5a2eb7..8798aebd1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -237,7 +237,7 @@ Extending Scrapy Understand the Scrapy architecture. :doc:`topics/addons` - Enable and configure built-in and third-party extensions. + Enable and configure third-party extensions. :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 35e40ca15..5d1a4f753 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -19,26 +19,18 @@ Add-ons and their configuration live in Scrapy's only then, the add-on manager will read a list of enabled add-ons and their configurations from your ``ADDONS`` setting. -The ``ADDONS`` setting a tuple in which every item is a path to an -add-on. The path can be both a Python or a file path. While more precise, it is -not necessary to specify the full add-on Python path if it is either built into -Scrapy or lives in your project's ``addons`` submodule. +The ``ADDONS`` setting is a dict in which every key is an addon class or its +import path and the vaoue is its priority. The configuration of an add-on, if necessary at all, is stored as a dictionary setting whose name is the uppercase add-on name. -This is an example where an internal add-on and two third-party add-ons (in this -case with one requiring no configuration) are enabled/configured in a project's -``settings.py``:: +This is an example where two add-ons (in this case with one requiring no +configuration) are enabled/configured in a project's ``settings.py``:: ADDONS = { - 'httpcache': 0, - 'path.to.some.addon': 0, - } - - HTTPCACHE = { - 'expiration_secs': 60, - 'ignore_http_codes': [404, 405], + 'path.to.someaddon': 0, + path.to.someaddon2: 1, } SOMEADDON = { @@ -67,7 +59,7 @@ add-ons framework, e.g.: * :meth:`~scrapy.addons.AddonManager.enable` and :meth:`~scrapy.addons.AddonManager.disable` methods, * the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the - configuration of all add-ons + configuration of all add-ons. In this example, we ensure that the ``httpcache`` add-on is loaded, and that its ``expiration_secs`` configuration is set to ``60``:: @@ -88,9 +80,9 @@ Python object up the developer. Examples: * for a small pipeline, the add-on interface could be implemented in the same class that also implements the ``open/close_spider`` and ``process_item`` - callbacks + callbacks, * for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module + by a stand-alone module. The absolute minimum interface consists of two attributes: @@ -137,9 +129,7 @@ crawling process: This method is called immediately before :meth:`update_settings`, and should be used to enable and configure other *add-ons* only. - When using this callback, be aware that there is no guarantee in which order - the :meth:`update_addons` callbacks of enabled add-ons will be called. - Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during + Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during this callback will also have their :meth:`update_addons` method called. :param config: Configuration of this add-on @@ -244,8 +234,7 @@ Check dependencies:: import boto except ImportError: raise RuntimeError("myaddon requires the boto library") - else: - self.export_config(config, settings) + self.export_config(config, settings) Enable a component that lives relative to the add-on (see :ref:`topics-api-settings`):: From 79bf8b1f2e8eae5dfb02443b4be5d474502793ca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 19:23:28 +0400 Subject: [PATCH 112/211] Test cleanup. --- tests/test_addons/__init__.py | 2 +- tests/test_crawl.py | 6 ++---- tests/test_middleware.py | 12 ------------ 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index d1c17a044..fa3f706e5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -198,7 +198,7 @@ class AddonManagerTest(unittest.TestCase): def test_load_settings_order(self): # Get three addons named 0, 1, 2 addonlist = [addons.GoodAddon(str(x)) for x in range(3)] - # Test both methods for every possible mutation + # Test for every possible ordering for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] settings = BaseSettings( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1e97863b0..920e5f4ae 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -421,10 +421,8 @@ with multiples lines addonmgr.add(FailedCheckAddon()) crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr - # Doesn't work in 'precise' test environment: - # with self.assertRaises(ValueError): - # yield crawler.crawl() - yield self.assertFailure(crawler.crawl(), ValueError) + with self.assertRaises(ValueError): + yield crawler.crawl() class CrawlSpiderTestCase(TestCase): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index fd57d846e..00ff746ee 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -87,15 +87,3 @@ class MiddlewareManagerTest(unittest.TestCase): mwman = TestMiddlewareManager.from_settings(settings) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) - - def test_instances_from_settings(self): - settings = Settings() - - class InstanceTestMiddlewareManager(MiddlewareManager): - @classmethod - def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2] - - mwman = InstanceTestMiddlewareManager.from_settings(settings) - self.assertIsInstance(mwman.middlewares[0], M1) - self.assertIsInstance(mwman.middlewares[1], M2) From 54287f733972dcaf13706664ba89ab1c52e3a290 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 19:53:34 +0400 Subject: [PATCH 113/211] Docs cleanup. --- scrapy/addons.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 66463afdb..aced6092a 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -30,9 +30,7 @@ class Addon(object): component_type = None """Component setting into which to export via :meth:`export_component`. Can - be any of the dictionary-like component setting names (e.g. - ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in - :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``, + be any of the dictionary-like component setting names. If ``None``, :meth:`export_component` will do nothing. """ @@ -50,9 +48,8 @@ class Addon(object): component = None """Component to be inserted via :meth:`export_component`. This can be anything that can be used in the dictionary-like component settings, i.e. - a class path, a class, or an instance. If ``None``, it is assumed that the - add-on itself is also provides the component interface, and ``self`` will be - used. + a class path or a class. If ``None``, it is assumed that the add-on itself + also provides the component interface, and ``self`` will be used. """ settings_prefix = None @@ -241,13 +238,13 @@ class AddonManager(Mapping): def get_addon(path): """Get an add-on object by its Python or file path. - ``path`` is assumed to be either a Python or a file path of a Scrapy - add-on. If the object or module pointed to by ``path`` has an attribute - named ``_addon`` that attribute will be assumed to be the add-on. - :meth:`get_addon` will keep following ``_addon`` attributes until it - finds an object that does not have an attribute named ``_addon``. + ``path`` is assumed to be an import path of an add-on. If the object or + module pointed to by ``path`` has an attribute named ``_addon`` that + attribute will be assumed to be the add-on. :meth:`get_addon` will keep + following ``_addon`` attributes until it finds an object that does not + have an attribute named ``_addon``. - :param path: Python or file path to an add-on + :param path: Import path of an add-on :type path: ``str`` """ if isinstance(path, str): @@ -366,7 +363,7 @@ class AddonManager(Mapping): elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " "enabled") + raise ValueError("Add-ons need to be added before they can be enabled") @property def disabled(self): From 777a6ea4128cf00d53fa71dc48385bf036cb64fc Mon Sep 17 00:00:00 2001 From: Serhii A Date: Fri, 16 Jun 2023 16:46:06 +0300 Subject: [PATCH 114/211] Make the retry middleware exception list configurable (#5929) --- docs/topics/downloader-middleware.rst | 32 ++++++++++++ scrapy/downloadermiddlewares/retry.py | 63 ++++++++++++------------ scrapy/settings/default_settings.py | 15 ++++++ tests/test_downloadermiddleware_retry.py | 39 +++++++++++++-- 4 files changed, 113 insertions(+), 36 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 7665a901a..a8e5b23bf 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -915,6 +915,7 @@ settings (see the settings documentation for more info): * :setting:`RETRY_ENABLED` * :setting:`RETRY_TIMES` * :setting:`RETRY_HTTP_CODES` +* :setting:`RETRY_EXCEPTIONS` .. reqmeta:: dont_retry @@ -966,6 +967,37 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_EXCEPTIONS + +RETRY_EXCEPTIONS +^^^^^^^^^^^^^^^^ + +Default:: + + [ + 'twisted.internet.defer.TimeoutError', + 'twisted.internet.error.TimeoutError', + 'twisted.internet.error.DNSLookupError', + 'twisted.internet.error.ConnectionRefusedError', + 'twisted.internet.error.ConnectionDone', + 'twisted.internet.error.ConnectError', + 'twisted.internet.error.ConnectionLost', + 'twisted.internet.error.TCPTimedOutError', + 'twisted.web.client.ResponseFailed', + IOError, + 'scrapy.core.downloader.handlers.http11.TunnelError', + ] + +List of exceptions to retry. + +Each list entry may be an exception type or its import path as a string. + +An exception will not be caught when the exception type is not in +:setting:`RETRY_EXCEPTIONS` or when the maximum number of retries for a request +has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught +exception propagation, see +:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 081642a4b..50cbc3111 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,31 +9,36 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ +import warnings from logging import Logger, getLogger from typing import Optional, Union -from twisted.internet import defer -from twisted.internet.error import ( - ConnectError, - ConnectionDone, - ConnectionLost, - ConnectionRefusedError, - DNSLookupError, - TCPTimedOutError, - TimeoutError, -) -from twisted.web.client import ResponseFailed - -from scrapy.core.downloader.handlers.http11 import TunnelError -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http.request import Request +from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.misc import load_object from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message retry_logger = getLogger(__name__) +class BackwardsCompatibilityMetaclass(type): + @property + def EXCEPTIONS_TO_RETRY(cls): + warnings.warn( + "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. " + "Use the RETRY_EXCEPTIONS setting instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return tuple( + load_object(x) if isinstance(x, str) else x + for x in Settings().getlist("RETRY_EXCEPTIONS") + ) + + def get_retry_request( request: Request, *, @@ -121,23 +126,7 @@ def get_retry_request( return None -class RetryMiddleware: - # IOError is raised by the HttpCompression middleware when trying to - # decompress an empty response - EXCEPTIONS_TO_RETRY = ( - defer.TimeoutError, - TimeoutError, - DNSLookupError, - ConnectionRefusedError, - ConnectionDone, - ConnectError, - ConnectionLost, - TCPTimedOutError, - ResponseFailed, - IOError, - TunnelError, - ) - +class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass): def __init__(self, settings): if not settings.getbool("RETRY_ENABLED"): raise NotConfigured @@ -147,6 +136,16 @@ class RetryMiddleware: ) self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") + if not hasattr( + self, "EXCEPTIONS_TO_RETRY" + ): # If EXCEPTIONS_TO_RETRY is not "overriden" + self.exceptions_to_retry = tuple( + load_object(x) if isinstance(x, str) else x + for x in settings.getlist("RETRY_EXCEPTIONS") + ) + else: + self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY + @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) @@ -160,7 +159,7 @@ class RetryMiddleware: return response def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get( + if isinstance(exception, self.exceptions_to_retry) and not request.meta.get( "dont_retry", False ): return self._retry(request, exception, spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 260ec1701..89837b4ab 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -258,6 +258,21 @@ RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 +RETRY_EXCEPTIONS = [ + "twisted.internet.defer.TimeoutError", + "twisted.internet.error.TimeoutError", + "twisted.internet.error.DNSLookupError", + "twisted.internet.error.ConnectionRefusedError", + "twisted.internet.error.ConnectionDone", + "twisted.internet.error.ConnectError", + "twisted.internet.error.ConnectionLost", + "twisted.internet.error.TCPTimedOutError", + "twisted.web.client.ResponseFailed", + # IOError is raised by the HttpCompression middleware when trying to + # decompress an empty response + IOError, + "scrapy.core.downloader.handlers.http11.TunnelError", +] ROBOTSTXT_OBEY = False ROBOTSTXT_PARSER = "scrapy.robotstxt.ProtegoRobotParser" diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 63bd61848..97ae1e29a 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,5 +1,6 @@ import logging import unittest +import warnings from testfixtures import LogCapture from twisted.internet import defer @@ -15,6 +16,7 @@ from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler @@ -110,19 +112,48 @@ class RetryTest(unittest.TestCase): == 2 ) - def _test_retry_exception(self, req, exception): + def test_exception_to_retry_added(self): + exc = ValueError + settings_dict = { + "RETRY_EXCEPTIONS": list(RETRY_EXCEPTIONS) + [exc], + } + crawler = get_crawler(Spider, settings_dict=settings_dict) + mw = RetryMiddleware.from_crawler(crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + self._test_retry_exception(req, exc("foo"), mw) + + def test_exception_to_retry_customMiddleware(self): + exc = ValueError + + with warnings.catch_warnings(record=True) as warns: + + class MyRetryMiddleware(RetryMiddleware): + EXCEPTIONS_TO_RETRY = RetryMiddleware.EXCEPTIONS_TO_RETRY + (exc,) + + self.assertEqual(len(warns), 1) + + mw2 = MyRetryMiddleware.from_crawler(self.crawler) + req = Request(f"http://www.scrapytest.org/{exc.__name__}") + req = mw2.process_exception(req, exc("foo"), self.spider) + assert isinstance(req, Request) + self.assertEqual(req.meta["retry_times"], 1) + + def _test_retry_exception(self, req, exception, mw=None): + if mw is None: + mw = self.mw + # first retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 1) # second retry - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) self.assertEqual(req.meta["retry_times"], 2) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) From 2122278d4bb7177a550bc0b04dd96d1fc1fad1a0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 18:37:50 +0400 Subject: [PATCH 115/211] Drop Python 3.7 support. --- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 13 +++++-------- .github/workflows/tests-windows.yml | 5 +---- README.rst | 2 +- docs/contributing.rst | 12 ++++++------ docs/intro/install.rst | 2 +- scrapy/__init__.py | 4 ++-- setup.py | 3 +-- tox.ini | 8 +++----- 9 files changed, 21 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 174d245ca..3044a1af3 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 96b26a1f8..39e3b0af7 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -8,9 +8,6 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.8 - env: - TOXENV: py - python-version: 3.9 env: TOXENV: py @@ -28,19 +25,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: asyncio-pinned - - python-version: pypy3.7 + - python-version: pypy3.8 env: TOXENV: pypy3-pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: extra-deps-pinned - - python-version: 3.7.13 + - python-version: 3.8.17 env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index f60c48841..5bcf74d5e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -8,12 +8,9 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - env: - TOXENV: windows-pinned - python-version: 3.8 env: - TOXENV: py + TOXENV: windows-pinned - python-version: 3.9 env: TOXENV: py diff --git a/README.rst b/README.rst index 970bf2c35..1918850d6 100644 --- a/README.rst +++ b/README.rst @@ -58,7 +58,7 @@ including a list of features. Requirements ============ -* Python 3.7+ +* Python 3.8+ * Works on Linux, Windows, macOS, BSD Install diff --git a/docs/contributing.rst b/docs/contributing.rst index eef92e148..2b3249601 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -265,15 +265,15 @@ To run a specific test (say ``tests/test_loader.py``) use: To run the tests on a specific :doc:`tox ` environment, use ``-e `` with an environment name from ``tox.ini``. For example, to run -the tests with Python 3.7 use:: +the tests with Python 3.10 use:: - tox -e py37 + tox -e py310 You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode ` to run the tests on multiple environments in parallel:: - tox -e py37,py38 -p auto + tox -e py39,py310 -p auto To pass command-line options to :doc:`pytest `, add them after ``--`` in your call to :doc:`tox `. Using ``--`` overrides the @@ -283,9 +283,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.7 :doc:`tox ` environment using all your CPU cores:: +the Python 3.10 :doc:`tox ` environment using all your CPU cores:: - tox -e py37 -- scrapy tests -n auto + tox -e py310 -- scrapy tests -n auto To see coverage report install :doc:`coverage ` (``pip install coverage``) and run: @@ -322,4 +322,4 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _good first issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy \ No newline at end of file +.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 2c2079f68..c90c1d2bf 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.7+, either the CPython implementation (default) or +Scrapy requires Python 3.8+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/scrapy/__init__.py b/scrapy/__init__.py index a757a9290..cc0e539c4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -34,8 +34,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro) # Check minimum required Python version -if sys.version_info < (3, 7): - print(f"Scrapy {__version__} requires Python 3.7+") +if sys.version_info < (3, 8): + print(f"Scrapy {__version__} requires Python 3.8+") sys.exit(1) diff --git a/setup.py b/setup.py index c6bcf2439..f1cd4c5e2 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ setup( "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -91,7 +90,7 @@ setup( "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=install_requires, extras_require=extras_require, ) diff --git a/tox.ini b/tox.ini index 5f8bf85f2..d5b6118f5 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,6 @@ deps = #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' - # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) - markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -96,7 +94,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} PyDispatcher==2.0.5 @@ -129,7 +127,7 @@ deps = Twisted[http2] [testenv:extra-deps-pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} boto3==1.20.0 @@ -211,7 +209,7 @@ commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} [testenv:botocore-pinned] -basepython = python3.7 +basepython = python3.8 deps = {[pinned]deps} botocore==1.4.87 From 1b2c9a3e0ae9766623a06ce7e739a3dfda399159 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 18:38:56 +0400 Subject: [PATCH 116/211] Bump isort and flake8 versions. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index faf8808f2..31e9ed1ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 # 6.0.0 drops Python 3.7 support + rev: 6.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.11.5 # 5.12 drops Python 3.7 support + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs From 075b89eab5e201246a5bddc4a6ce9c7921de082b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 18 Jun 2023 19:08:41 +0400 Subject: [PATCH 117/211] Bump lxml and cryptography to versions with 3.8 wheels available. --- setup.py | 4 ++-- tox.ini | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f1cd4c5e2..ccfe20ae5 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ def has_environment_marker_platform_impl_support(): install_requires = [ "Twisted>=18.9.0", - "cryptography>=3.4.6", + "cryptography>=36.0.0", "cssselect>=0.9.1", "itemloaders>=1.0.1", "parsel>=1.5.0", @@ -34,7 +34,7 @@ install_requires = [ "setuptools", "packaging", "tldextract", - "lxml>=4.3.0", + "lxml>=4.4.1", ] extras_require = {} cpython_dependencies = [ diff --git a/tox.ini b/tox.ini index d5b6118f5..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = [pinned] deps = - cryptography==3.4.6 + cryptography==36.0.0 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 @@ -81,7 +81,7 @@ deps = Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.3.0 + lxml==4.4.1 -rtests/requirements.txt # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies From 0097b4c0bb4de6e651e8b9d064aae140e11698d5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 16:40:38 -0300 Subject: [PATCH 118/211] cleanup: Remove `pkg_resources` usage --- docs/topics/components.rst | 2 +- scrapy/cmdline.py | 5 ++--- setup.py | 2 +- tests/test_crawler.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1ed55f000..478dd9647 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -70,7 +70,7 @@ If your requirement is a minimum Scrapy version, you may use .. code-block:: python - from pkg_resources import parse_version + from packaging.version import parse as parse_version import scrapy diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 730e55350..cfa771104 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -3,8 +3,7 @@ import cProfile import inspect import os import sys - -import pkg_resources +from importlib.metadata import entry_points import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -49,7 +48,7 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in pkg_resources.iter_entry_points(group): + for entry_point in entry_points(group): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() diff --git a/setup.py b/setup.py index ccfe20ae5..f918db09e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ecb9c9b62..d54a2cb7e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,7 @@ import sys import warnings from pathlib import Path -from pkg_resources import parse_version +from packaging.version import parse as parse_version from pytest import mark, raises from twisted import version as twisted_version from twisted.internet import defer From 6afb31b82b5a0a5d2f37962c250fbf34c21d8580 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 16:48:07 -0300 Subject: [PATCH 119/211] chore: Add `packaging` to tests deps --- tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..72350b216 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,3 +14,4 @@ brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" +packaging \ No newline at end of file From 6e1af20ac4dd537a4643df5c022f948cf07d05ec Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:00:01 -0300 Subject: [PATCH 120/211] fix: add `build-system` --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index ec3a59366..79d692599 100644 --- a/tox.ini +++ b/tox.ini @@ -218,3 +218,6 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[build-system] +build-backend = 'setuptools.build_meta' \ No newline at end of file From a93a63c208af1d13d5ea84623d160337c7fec6c5 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:05:49 -0300 Subject: [PATCH 121/211] fix: move import to inside function --- setup.py | 3 ++- tests/requirements.txt | 1 - tox.ini | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index f918db09e..dfe5b80ec 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ from pathlib import Path -from packaging.version import parse as parse_version from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup @@ -15,6 +14,8 @@ def has_environment_marker_platform_impl_support(): it is 18.5, see: https://setuptools.readthedocs.io/en/latest/history.html#id235 """ + from packaging.version import parse as parse_version + return parse_version(setuptools_version) >= parse_version("18.5") diff --git a/tests/requirements.txt b/tests/requirements.txt index 72350b216..618949795 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,4 +14,3 @@ brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" -packaging \ No newline at end of file diff --git a/tox.ini b/tox.ini index 79d692599..ec3a59366 100644 --- a/tox.ini +++ b/tox.ini @@ -218,6 +218,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - -[build-system] -build-backend = 'setuptools.build_meta' \ No newline at end of file From 0b1da44a05cc64970aa11ccc4d7a4a3bec143443 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 17:14:21 -0300 Subject: [PATCH 122/211] chore: Remove deprecated code --- setup.py | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index dfe5b80ec..1f214571b 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,10 @@ from pathlib import Path -from setuptools import __version__ as setuptools_version from setuptools import find_packages, setup version = (Path(__file__).parent / "scrapy/VERSION").read_text("ascii").strip() -def has_environment_marker_platform_impl_support(): - """Code extracted from 'pytest/setup.py' - https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 - - The first known release to support environment marker with range operators - it is 18.5, see: - https://setuptools.readthedocs.io/en/latest/history.html#id235 - """ - from packaging.version import parse as parse_version - - return parse_version(setuptools_version) >= parse_version("18.5") - - install_requires = [ "Twisted>=18.9.0", "cryptography>=36.0.0", @@ -37,19 +23,10 @@ install_requires = [ "tldextract", "lxml>=4.4.1", ] -extras_require = {} -cpython_dependencies = [ - "PyDispatcher>=2.0.5", -] -if has_environment_marker_platform_impl_support(): - extras_require[ - ':platform_python_implementation == "CPython"' - ] = cpython_dependencies - extras_require[':platform_python_implementation == "PyPy"'] = [ - "PyPyDispatcher>=2.1.0", - ] -else: - install_requires.extend(cpython_dependencies) +extras_require = { + ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], + ':platform_python_implementation == "PyPy"': ["PyPyDispatcher>=2.1.0"], +} setup( From 82cf00bbc931723320c9aca3cf0305372027d0a0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 20 Jun 2023 18:27:03 -0300 Subject: [PATCH 123/211] fix: default value --- scrapy/cmdline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cfa771104..efc9b36ea 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -48,7 +48,7 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in entry_points(group): + for entry_point in entry_points().get(group, {}): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() From ee215a29704adbc61a40baeca1d27f6179a1735e Mon Sep 17 00:00:00 2001 From: Aaron Smith <60046611+medic-code@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:05:39 +0100 Subject: [PATCH 124/211] Change redirect text from Response.request docs (#5937) --- docs/topics/request-response.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 407df32d2..41df51589 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1103,9 +1103,10 @@ Response objects through all :ref:`Downloader Middlewares `. In particular, this means that: - - HTTP redirections will cause the original request (to the URL before - redirection) to be assigned to the redirected response (with the final - URL after redirection). + - HTTP redirections will create a new request from the request before + redirection. It has the majority of the same metadata and original + request attributes and gets assigned to the redirected response + instead of the propagation of the original request. - Response.request.url doesn't always equal Response.url From 5360ba34bc345667f77a4d4256f15fd648e42e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Wed, 21 Jun 2023 11:08:53 -0700 Subject: [PATCH 125/211] IOError and other cleanup (#4716) --- docs/utils/linkfix.py | 2 +- scrapy/downloadermiddlewares/decompression.py | 4 ++-- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/settings/default_settings.py | 4 ++-- scrapy/utils/gz.py | 2 +- scrapy/utils/python.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware.py | 4 ++-- tests/test_mail.py | 2 -- tests/test_robotstxt_interface.py | 1 - tests/test_utils_gz.py | 2 +- tests/test_utils_iterators.py | 4 ++-- 12 files changed, 14 insertions(+), 17 deletions(-) diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 1f270837c..c17b9d511 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -30,7 +30,7 @@ def main(): try: with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() - except IOError: + except OSError: print("linkcheck output not found; please run linkcheck first.") sys.exit(1) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 5839dc243..3b8702419 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -63,7 +63,7 @@ class DecompressionMiddleware: archive = BytesIO(response.body) try: body = gzip.GzipFile(fileobj=archive).read() - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) @@ -72,7 +72,7 @@ class DecompressionMiddleware: def _is_bzip2(self, response): try: body = bz2.decompress(response.body) - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index b9316c43a..ac87d4a4e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -37,7 +37,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError, + OSError, ) def __init__(self, settings: Settings, stats: StatsCollector) -> None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 89837b4ab..a4cb555bd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -268,9 +268,9 @@ RETRY_EXCEPTIONS = [ "twisted.internet.error.ConnectionLost", "twisted.internet.error.TCPTimedOutError", "twisted.web.client.ResponseFailed", - # IOError is raised by the HttpCompression middleware when trying to + # OSError is raised by the HttpCompression middleware when trying to # decompress an empty response - IOError, + OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..77e0197d8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -15,7 +15,7 @@ def gunzip(data): try: chunk = f.read1(8196) output_list.append(chunk) - except (IOError, EOFError, struct.error): + except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output_list is empty and f.extrabuf diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..bb5dbebbc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -291,7 +291,7 @@ def without_none_values(iterable): try: return {k: v for k, v in iterable.items() if v is not None} except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + return type(iterable)(v for v in iterable if v is not None) def global_object_name(obj): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fd4176e2f..9731b62c4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase): def test_non_existent(self): request = Request(f"file://{self.mktemp()}") d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, IOError) + return self.assertFailure(d, OSError) class ContentLengthHeaderResource(resource.Resource): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 060cfe08b..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase): In particular when some website returns a 30x response with header 'Content-Encoding: gzip' giving as result the error below: - exceptions.IOError: Not a gzipped file + BadGzipFile: Not a gzipped file (...) """ req = Request("http://example.com") @@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(IOError, self._download, request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_mail.py b/tests/test_mail.py index bc7298e9d..504c78486 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,5 +1,3 @@ -# coding=utf-8 - import unittest from email.charset import Charset from io import BytesIO diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 8d87a322a..d7a923085 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,3 @@ -# coding=utf-8 from twisted.trial import unittest diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 6b2a458bc..7b7a25db8 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase): def test_gunzip_no_gzip_file_raises(self): self.assertRaises( - IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() + OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() ) def test_gunzip_truncated_short(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index faf7d2709..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) + self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t") From 04ee3303e4487270a433f5c3a087bda9a87d7008 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 21 Jun 2023 22:04:06 -0700 Subject: [PATCH 126/211] Adding support for Windows of absolute pathlib.Path objects in FeedExporter (#5939) --- docs/topics/feed-exports.rst | 4 ++-- scrapy/extensions/feedexport.py | 7 +++++-- tests/test_feedexport.py | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b31dc069e..aba47d998 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -156,8 +156,8 @@ The feeds are stored in the local filesystem. - Required external libraries: none Note that for the local filesystem storage (only) you can omit the scheme if -you specify an absolute path like ``/tmp/export.csv``. This only works on Unix -systems though. +you specify an absolute path like ``/tmp/export.csv`` (Unix systems only). +Alternatively you can also use a :class:`pathlib.Path` object. .. _topics-feed-storage-ftp: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 39934cbf3..1cdc78f59 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -382,7 +382,9 @@ class FeedExporter: category=ScrapyDeprecationWarning, stacklevel=2, ) - uri = str(self.settings["FEED_URI"]) # handle pathlib.Path objects + uri = self.settings["FEED_URI"] + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() feed_options = {"format": self.settings.get("FEED_FORMAT", "jsonlines")} self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings @@ -392,7 +394,8 @@ class FeedExporter: # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict("FEEDS").items(): - uri = str(uri) # handle pathlib.Path objects + # handle pathlib.Path objects + uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 62a5697cd..8df86dbd7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2758,6 +2758,31 @@ class FeedExportInitTest(unittest.TestCase): with self.assertRaises(NotConfigured): FeedExporter.from_crawler(crawler) + def test_absolute_pathlib_as_uri(self): + with tempfile.NamedTemporaryFile(suffix="json") as tmp: + settings = { + "FEEDS": { + Path(tmp.name).resolve(): { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) + + def test_relative_pathlib_as_uri(self): + settings = { + "FEEDS": { + Path("./items.json"): { + "format": "json", + }, + }, + } + crawler = get_crawler(settings_dict=settings) + exporter = FeedExporter.from_crawler(crawler) + self.assertIsInstance(exporter, FeedExporter) + class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage): def __init__(self, uri): From e71d6d67e56e35642fddc226e34e4d523041ad17 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 22 Jun 2023 21:10:50 +0400 Subject: [PATCH 127/211] Apply suggestions from code review --- scrapy/extensions/feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 7e93bc366..d088450a7 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -439,10 +439,10 @@ class FeedExporter: return slot_.file if slot.itemcount: - # Nomal case + # Normal case slot.finish_exporting() elif slot.store_empty and slot.batch_id == 1: - # Need Store Empty + # Need to store the empty file slot.start_exporting() slot.finish_exporting() else: From 080b9bd0b8f65dcf09ea8ad94505fec6c77f820c Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 22 Jun 2023 23:58:03 -0300 Subject: [PATCH 128/211] chore: Implement `pop` method on `BaseSettings` class --- scrapy/settings/__init__.py | 14 ++++++++++++++ tests/test_settings/__init__.py | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index a3b849f7b..57fe1d17a 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -75,6 +75,8 @@ class BaseSettings(MutableMapping): highest priority will be retrieved. """ + __default = object() + def __init__(self, values=None, priority="project"): self.frozen = False self.attributes = {} @@ -445,6 +447,18 @@ class BaseSettings(MutableMapping): else: p.text(pformat(self.copy_to_dict())) + def pop(self, name, default=__default): + try: + value = self.attributes[name] + except KeyError: + if default is self.__default: + raise + + return SettingsAttribute(default, get_settings_priority("project")) + else: + del self.attributes[name] + return value + class Settings(BaseSettings): """ diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4a577cd8c..0e2f4aa98 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -451,6 +451,19 @@ class SettingsTest(unittest.TestCase): self.assertIsInstance(myhandler_instance, FileDownloadHandler) self.assertTrue(hasattr(myhandler_instance, "download_request")) + def test_pop_item_with_default_value(self): + settings = Settings() + + with self.assertRaises(KeyError): + settings.pop("DUMMY_CONFIG") + + dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value") + + self.assertEqual( + repr(dummy_config), "" + ) + self.assertEqual(dummy_config.value, "dummy_value") + if __name__ == "__main__": unittest.main() From 876feaf339e181c9c5a6b9a5f8ffedc03a9ed3d2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 23 Jun 2023 00:14:31 -0300 Subject: [PATCH 129/211] chore: Use dunder to delete item instead of del keyword to handle immutable settings --- scrapy/settings/__init__.py | 2 +- tests/test_settings/__init__.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 57fe1d17a..8b3bdbabe 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -456,7 +456,7 @@ class BaseSettings(MutableMapping): return SettingsAttribute(default, get_settings_priority("project")) else: - del self.attributes[name] + self.__delitem__(name) return value diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 0e2f4aa98..125b1d96f 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -464,6 +464,22 @@ class SettingsTest(unittest.TestCase): ) self.assertEqual(dummy_config.value, "dummy_value") + def test_pop_item_with_frozen_settings(self): + settings = Settings( + {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} + ) + + self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value") + + settings.freeze() + + with self.assertRaises(TypeError) as error: + settings.pop("OTHER_DUMMY_CONFIG") + + self.assertEqual( + str(error.exception), "Trying to modify an immutable Settings object" + ) + if __name__ == "__main__": unittest.main() From a3f8912d69eacdd2208617e6afb418e4e1847e36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 23 Jun 2023 00:15:32 -0300 Subject: [PATCH 130/211] chore: Rename test --- tests/test_settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 125b1d96f..bb6dc67fa 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -464,7 +464,7 @@ class SettingsTest(unittest.TestCase): ) self.assertEqual(dummy_config.value, "dummy_value") - def test_pop_item_with_frozen_settings(self): + def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) From e7124447f7e86ee93ad78ffd0ebfa6acbebc73ce Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 26 Jun 2023 16:57:46 +0400 Subject: [PATCH 131/211] Remove unneeded code. --- docs/topics/addons.rst | 241 ++------------------- scrapy/addons.py | 347 ++++-------------------------- scrapy/cmdline.py | 5 +- scrapy/crawler.py | 23 +- scrapy/interfaces.py | 21 +- scrapy/utils/conf.py | 12 +- scrapy/utils/misc.py | 18 +- scrapy/utils/test.py | 3 +- tests/test_addons.py | 72 +++++++ tests/test_addons/__init__.py | 342 ----------------------------- tests/test_addons/addonmod.py | 16 -- tests/test_addons/addons.py | 33 --- tests/test_crawl.py | 1 - tests/test_crawler.py | 27 --- tests/test_utils_misc/__init__.py | 7 - tests/test_utils_misc/testmod.py | 1 - 16 files changed, 140 insertions(+), 1029 deletions(-) create mode 100644 tests/test_addons.py delete mode 100644 tests/test_addons/__init__.py delete mode 100644 tests/test_addons/addonmod.py delete mode 100644 tests/test_addons/addons.py delete mode 100644 tests/test_utils_misc/testmod.py diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 5d1a4f753..6a9590f33 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -15,12 +15,13 @@ Activating and configuring add-ons ================================== Add-ons and their configuration live in Scrapy's -:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and -only then, the add-on manager will read a list of enabled add-ons and their -configurations from your ``ADDONS`` setting. +:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` +initialization the add-on manager will read a list of enabled add-ons from your +``ADDONS`` setting and their optional configuration from the respective +settings. The ``ADDONS`` setting is a dict in which every key is an addon class or its -import path and the vaoue is its priority. +import path and the value is its priority. The configuration of an add-on, if necessary at all, is stored as a dictionary setting whose name is the uppercase add-on name. @@ -37,66 +38,17 @@ configuration) are enabled/configured in a project's ``settings.py``:: 'some_config': True, } -Enabling and configuring add-ons within Python code ---------------------------------------------------- - -The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings -*at the beginning* of Scrapy's start-up process. -Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is -populated, changing the ``ADDONS`` setting or any of the add-on -configuration dictionary settings will have no effect. - -If you want to enable, disable, or configure add-ons in Python code, for example -when writing your own add-on, you will have to use the -:class:`~scrapy.addons.AddonManager`. You can access the add-on manager through -either ``crawler.addons`` or, if you are writing an add-on, through the -``addons`` argument of the :meth:`update_addons` callback. The add-on manager -provides many useful methods and attributes to facilitate interacting with the -add-ons framework, e.g.: - -* an :meth:`~scrapy.addons.AddonManager.add` method to load add-ons, -* the :attr:`~scrapy.addons.AddonManager.enabled` list of enabled add-ons, -* :meth:`~scrapy.addons.AddonManager.enable` and - :meth:`~scrapy.addons.AddonManager.disable` methods, -* the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the - configuration of all add-ons. - -In this example, we ensure that the ``httpcache`` add-on is loaded, and that -its ``expiration_secs`` configuration is set to ``60``:: - - # addons is an instance of AddonManager - if 'httpcache' not in addons: - addons.add('httpcache', {'expiration_secs': 60}) - else: - addons.configs['httpcache']['expiration_secs'] = 60 - Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*. -The interface is enforced through ``zope.interface``. This leaves the choice of -Python object up the developer. Examples: - -* for a small pipeline, the add-on interface could be implemented in the same - class that also implements the ``open/close_spider`` and ``process_item`` - callbacks, -* for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module. - -The absolute minimum interface consists of two attributes: +Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: .. attribute:: name string with add-on name -.. attribute:: version - - version string (PEP-404, e.g. ``'1.0.1'``) - -Of course, stating just these two attributes will not get you very far. Add-ons -can provide three callback methods that are called at various stages before the -crawling process: + :type: ``str`` .. method:: update_settings(config, settings) @@ -124,75 +76,18 @@ crawling process: :param crawler: Fully initialized Scrapy crawler :type crawler: :class:`~scrapy.crawler.Crawler` -.. method:: update_addons(config, addons) - - This method is called immediately before :meth:`update_settings`, and should - be used to enable and configure other *add-ons* only. - - Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during - this callback will also have their :meth:`update_addons` method called. - - :param config: Configuration of this add-on - :type config: ``dict`` - - :param addons: Add-on manager holding all loaded add-ons - :type addons: :class:`~scrapy.addons.AddonManager` - -Additionally, add-ons may (and should, where appropriate) provide one or more -attributes that can be used for limited automated detection of possible -dependency clashes: - -.. attribute:: requires - - list of built-in or custom components needed by this add-on, as strings. - -.. attribute:: modifies - - list of built-in or custom components whose functionality is affected or - replaced by this add-on (a custom HTTP cache should list ``httpcache`` here) - -.. attribute:: provides - - list of components provided by this add-on (e.g. ``mongodb`` for an - extension that provides generic read/write access to a MongoDB database) - -The entries in the :attr:`requires` and :attr:`modifies` attributes can be add-on -names or components from other add-ons' :attr:`provides` attribute. You can -specify :pep:`440`-style information about required versions. Examples:: - - requires = ['httpcache'] - requires = ['otheraddon >= 2.0', 'yetanotheraddon'] - -The Python object or module that is pointed to by an add-on path (e.g. given in -the ``ADDONS`` setting, or given to -:meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an -add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be -either an add-on or another add-on path. - Add-on base class ================= Scrapy comes with a built-in base class for add-ons which provides some -convenience functionality: - -* basic settings can be exported via :meth:`~scrapy.addons.Addon.export_basics`, - configurable via :attr:`~scrapy.addons.Addon.basic_settings`. -* a single component (e.g. an item pipeline or a downloader middleware) can be - inserted into Scrapy's settings via - :meth:`~scrapy.addons.Addon.export_component`, configurable via - :attr:`~scrapy.addons.Addon.component_type`, - :attr:`~scrapy.addons.Addon.component_key`, - :attr:`~scrapy.addons.Addon.component`, and the ``order`` key in - :attr:`~scrapy.addons.Addon.default_config`. -* the add-on configuration can be exposed into Scrapy's settings via - :meth:`~scrapy.addons.Addon.export_config`, configurable via - :attr:`~scrapy.addons.Addon.default_config`, - :attr:`~scrapy.addons.Addon.config_mapping`, and - :attr:`~scrapy.addons.Addon.settings_prefix`. +convenience functionality: the add-on configuration can be exposed into +Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable +via :attr:`~scrapy.addons.Addon.default_config` and +:attr:`~scrapy.addons.Addon.config_mapping`. By default, the base add-on class will expose the add-on configuration into -Scrapy's settings namespace, in caps and with the add-on name prepended. It is +Scrapy's settings namespace, in upper case. It is easy to write your own functionality while still being able to use the convenience functions by overwriting :meth:`~scrapy.addons.Addon.update_settings`. @@ -213,13 +108,11 @@ Set some basic configuration using the :class:`Addon` base class:: class MyAddon(Addon): name = 'myaddon' - version = '1.0' - component = 'path.to.mypipeline' - component_type = 'ITEM_PIPELINES' - component_order = 200 - basic_settings = { - 'DNSCACHE_ENABLED': False, - } + + def update_settings(self, config, settings): + super().update_settings(settings) + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + settings["DNSCACHE_ENABLED"] = True Check dependencies:: @@ -227,78 +120,22 @@ Check dependencies:: class MyAddon(Addon): name = 'myaddon' - version = '1.0' def update_settings(self, config, settings): try: import boto except ImportError: raise RuntimeError("myaddon requires the boto library") - self.export_config(config, settings) - -Enable a component that lives relative to the add-on (see -:ref:`topics-api-settings`):: - - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - version = '1.0' - component = __name__ + '.downloadermw.coolmw' - component_type = 'DOWNLOADER_MIDDLEWARES' - component_order = 900 - -Instantiate components ad hoc:: - - from path.to.my.pipelines import MySQLPipeline - - class MyAddon(object): - name = 'myaddon' - version = '1.0' - - def update_settings(self, config, settings): - mysqlpl = MySQLPipeline(password=config['password']) - settings.set( - 'ITEM_PIPELINES', - {mysqlpl: 200}, - priority='addon', - ) - -Provide add-on interface along component interface:: - - class MyPipeline(object): - name = 'mypipeline' - version = '1.0' - - def process_item(self, item, spider): - # Do some processing here - return item - - def update_settings(self, config, settings): - settings.set( - 'ITEM_PIPELINES', - {self: 200}, - priority='addon', - ) - -Enable another addon (see :ref:`topics-api-addonmanager`):: - - class MyAddon(object): - name = 'myaddon' - version = '1.0' - - def update_addons(self, config, addons): - if 'httpcache' not in addons.enabled: - addons.add('httpcache', {'expiration_secs': 60}) + super().update_settings(settings) Check configuration of fully initialized crawler (see :ref:`topics-api-crawler`):: class MyAddon(object): name = 'myaddon' - version = '1.0' def update_settings(self, config, settings): + super().update_settings(settings) settings.set('DNSCACHE_ENABLED', False, priority='addon') def check_configuration(self, config, crawler): @@ -306,43 +143,3 @@ Check configuration of fully initialized crawler (see # The spider, some other add-on, or the user messed with the # DNS cache setting raise ValueError("myaddon is incompatible with DNS cache") - -Provide add-on interface through a module: - -.. code-block:: python - - name = "AddonModule" - version = "1.0" - - - class MyPipeline(object): - ... - - - class MyDownloaderMiddleware(object): - ... - - - def update_settings(config, settings): - settings.set( - "ITEM_PIPELINES", - {MyPipeline(): 200}, - priority="addon", - ) - settings.set( - "DOWNLOADER_MIDDLEWARES", - {MyDownloaderMiddleware(): 800}, - priority="addon", - ) - -Forward to other add-ons depending on Python version:: - - # This could be a Python module, say project/pipelines/mypipeline.py, but - # could also be done inside a class, etc. - import six - - if six.PY3: - # We're running Python 3 - _addon = 'path.to.addon' - else: - _addon = 'path.to.other.addon' diff --git a/scrapy/addons.py b/scrapy/addons.py index aced6092a..a54086fda 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,108 +1,27 @@ -import warnings -from collections import OrderedDict, defaultdict from collections.abc import Mapping -from inspect import isclass -from typing import Dict +from typing import Any, Dict, Iterator, Optional, OrderedDict -import zope.interface -from packaging.requirements import Requirement -from packaging.version import Version -from zope.interface.verify import verifyObject - -from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_module_or_object +from scrapy.utils.misc import load_object -@zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None - """``dict`` of settings that will be exported via :meth:`export_basics`.""" + name: str default_config = None """``dict`` with default configuration.""" config_mapping = None """``dict`` with mappings from config names to setting names. The given - setting names will be taken as given, i.e. they will be neither prefixed - nor uppercased. + setting names will be taken as given, not uppercased. """ - component_type = None - """Component setting into which to export via :meth:`export_component`. Can - be any of the dictionary-like component setting names. If ``None``, - :meth:`export_component` will do nothing. - """ - - component_key = None - """Key to be used in the component dictionary setting when exporting via - :meth:`export_component`. This is only useful for the settings that have - no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``. - """ - - component_order = 0 - """Component order to use when not given in the add-on configuration. Has - no effect for component types that use :attr:`component_key`. - """ - - component = None - """Component to be inserted via :meth:`export_component`. This can be - anything that can be used in the dictionary-like component settings, i.e. - a class path or a class. If ``None``, it is assumed that the add-on itself - also provides the component interface, and ``self`` will be used. - """ - - settings_prefix = None - """Prefix with which the add-on configuration will be exported into the - global settings namespace via :meth:`export_config`. If ``None``, - :attr:`name` will be used. If ``False``, no configuration will be exported. - """ - - def export_component(self, config, settings): - """Export the component in :attr:`component` into the dictionary-like - component setting derived from :attr:`component_type`. - - Where applicable, the order parameter of the component (i.e. the - dictionary value) will be retrieved from the ``order`` add-on - configuration value. - - :param config: Add-on configuration from which to read component order - :type config: ``dict`` - - :param settings: Settings object into which to export component - :type settings: :class:`~scrapy.settings.Settings` - """ - if self.component_type: - comp = self.component or self - if self.component_key: - # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'} - k = self.component_key - v = comp - else: - # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} - k = comp - v = config.get("order", self.component_order) - settings[self.component_type].update({k: v}, "addon") - - def export_basics(self, settings): - """Export the :attr:`basic_settings` attribute into the settings object. - - All settings will be exported with ``addon`` priority (see - :ref:`topics-api-settings`). - - :param settings: Settings object into which to expose the basic settings - :type settings: :class:`~scrapy.settings.Settings` - """ - for setting, value in (self.basic_settings or {}).items(): - settings.set(setting, value, "addon") - def export_config(self, config, settings): - """Export the add-on configuration, all keys in caps and with - :attr:`settings_prefix` or :attr:`name` prepended, into the settings + """Export the add-on configuration, all keys in caps, into the settings object. For example, the add-on configuration ``{'key': 'value'}`` will export - the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings + the setting ``KEY`` with a value of ``value``. All settings will be exported with ``addon`` priority (see :ref:`topics-api-settings`). @@ -112,11 +31,8 @@ class Addon(object): :param settings: Settings object into which to export the configuration :type settings: :class:`~scrapy.settings.Settings` """ - if self.settings_prefix is False: - return conf = self.default_config or {} conf.update(config) - prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} @@ -124,14 +40,11 @@ class Addon(object): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + "_" + key).upper() + key = key.upper() settings.set(key, val, "addon") def update_settings(self, config, settings): - """Export both the basic settings and the add-on configuration. I.e., - call :meth:`export_basics` and :meth:`export_config`. - - For more advanced add-ons, you may want to override this callback. + """Modifiy `settings` to enable and configure required components. :param config: Add-on configuration :type config: ``dict`` @@ -139,12 +52,21 @@ class Addon(object): :param settings: Crawler settings object :type settings: :class:`~scrapy.settings.Settings` """ - self.export_component(config, settings) - self.export_basics(settings) self.export_config(config, settings) + def check_configuration(self, config, crawler): + """Perform post-initialization checks on fully configured `crawler`. -class AddonManager(Mapping): + :param config: Add-on configuration + :type config: ``dict`` + + :param crawler: the fully-initialized crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + """ + pass + + +class AddonManager(Mapping[str, Addon]): """This class facilitates loading and storing :ref:`topics-addons`. You can treat it like a read-only dictionary in which keys correspond to @@ -160,108 +82,42 @@ class AddonManager(Mapping): """ - def __init__(self): - self._addons = OrderedDict() - self.configs = {} - self._disable_on_add = [] + def __init__(self) -> None: + self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]() + self.configs: Dict[str, Dict[str, Any]] = {} - def __getitem__(self, name): + def __getitem__(self, name: str) -> Addon: return self._addons[name] - def __delitem__(self, name): - del self._addons[name] - del self.configs[name] - - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._addons) - def __len__(self): + def __len__(self) -> int: return len(self._addons) - def add(self, addon, config=None): + def add(self, addon: Any, config: Optional[Dict[str, Any]] = None): """Store an add-on. - If ``addon`` is a string, it will be treated as add-on path and passed - to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object - implementing or providing Scrapy's add-on interface. The interface - will be enforced through ``zope.interface``'s ``verifyObject()``. - - If ``addon`` is a class, it will be instantiated. You can avoid this - (for example if you have implemented the add-on callbacks as class - methods) by declaring -- via ``zope.interface`` -- that your class - directly *provides* ``scrapy.interfaces.IAddon``. - :param addon: The add-on object (or path) to be stored - :type addon: Any Python object providing the add-on interface or ``str`` + :type addon: Python object, class or ``str`` :param config: The add-on configuration dictionary :type config: ``dict`` """ - addon = self.get_addon(addon) - if isclass(addon) and not IAddon.providedBy(addon): + if isinstance(addon, (type, str)): + addon = load_object(addon) + if isinstance(addon, type): addon = addon() - if not IAddon.providedBy(addon): - zope.interface.alsoProvides(addon, IAddon) - # zope.interface's exceptions are already quite helpful. Still, should - # we catch them and log an error message? - verifyObject(IAddon, addon) name = addon.name if name in self: raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} - if name in self._disable_on_add: - self.configs[name]["_enabled"] = False - self._disable_on_add.remove(name) - - def remove(self, addon): - """Remove an add-on. - - If ``addon`` is the name of a stored add-on, that add-on will be - removed. Otherwise, you can use the argument in the same fashion as - in :meth:`add`. - - :param addon: The add-on name, object, or path to be removed - :type addon: Any Python object providing the add-on interface or ``str`` - """ - if addon in self: - del self[addon] - elif hasattr(addon, "name") and addon.name in self: - del self[addon.name] - else: - try: - del self[self.get_addon(addon).name] - except NameError: - raise KeyError - - @staticmethod - def get_addon(path): - """Get an add-on object by its Python or file path. - - ``path`` is assumed to be an import path of an add-on. If the object or - module pointed to by ``path`` has an attribute named ``_addon`` that - attribute will be assumed to be the add-on. :meth:`get_addon` will keep - following ``_addon`` attributes until it finds an object that does not - have an attribute named ``_addon``. - - :param path: Import path of an add-on - :type path: ``str`` - """ - if isinstance(path, str): - try: - obj = load_module_or_object(path) - except NameError: - raise NameError(f"Could not find add-on '{path}'") - else: - obj = path - if hasattr(obj, "_addon"): - obj = AddonManager.get_addon(obj._addon) - return obj def load_settings(self, settings): """Load add-ons and configurations from settings object. - This will invoke :meth:`get_addon` for every add-on path in the + This will load the addon for every add-on path in the ``ADDONS`` setting. For each of these add-ons, the configuration will be read from the dictionary setting whose name matches the uppercase add-on name. @@ -271,141 +127,12 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ paths = build_component_list(settings["ADDONS"]) - addons = [self.get_addon(path) for path in paths] + addons = [load_object(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): self.add(a, c) - def check_dependency_clashes(self) -> None: - """Check for incompatibilities in add-on dependencies. - - Add-ons can provide information about their dependencies in their - ``provides``, ``modifies`` and ``requires`` attributes. This method will - raise an ``ImportError`` if - - * a component required by an add-on is not provided by any other add-on, - or - * a component modified by an add-on is not provided by any other add-on, - or - * the same component is provided by more than one add-on, - - and warn when a component required by an add-on is modified by any other - add-on. - """ - # Collect all active add-ons and the components they provide - versions: Dict[str, Version] = {} - - def add_version(project_name, version): - if project_name in versions: - raise ImportError( - f"Component {project_name} provided by multiple add-ons" - ) - versions[project_name] = Version(version) - - for name in self: - ver = self[name].version - add_version(name, ver) - for provides_name in getattr(self[name], "provides", []): - add_version(provides_name, ver) - - # Collect all required and modified components - def compile_attribute_dict(attribute_name): - attrs = defaultdict(list) - for name in self: - for entry in getattr(self[name], attribute_name, []): - attrs[entry].append(name) - return attrs - - modified = compile_attribute_dict("modifies") - required = compile_attribute_dict("requires") - - req_or_mod = set(required.keys()).union(modified.keys()) - for reqstr in req_or_mod: - req = Requirement(reqstr) - if req.name not in versions or versions[req.name] not in req.specifier: - raise ImportError( - f"Add-ons {required[reqstr] + modified[reqstr]} require" - f" or modify missing component {reqstr}" - ) - - mod_and_req = set(required.keys()).intersection(modified.keys()) - for conflict in mod_and_req: - warnings.warn( - f"Component '{conflict}', required by add-ons {required[conflict]}," - f" is modified by add-ons {modified[conflict]}" - ) - - def disable(self, addon): - """Disable an add-on, i.e. prevent its callbacks from being called. - - If you disable an add-on before it is loaded, it will be disabled as - soon as it is added to the :class:`AddonManager`. - - :param addon: Name of the add-on to be disabled - :type addon: ``str`` - """ - if addon in self: - self.configs[addon]["_enabled"] = False - else: - self._disable_on_add.append(addon) - - def enable(self, addon): - """Re-enable a disabled add-on. - - Will raise ``ValueError`` if the add-on is neither already loaded nor - marked for being disabled on adding. - - :param addon: Name of the add-on to be enabled - :type addon: ``str`` - """ - if addon in self: - self.configs[addon]["_enabled"] = True - elif addon in self._disable_on_add: - self._disable_on_add.remove(addon) - else: - raise ValueError("Add-ons need to be added before they can be enabled") - - @property - def disabled(self): - """Names of disabled add-ons""" - return [ - a for a in self if not self.configs[a].get("_enabled", True) - ] + self._disable_on_add - - @property - def enabled(self): - """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get("_enabled", True)] - - def _call_if_exists(self, obj, cbname, *args, **kwargs): - if obj is None: - return - try: - cb = getattr(obj, cbname) - except AttributeError: - return - else: - cb(*args, **kwargs) - - def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get("_enabled", True): - self._call_if_exists( - self[addonname], cbname, self.configs[addonname], *args, **kwargs - ) - - def update_addons(self): - """Call ``update_addons()`` of all held add-ons. - - This will also call ``update_addons()`` of all add-ons that are added - last minute during the ``update_addons()`` routine of other add-ons. - """ - called_addons = set() - while called_addons != set(self): - for name in set(self).difference(called_addons): - called_addons.add(name) - self._call_addon(name, "update_addons", self) - - def update_settings(self, settings): + def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. :param settings: The :class:`~scrapy.settings.Settings` object to be \ @@ -413,13 +140,13 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, "update_settings", settings) + self[name].update_settings(self.configs[name], settings) - def check_configuration(self, crawler): + def check_configuration(self, crawler) -> None: """Call ``check_configuration()`` of all held add-ons. :param crawler: the fully-initialized crawler :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, "check_configuration", crawler) + self[name].check_configuration(self.configs[name], crawler) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 1b579f10e..efc9b36ea 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,6 @@ import sys from importlib.metadata import entry_points import scrapy -from scrapy.addons import AddonManager from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError @@ -131,8 +130,6 @@ def execute(argv=None, settings=None): else: settings["EDITOR"] = editor - addons = AddonManager() - inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) @@ -156,7 +153,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings, addons=addons) + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0c9861dbd..a0cb368ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -58,7 +58,6 @@ class Crawler: spidercls: Type[Spider], settings: Union[None, dict, Settings] = None, init_reactor: bool = False, - addons: Optional[AddonManager] = None, ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") @@ -70,10 +69,8 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = addons if addons is not None else AddonManager() + self.addons: AddonManager = AddonManager() self.addons.load_settings(self.settings) - self.addons.update_addons() - self.addons.check_dependency_clashes() self.addons.update_settings(self.settings) self.signals: SignalManager = SignalManager(self) @@ -200,15 +197,10 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__( - self, - settings: Union[Dict[str, Any], Settings, None] = None, - addons: Optional[AddonManager] = None, - ): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.addons: Optional[AddonManager] = addons self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() self._active: Set[defer.Deferred] = set() @@ -292,7 +284,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, addons=self.addons) + return Crawler(spidercls, self.settings) def stop(self): """ @@ -338,13 +330,8 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__( - self, - settings=None, - install_root_handler: bool = True, - addons: Optional[AddonManager] = None, - ): - super().__init__(settings, addons) + def __init__(self, settings=None, install_root_handler=True): + super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index b8aa77ced..9a2c5f170 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,4 +1,4 @@ -from zope.interface import Attribute, Interface +from zope.interface import Interface class ISpiderLoader(Interface): @@ -15,22 +15,3 @@ class ISpiderLoader(Interface): def find_by_request(request): """Return the list of spiders names that can handle the given request""" - - -class IAddon(Interface): - """Scrapy add-on""" - - name = Attribute("""Add-on name""") - version = Attribute("""Add-on version string (PEP440)""") - - # XXX: Can methods be declared optional? I.e., can I enforce the signature - # but not the existence of a method? - - # def update_addons(config, addons): - # """Enables and configures other add-ons""" - - # def update_settings(config, settings): - # """Modifies `settings` to enable and configure required components""" - - # def check_configuration(config, crawler): - # """Performs post-initialization checks on fully configured `crawler`""" diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 8d1544c68..3ade1d105 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -99,18 +99,12 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def config_from_filepath(sources): - """Create a ConfigParser and read in the given `sources`, which can be - either a filename or a list of filenames.""" - cfg = ConfigParser() - cfg.read(sources) - return cfg - - def get_config(use_closest=True): """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) - return config_from_filepath(sources) + cfg = ConfigParser() + cfg.read(sources) + return cfg def get_sources(use_closest=True) -> List[str]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4e23b01c3..b3c28da92 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -67,7 +67,7 @@ def load_object(path: Union[str, Callable]) -> Any: if callable(path): return path raise TypeError( - "Unexpected argument type, expected string " f"or object, got: {type(path)}" + f"Unexpected argument type, expected string or object, got: {type(path)}" ) try: @@ -86,22 +86,6 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def load_module_or_object(path): - """Load python module or (non-module) object from given path. - - Path can be both a Python or a file path. - """ - try: - return import_module(path) - except ImportError: - pass - try: - return load_object(path) - except (ValueError, NameError, ImportError): - pass - raise NameError(f"Could not load '{path}'") - - def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fe26e1708..97de8d25a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -73,7 +73,6 @@ def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, prevent_warnings: bool = True, - addons=None, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -87,7 +86,7 @@ def get_crawler( if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) - runner = CrawlerRunner(settings, addons) + runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_addons.py b/tests/test_addons.py new file mode 100644 index 000000000..ecdc0426c --- /dev/null +++ b/tests/test_addons.py @@ -0,0 +1,72 @@ +import unittest + +from scrapy.addons import Addon, AddonManager +from scrapy.settings import BaseSettings + + +class GoodAddon(object): + name = "GoodAddon" + + def update_settings(self, config, settings): + pass + + def check_configuration(self, config, crawler): + pass + + +class AddonTest(unittest.TestCase): + def setUp(self): + class AddonWithAttributes(Addon): + name = "Test" + + self.testaddon = AddonWithAttributes() + + def test_export_config(self): + settings = BaseSettings() + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["KEY"], 313) + self.assertEqual(settings["DEFAULTKEY"], 100) + self.assertEqual(settings["OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("KEY"), 15) + + def test_update_settings(self): + settings = BaseSettings() + settings.set("KEY1", "default", priority="default") + settings.set("KEY2", "project", priority="project") + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} + self.testaddon.update_settings(addon_config, settings) + self.assertEqual(settings["KEY1"], "addon") + self.assertEqual(settings["KEY2"], "project") + self.assertEqual(settings["KEY3"], "addon") + + +class AddonManagerTest(unittest.TestCase): + def setUp(self): + self.manager = AddonManager() + + def test_add(self): + manager = AddonManager() + manager.add("tests.test_addons.GoodAddon") + self.assertCountEqual(manager, ["GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], GoodAddon) + + def test_load_settings(self): + settings = BaseSettings() + settings.set( + "ADDONS", + {"tests.test_addons.GoodAddon": 0}, + ) + settings.set("GOODADDON", {"key": "val2"}) + manager = AddonManager() + manager.load_settings(settings) + self.assertCountEqual(manager, ["GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py deleted file mode 100644 index fa3f706e5..000000000 --- a/tests/test_addons/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -import itertools -import unittest -import warnings -from unittest import mock - -from zope.interface import directlyProvides -from zope.interface.exceptions import BrokenImplementation, MultipleInvalid -from zope.interface.verify import verifyObject - -from scrapy.addons import Addon, AddonManager -from scrapy.crawler import Crawler -from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings - -from . import addonmod, addons - - -class AddonTest(unittest.TestCase): - def setUp(self): - self.rawaddon = Addon() - - class AddonWithAttributes(Addon): - name = "Test" - version = "1.0" - - self.testaddon = AddonWithAttributes() - - def test_interface(self): - # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, "name")) - self.assertFalse(hasattr(self.rawaddon, "version")) - self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) - verifyObject(IAddon, self.testaddon) - - def test_export_component(self): - settings = BaseSettings( - {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, - "default", - ) - self.testaddon.component_type = None - self.testaddon.export_component({}, settings) - self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) - self.testaddon.component_type = "ITEM_PIPELINES" - self.testaddon.component = "test.component" - self.testaddon.export_component({}, settings) - self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) - self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) - self.testaddon.component_order = 313 - self.testaddon.export_component({}, settings) - self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) - self.testaddon.component_type = "DOWNLOAD_HANDLERS" - self.testaddon.component_key = "http" - self.testaddon.export_component({}, settings) - self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") - - def test_export_basics(self): - settings = BaseSettings() - self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} - self.testaddon.export_basics(settings) - self.assertEqual(settings["TESTKEY"], 313) - self.assertEqual(settings["OTHERKEY"], True) - self.assertEqual(settings.getpriority("TESTKEY"), 15) - - def test_export_config(self): - settings = BaseSettings() - self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} - self.testaddon.default_config = {"key": 55, "defaultkey": 100} - self.testaddon.export_config( - {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings - ) - self.assertEqual(settings["TEST_KEY"], 313) - self.assertEqual(settings["TEST_DEFAULTKEY"], 100) - self.assertEqual(settings["TEST_OTHERKEY"], True) - self.assertNotIn("MAPPED_key", settings) - self.assertNotIn("MAPPED_KEY", settings) - self.assertEqual(settings["MAPPING_WORKED"], 99) - self.assertEqual(settings.getpriority("TEST_KEY"), 15) - - self.testaddon.settings_prefix = "PREF" - self.testaddon.export_config({"newkey": 99}, settings) - self.assertEqual(settings["PREF_NEWKEY"], 99) - - with mock.patch.object(settings, "set") as mock_set: - self.testaddon.settings_prefix = False - self.testaddon.export_config({"thirdnewkey": 99}, settings) - self.assertEqual(mock_set.call_count, 0) - - def test_update_settings(self): - settings = BaseSettings() - settings.set("TEST_KEY1", "default", priority="default") - settings.set("TEST_KEY2", "project", priority="project") - self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} - addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} - self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings["OTHERTEST_KEY"], "addon") - self.assertEqual(settings["TEST_KEY1"], "addon") - self.assertEqual(settings["TEST_KEY2"], "project") - self.assertEqual(settings["TEST_KEY3"], "addon") - - -class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - - def test_add(self): - manager = AddonManager() - manager.add(addonmod, {"key": "val1"}) - manager.add("tests.test_addons.addons.GoodAddon") - self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["AddonModule"], ["key"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - self.assertRaises(ValueError, manager.add, addonmod) - - def test_add_dont_instantiate_providing_classes(self): - class ProviderGoodAddon(addons.GoodAddon): - pass - - directlyProvides(ProviderGoodAddon, IAddon) - manager = AddonManager() - manager.add(ProviderGoodAddon) - self.assertIs(manager["GoodAddon"], ProviderGoodAddon) - - def test_add_verifies(self): - brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") - self.assertRaises( - BrokenImplementation, - self.manager.add, - brokenaddon, - ) - - def test_add_adds_missing_interface_declaration(self): - class GoodAddonWithoutDeclaration(object): - name = "GoodAddonWithoutDeclaration" - version = "1.0" - - self.manager.add(GoodAddonWithoutDeclaration) - - def test_remove(self): - manager = AddonManager() - - def test_gets_removed(removearg): - manager.add(addonmod) - self.assertIn("AddonModule", manager) - manager.remove(removearg) - self.assertNotIn("AddonModule", manager) - - test_gets_removed("AddonModule") - test_gets_removed(addonmod) - test_gets_removed("tests.test_addons.addonmod") - self.assertRaises(KeyError, manager.remove, "nonexistent") - self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) - - def test_get_addon(self): - goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") - self.assertIs(goodaddon, addons.GoodAddon) - - loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") - self.assertIs(loaded_addonmod, addonmod) - - goodaddon = self.manager.get_addon("tests.test_addons.addons") - self.assertIsInstance(goodaddon, addons.GoodAddon) - - self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") - - def test_get_addon_forward(self): - class SomeCls(object): - _addon = "tests.test_addons.addons.GoodAddon" - - self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) - - def test_get_addon_nested(self): - x = addons.GoodAddon("outer") - x._addon = addons.GoodAddon("middle") - x._addon._addon = addons.GoodAddon("inner") - self.assertIs(self.manager.get_addon(x), x._addon._addon) - - def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, - ) - settings.set("ADDONMODULE", {"key": "val1"}) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) - self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") - self.assertEqual(manager["AddonModule"], addonmod) - self.assertIn("key", manager.configs["AddonModule"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - - def test_load_settings_order(self): - # Get three addons named 0, 1, 2 - addonlist = [addons.GoodAddon(str(x)) for x in range(3)] - # Test for every possible ordering - for ordered_addons in itertools.permutations(addonlist): - expected_order = [a.name for a in ordered_addons] - settings = BaseSettings( - {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} - ) - manager = AddonManager() - manager.load_settings(settings) - self.assertEqual(list(manager.keys()), expected_order) - - def test_enabled_disabled(self): - manager = AddonManager() - manager.add(addons.GoodAddon("FirstAddon")) - manager.add(addons.GoodAddon("SecondAddon")) - self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) - self.assertEqual(manager.disabled, []) - manager.disable("FirstAddon") - self.assertEqual(manager.enabled, ["SecondAddon"]) - self.assertEqual(manager.disabled, ["FirstAddon"]) - manager.enable("FirstAddon") - self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) - self.assertEqual(manager.disabled, []) - - def test_enable_before_add(self): - manager = AddonManager() - self.assertRaises(ValueError, manager.enable, "FirstAddon") - manager.disable("FirstAddon") - manager.enable("FirstAddon") - manager.add(addons.GoodAddon("FirstAddon")) - self.assertIn("FirstAddon", manager.enabled) - - def test_disable_before_add(self): - manager = AddonManager() - manager.disable("FirstAddon") - manager.add(addons.GoodAddon("FirstAddon")) - self.assertEqual(manager.disabled, ["FirstAddon"]) - - def test_callbacks(self): - first_addon = addons.GoodAddon("FirstAddon") - second_addon = addons.GoodAddon("SecondAddon") - - manager = AddonManager() - manager.add(first_addon, {"test": "first"}) - manager.add(second_addon, {"test": "second"}) - crawler = mock.create_autospec(Crawler) - settings = BaseSettings() - - with mock.patch.object( - first_addon, "update_addons" - ) as ua_first, mock.patch.object( - second_addon, "update_addons" - ) as ua_second, mock.patch.object( - first_addon, "update_settings" - ) as us_first, mock.patch.object( - second_addon, "update_settings" - ) as us_second, mock.patch.object( - first_addon, "check_configuration" - ) as cc_first, mock.patch.object( - second_addon, "check_configuration" - ) as cc_second: - manager.update_addons() - ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) - ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) - manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) - us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) - manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) - cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) - self.assertEqual(ua_first.call_count, 1) - self.assertEqual(ua_second.call_count, 1) - self.assertEqual(us_first.call_count, 1) - self.assertEqual(us_second.call_count, 1) - - us_first.reset_mock() - us_second.reset_mock() - manager.disable("FirstAddon") - manager.update_settings(settings) - self.assertEqual(us_first.call_count, 0) - manager.enable("FirstAddon") - manager.update_settings(settings) - self.assertEqual(us_first.call_count, 1) - self.assertEqual(us_second.call_count, 2) - - # This will become relevant when we let spiders implement the add-on - # interface and should be replaced with a test where - # AddonManager.spidercls = None then. - manager._call_if_exists(None, "irrelevant") - - def test_update_addons_last_minute_add(self): - class AddedAddon(addons.GoodAddon): - name = "AddedAddon" - - class FirstAddon(addons.GoodAddon): - name = "FirstAddon" - - def update_addons(self, config, addons): - addons.add(AddedAddon()) - - manager = AddonManager() - first_addon = FirstAddon() - with mock.patch.object( - first_addon, "update_addons", wraps=first_addon.update_addons - ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: - manager.add(first_addon, {"non-empty": "dict"}) - manager.update_addons() - self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) - ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) - ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) - - def test_check_dependency_clashes_attributes(self): - provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ("test",) - provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ("test",) - requires = addons.GoodAddon("RequiresAddon") - requires.requires = ("test",) - requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ("ProvidesAddon",) - requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ("test>=2.0",) - modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ("test",) - - def check_with(*addons): - manager = AddonManager() - for a in addons: - manager.add(a) - return manager.check_dependency_clashes() - - self.assertRaises(ImportError, check_with, requires) - self.assertRaises(ImportError, check_with, modifies) - self.assertRaises(ImportError, check_with, provides, provides2) - self.assertRaises(ImportError, check_with, provides, requires_newer) - with warnings.catch_warnings(record=True) as w: - check_with(provides, modifies) - check_with(provides) - check_with(provides, requires) - check_with(provides, requires_name) - self.assertEqual(len(w), 0) - check_with(requires, provides, modifies) - self.assertEqual(len(w), 1) diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py deleted file mode 100644 index 092c3c0eb..000000000 --- a/tests/test_addons/addonmod.py +++ /dev/null @@ -1,16 +0,0 @@ -from zope.interface import moduleProvides - -from scrapy.interfaces import IAddon - -moduleProvides(IAddon) - -name = "AddonModule" -version = "1.0" - - -def update_settings(config, settings): - pass - - -def check_configuration(config, crawler): - pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py deleted file mode 100644 index d878f37ea..000000000 --- a/tests/test_addons/addons.py +++ /dev/null @@ -1,33 +0,0 @@ -from zope.interface import implementer - -from scrapy.interfaces import IAddon - - -@implementer(IAddon) -class GoodAddon(object): - name = "GoodAddon" - version = "1.0" - - def __init__(self, name=None, version=None): - if name is not None: - self.name = name - if version is not None: - self.version = version - - def update_addons(self, config, addons): - pass - - def update_settings(self, config, settings): - pass - - def check_configuration(self, config, crawler): - pass - - -@implementer(IAddon) -class BrokenAddon(object): - name = "BrokenAddon" - # No version - - -_addon = GoodAddon() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 920e5f4ae..d844a645f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -412,7 +412,6 @@ with multiples lines def test_abort_on_addon_failed_check(self): class FailedCheckAddon(Addon): name = "FailedCheckAddon" - version = "1.0" def check_configuration(self, config, crawler): raise ValueError diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ed34d9e58..d54a2cb7e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -15,7 +15,6 @@ from twisted.trial import unittest from w3lib import __version__ as w3lib_version import scrapy -from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet @@ -56,32 +55,6 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) - def test_populate_addons_settings(self): - class TestAddon(Addon): - name = "TestAddon" - version = "1.0" - - addonconfig = {"TEST1": "addon", "TEST2": "addon", "TEST3": "addon"} - - class TestAddon2(Addon): - name = "testAddon2" - version = "1.0" - - addonconfig2 = {"TEST": "addon2"} - - settings = Settings() - settings.set("TESTADDON_TEST1", "project", priority="project") - settings.set("TESTADDON_TEST2", "default", priority="default") - addonmgr = AddonManager() - addonmgr.add(TestAddon(), addonconfig) - addonmgr.add(TestAddon2(), addonconfig2) - crawler = Crawler(DefaultSpider, settings, addons=addonmgr) - - self.assertEqual(crawler.settings["TESTADDON_TEST1"], "project") - self.assertEqual(crawler.settings["TESTADDON_TEST2"], "addon") - self.assertEqual(crawler.settings["TESTADDON_TEST3"], "addon") - self.assertEqual(crawler.settings["TESTADDON2_TEST"], "addon2") - def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) self.assertEqual(crawler.settings["foo"], "bar") diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 7932ca04c..69793ee75 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,7 +8,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, - load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -36,12 +35,6 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) - def test_load_module_or_object(self): - testmod = load_module_or_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ diff --git a/tests/test_utils_misc/testmod.py b/tests/test_utils_misc/testmod.py deleted file mode 100644 index eb540335f..000000000 --- a/tests/test_utils_misc/testmod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR = True From 760c0db094b3147236294f656f4168c16502de97 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 26 Jun 2023 17:15:13 +0400 Subject: [PATCH 132/211] Fix typing on 3.8. --- scrapy/addons.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index a54086fda..523c36e9c 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,5 +1,4 @@ -from collections.abc import Mapping -from typing import Any, Dict, Iterator, Optional, OrderedDict +from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_object From b6196309cb654e6662197e19de36a9d18a83f12f Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 28 Jun 2023 03:28:49 -0300 Subject: [PATCH 133/211] fix: Return value instead of `SettingsAttribute` object when using `pop` method (#5963) --- scrapy/settings/__init__.py | 4 ++-- tests/test_settings/__init__.py | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 8b3bdbabe..cc44d67e8 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -449,12 +449,12 @@ class BaseSettings(MutableMapping): def pop(self, name, default=__default): try: - value = self.attributes[name] + value = self.attributes[name].value except KeyError: if default is self.__default: raise - return SettingsAttribute(default, get_settings_priority("project")) + return default else: self.__delitem__(name) return value diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index bb6dc67fa..5fc825393 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -457,19 +457,15 @@ class SettingsTest(unittest.TestCase): with self.assertRaises(KeyError): settings.pop("DUMMY_CONFIG") - dummy_config = settings.pop("DUMMY_CONFIG", "dummy_value") - - self.assertEqual( - repr(dummy_config), "" - ) - self.assertEqual(dummy_config.value, "dummy_value") + dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") + self.assertEqual(dummy_config_value, "dummy_value") def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) - self.assertEqual(settings.pop("DUMMY_CONFIG").value, "dummy_value") + self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value") settings.freeze() From 9612ae3e93239b86cedcd124073de6fff2736e99 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Jun 2023 12:41:33 +0400 Subject: [PATCH 134/211] Remove more code. --- docs/topics/addons.rst | 100 ++++----------------------------- scrapy/addons.py | 124 ++++------------------------------------- tests/test_addons.py | 48 +++++----------- tests/test_crawl.py | 16 ------ 4 files changed, 37 insertions(+), 251 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 6a9590f33..3421864fa 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -17,40 +17,26 @@ Activating and configuring add-ons Add-ons and their configuration live in Scrapy's :class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` initialization the add-on manager will read a list of enabled add-ons from your -``ADDONS`` setting and their optional configuration from the respective -settings. +``ADDONS`` setting. The ``ADDONS`` setting is a dict in which every key is an addon class or its import path and the value is its priority. -The configuration of an add-on, if necessary at all, is stored as a dictionary -setting whose name is the uppercase add-on name. - -This is an example where two add-ons (in this case with one requiring no -configuration) are enabled/configured in a project's ``settings.py``:: +This is an example where two add-ons are enabled in a project's +``settings.py``:: ADDONS = { 'path.to.someaddon': 0, path.to.someaddon2: 1, } - SOMEADDON = { - 'some_config': True, - } - Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: +Add-ons are (any) Python *objects* that include the following method: -.. attribute:: name - - string with add-on name - - :type: ``str`` - -.. method:: update_settings(config, settings) +.. method:: update_settings(settings) This method is called during the initialization of the :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks @@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: :class:`~scrapy.settings.Settings` object as wished, e.g. enable components for this add-on or set required configuration of other extensions. - :param config: Configuration of this add-on - :type config: ``dict`` - :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` -.. method:: check_configuration(config, crawler) - - This method is called when the :class:`~scrapy.crawler.Crawler` has been - fully initialized, immediately before it starts crawling. You can perform - additional dependency and configuration checks here. - - :param config: Configuration of this add-on - :type config: ``dict`` - - :param crawler: Fully initialized Scrapy crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - - -Add-on base class -================= - -Scrapy comes with a built-in base class for add-ons which provides some -convenience functionality: the add-on configuration can be exposed into -Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable -via :attr:`~scrapy.addons.Addon.default_config` and -:attr:`~scrapy.addons.Addon.config_mapping`. - -By default, the base add-on class will expose the add-on configuration into -Scrapy's settings namespace, in upper case. It is -easy to write your own functionality while still being able to use the -convenience functions by overwriting -:meth:`~scrapy.addons.Addon.update_settings`. - -.. module:: scrapy.addons - :noindex: - -.. autoclass:: Addon - :members: - Add-on examples =============== -Set some basic configuration using the :class:`Addon` base class:: +Set some basic configuration:: - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - - def update_settings(self, config, settings): - super().update_settings(settings) + class MyAddon: + def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings["DNSCACHE_ENABLED"] = True Check dependencies:: - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - - def update_settings(self, config, settings): + class MyAddon: + def update_settings(self, settings): try: import boto except ImportError: - raise RuntimeError("myaddon requires the boto library") - super().update_settings(settings) - -Check configuration of fully initialized crawler (see -:ref:`topics-api-crawler`):: - - class MyAddon(object): - name = 'myaddon' - - def update_settings(self, config, settings): - super().update_settings(settings) - settings.set('DNSCACHE_ENABLED', False, priority='addon') - - def check_configuration(self, config, crawler): - if crawler.settings.getbool('DNSCACHE_ENABLED'): - # The spider, some other add-on, or the user messed with the - # DNS cache setting - raise ValueError("myaddon is incompatible with DNS cache") + raise RuntimeError("MyAddon requires the boto library") + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index 523c36e9c..bb4664d8e 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,100 +1,16 @@ -from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict +from typing import Any, List from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_object -class Addon(object): - name: str - - default_config = None - """``dict`` with default configuration.""" - - config_mapping = None - """``dict`` with mappings from config names to setting names. The given - setting names will be taken as given, not uppercased. - """ - - def export_config(self, config, settings): - """Export the add-on configuration, all keys in caps, into the settings - object. - - For example, the add-on configuration ``{'key': 'value'}`` will export - the setting ``KEY`` with a value of ``value``. All settings - will be exported with ``addon`` priority (see - :ref:`topics-api-settings`). - - :param config: Add-on configuration to be exposed - :type config: ``dict`` - - :param settings: Settings object into which to export the configuration - :type settings: :class:`~scrapy.settings.Settings` - """ - conf = self.default_config or {} - conf.update(config) - # Since default exported config is case-insensitive (everything will be - # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} - for key, val in conf.items(): - if key.lower() in conf_mapping: - key = conf_mapping[key.lower()] - else: - key = key.upper() - settings.set(key, val, "addon") - - def update_settings(self, config, settings): - """Modifiy `settings` to enable and configure required components. - - :param config: Add-on configuration - :type config: ``dict`` - - :param settings: Crawler settings object - :type settings: :class:`~scrapy.settings.Settings` - """ - self.export_config(config, settings) - - def check_configuration(self, config, crawler): - """Perform post-initialization checks on fully configured `crawler`. - - :param config: Add-on configuration - :type config: ``dict`` - - :param crawler: the fully-initialized crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - """ - pass - - -class AddonManager(Mapping[str, Addon]): - """This class facilitates loading and storing :ref:`topics-addons`. - - You can treat it like a read-only dictionary in which keys correspond to - add-on names and values correspond to the add-on objects. Add-on - configurations are saved in the :attr:`config` dictionary attribute:: - - addons = AddonManager() - # ... load some add-ons here - print(addons.enabled) # prints names of all enabled add-ons - print(addons['TestAddon'].version) # prints version of add-on with name - # 'TestAddon' - print(addons.configs['TestAddon']) # prints configuration of 'TestAddon' - - """ +class AddonManager: + """This class facilitates loading and storing :ref:`topics-addons`.""" def __init__(self) -> None: - self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]() - self.configs: Dict[str, Dict[str, Any]] = {} + self.addons: List[Any] = [] - def __getitem__(self, name: str) -> Addon: - return self._addons[name] - - def __iter__(self) -> Iterator[str]: - return iter(self._addons) - - def __len__(self) -> int: - return len(self._addons) - - def add(self, addon: Any, config: Optional[Dict[str, Any]] = None): + def add(self, addon: Any) -> None: """Store an add-on. :param addon: The add-on object (or path) to be stored @@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]): addon = load_object(addon) if isinstance(addon, type): addon = addon() - name = addon.name - if name in self: - raise ValueError(f"Addon '{name}' already loaded") - self._addons[name] = addon - self.configs[name] = config or {} + self.addons.append(addon) - def load_settings(self, settings): + def load_settings(self, settings) -> None: """Load add-ons and configurations from settings object. This will load the addon for every add-on path in the - ``ADDONS`` setting. For each of these add-ons, the configuration will be - read from the dictionary setting whose name matches the uppercase add-on - name. + ``ADDONS`` setting. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration @@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]): """ paths = build_component_list(settings["ADDONS"]) addons = [load_object(path) for path in paths] - configs = [settings.getdict(addon.name.upper()) for addon in addons] - for a, c in zip(addons, configs): - self.add(a, c) + for a in addons: + self.add(a) def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. @@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]): updated :type settings: :class:`~scrapy.settings.Settings` """ - for name in self: - self[name].update_settings(self.configs[name], settings) - - def check_configuration(self, crawler) -> None: - """Call ``check_configuration()`` of all held add-ons. - - :param crawler: the fully-initialized crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - """ - for name in self: - self[name].check_configuration(self.configs[name], crawler) + for addon in self.addons: + addon.update_settings(settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index ecdc0426c..8ba27236d 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,47 +1,29 @@ import unittest +from typing import Any, Dict, Optional -from scrapy.addons import Addon, AddonManager +from scrapy.addons import AddonManager from scrapy.settings import BaseSettings -class GoodAddon(object): +class GoodAddon: name = "GoodAddon" - def update_settings(self, config, settings): - pass + def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: + super().__init__() + self.config = config or {} - def check_configuration(self, config, crawler): - pass + def update_settings(self, settings): + settings.update(self.config, "addon") class AddonTest(unittest.TestCase): - def setUp(self): - class AddonWithAttributes(Addon): - name = "Test" - - self.testaddon = AddonWithAttributes() - - def test_export_config(self): - settings = BaseSettings() - self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} - self.testaddon.default_config = {"key": 55, "defaultkey": 100} - self.testaddon.export_config( - {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings - ) - self.assertEqual(settings["KEY"], 313) - self.assertEqual(settings["DEFAULTKEY"], 100) - self.assertEqual(settings["OTHERKEY"], True) - self.assertNotIn("MAPPED_key", settings) - self.assertNotIn("MAPPED_KEY", settings) - self.assertEqual(settings["MAPPING_WORKED"], 99) - self.assertEqual(settings.getpriority("KEY"), 15) - def test_update_settings(self): settings = BaseSettings() settings.set("KEY1", "default", priority="default") settings.set("KEY2", "project", priority="project") - addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} - self.testaddon.update_settings(addon_config, settings) + addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} + testaddon = GoodAddon(addon_config) + testaddon.update_settings(settings) self.assertEqual(settings["KEY1"], "addon") self.assertEqual(settings["KEY2"], "project") self.assertEqual(settings["KEY3"], "addon") @@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase): def test_add(self): manager = AddonManager() manager.add("tests.test_addons.GoodAddon") - self.assertCountEqual(manager, ["GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], GoodAddon) + self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): settings = BaseSettings() @@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase): settings.set("GOODADDON", {"key": "val2"}) manager = AddonManager() manager.load_settings(settings) - self.assertCountEqual(manager, ["GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertIsInstance(manager.addons[0], GoodAddon) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d844a645f..ca9084294 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -13,7 +13,6 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy import signals -from scrapy.addons import Addon, AddonManager from scrapy.crawler import CrawlerRunner from scrapy.exceptions import StopDownload from scrapy.http import Request @@ -408,21 +407,6 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) - @defer.inlineCallbacks - def test_abort_on_addon_failed_check(self): - class FailedCheckAddon(Addon): - name = "FailedCheckAddon" - - def check_configuration(self, config, crawler): - raise ValueError - - addonmgr = AddonManager() - addonmgr.add(FailedCheckAddon()) - crawler = get_crawler(SimpleSpider) - crawler.addons = addonmgr - with self.assertRaises(ValueError): - yield crawler.crawl() - class CrawlSpiderTestCase(TestCase): def setUp(self): From f1ed5598f4d312337a9dd76e0c1a5856cabea8bc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Jun 2023 19:13:46 +0400 Subject: [PATCH 135/211] Remove the check_configuration call. --- scrapy/crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a0cb368ed..7a26cd2e4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -133,7 +133,6 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() - self.addons.check_configuration(self) start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) From c92c9af075217c2296af364a4d9ea55463dbe7f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Jun 2023 15:04:46 +0400 Subject: [PATCH 136/211] Add create_instance support to addons. --- docs/topics/addons.rst | 30 +++++++++++++++++++++++++- scrapy/addons.py | 12 +++++++---- scrapy/crawler.py | 2 +- tests/test_addons.py | 48 +++++++++++++++++++++++++++++------------- 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 3421864fa..f1cc070ad 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that include the following method: +Add-ons are (any) Python objects that include the following method: .. method:: update_settings(settings) @@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an addon instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the addon. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for pipeline to + access them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this addon + :type crawler: :class:`~scrapy.crawler.Crawler` + Add-on examples =============== @@ -67,3 +81,17 @@ Check dependencies:: except ImportError: raise RuntimeError("MyAddon requires the boto library") ... + +Access the crawler instance:: + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index bb4664d8e..ba33f1865 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,17 @@ -from typing import Any, List +from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self) -> None: + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler self.addons: List[Any] = [] def add(self, addon: Any) -> None: @@ -22,7 +26,7 @@ class AddonManager: if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): - addon = addon() + addon = create_instance(addon, settings=None, crawler=self.crawler) self.addons.append(addon) def load_settings(self, settings) -> None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7a26cd2e4..12256440b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = AddonManager() + self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) self.addons.update_settings(self.settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index 8ba27236d..d52665869 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,13 +1,12 @@ import unittest from typing import Any, Dict, Optional -from scrapy.addons import AddonManager +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings +from scrapy.utils.test import get_crawler class GoodAddon: - name = "GoodAddon" - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: super().__init__() self.config = config or {} @@ -16,6 +15,20 @@ class GoodAddon: settings.update(self.config, "addon") +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + class AddonTest(unittest.TestCase): def test_update_settings(self): settings = BaseSettings() @@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - def test_add(self): - manager = AddonManager() + crawler = get_crawler() + manager = crawler.addons manager.add("tests.test_addons.GoodAddon") self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.GoodAddon": 0}, - ) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) + settings_dict = { + "ADDONS": {"tests.test_addons.GoodAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons self.assertIsInstance(manager.addons[0], GoodAddon) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") From d5f74c72247e8fd4775e380569e0b809ad1cc6b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Jun 2023 20:43:46 +0400 Subject: [PATCH 137/211] Log the enabled addons. --- scrapy/addons.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/addons.py b/scrapy/addons.py index ba33f1865..612c7effc 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,3 +1,4 @@ +import logging from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list @@ -6,6 +7,8 @@ from scrapy.utils.misc import create_instance, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler +logger = logging.getLogger(__name__) + class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" @@ -43,6 +46,13 @@ class AddonManager: addons = [load_object(path) for path in paths] for a in addons: self.add(a) + logger.info( + "Enabled addons:\n%(addons)s", + { + "addons": addons, + }, + extra={"crawler": self.crawler}, + ) def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. From 7ce3d8f98ad6b28d138757bdc5655d8c91ffd042 Mon Sep 17 00:00:00 2001 From: Anderson Carlos Ferreira da Silva Date: Wed, 5 Jul 2023 16:56:34 +0900 Subject: [PATCH 138/211] removing hard code entries --- scrapy/utils/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index ab1b8e3ee..652b74759 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -12,8 +12,8 @@ DATADIR_CFG_SECTION = "datadir" def inside_project(): - scrapy_module = os.environ.get("SCRAPY_SETTINGS_MODULE") - if scrapy_module is not None: + scrapy_module = os.environ.get(ENVVAR) + if scrapy_module: try: import_module(scrapy_module) except ImportError as exc: From a2264d3b8b70455f0ed481a9e76abc96056bc546 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 18:57:27 +0400 Subject: [PATCH 139/211] Improve docs about setting settings in addons. --- docs/topics/addons.rst | 10 +++++++++- docs/topics/api.rst | 1 + docs/topics/settings.rst | 15 +++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index f1cc070ad..901a8bf8f 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -61,6 +61,14 @@ They can also have the following method: :param crawler: The crawler that uses this addon :type crawler: :class:`~scrapy.crawler.Crawler` +The settings set by the addon should use the ``addon`` priority (see +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`). This +allows users to override these settings in the project or spider configuration. +This is not possible with settings that are mutable objects, such as the dict +that is a value of :setting:`ITEM_PIPELINES`. In these cases you can provide an +addon-specific setting that governs whether the addon will modify +:setting:`ITEM_PIPELINES`. + Add-on examples =============== @@ -70,7 +78,7 @@ Set some basic configuration:: class MyAddon: def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 - settings["DNSCACHE_ENABLED"] = True + settings.set("DNSCACHE_ENABLED", True, "addon") Check dependencies:: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index bb46b2b7d..d1a5497fb 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -137,6 +137,7 @@ Settings API SETTINGS_PRIORITIES = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 143002360..139e0a35f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,8 +40,9 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Default settings per-command - 5. Default global settings (less precedence) + 4. Settings set by addons + 5. Default settings per-command + 6. Default global settings (less precedence) The population of these settings sources is taken care of internally, but a manual handling is possible using API calls. See the @@ -89,7 +90,13 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Default settings per-command +4. Settings set by addons +------------------------- + +:ref:`Addons ` can modify settings. They should do this with +this priority, though this is not enforced. + +5. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -97,7 +104,7 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +6. Default global settings -------------------------- The global defaults are located in the ``scrapy.settings.default_settings`` From 93962ebefc89e40aea71ef80954b7dce1545ef56 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 22:09:02 +0400 Subject: [PATCH 140/211] Bump typing package versions. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 223ba4258..fe7df9782 100644 --- a/tox.ini +++ b/tox.ini @@ -33,13 +33,13 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.2.0 + mypy==1.4.1 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==9.5.0.2 - types-Pygments==2.15.0.0 - types-pyOpenSSL==23.1.0.2 - types-setuptools==67.7.0.1 + types-Pillow==10.0.0.1 + types-Pygments==2.15.0.1 + types-pyOpenSSL==23.2.0.1 + types-setuptools==68.0.0.1 commands = mypy --show-error-codes {posargs: scrapy tests} From 187e8f9a2d2fbf2c84fb5b39c25e7e7fc155ced0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 00:01:36 +0400 Subject: [PATCH 141/211] Typing for scrapy/settings/__init__.py. --- scrapy/crawler.py | 2 +- scrapy/settings/__init__.py | 166 ++++++++++++++++++++++---------- scrapy/utils/conf.py | 1 + tests/test_settings/__init__.py | 21 ++++ tox.ini | 1 + 5 files changed, 140 insertions(+), 51 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 69ff07bb7..192541dd0 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -55,7 +55,7 @@ class Crawler: def __init__( self, spidercls: Type[Spider], - settings: Union[None, dict, Settings] = None, + settings: Union[None, Dict[str, Any], Settings] = None, init_reactor: bool = False, ): if isinstance(spidercls, Spider): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index cc44d67e8..658c27f0a 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,12 +1,41 @@ +from __future__ import annotations + import copy import json -from collections.abc import MutableMapping from importlib import import_module from pprint import pformat +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + MutableMapping, + Optional, + Tuple, + Union, + cast, +) from scrapy.settings import default_settings -SETTINGS_PRIORITIES = { +# The key types are restricted in BaseSettings._get_key() to ones supported by JSON, +# see https://github.com/scrapy/scrapy/issues/5383. +_SettingsKeyT = Union[bool, float, int, str, None] + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + # https://github.com/python/typing/issues/445#issuecomment-1131458824 + from _typeshed import SupportsItems + from typing_extensions import Self + + _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] + + +SETTINGS_PRIORITIES: Dict[str, int] = { "default": 0, "command": 10, "project": 20, @@ -15,7 +44,7 @@ SETTINGS_PRIORITIES = { } -def get_settings_priority(priority): +def get_settings_priority(priority: Union[int, str]) -> int: """ Small helper function that looks up a given string priority in the :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its @@ -34,14 +63,15 @@ class SettingsAttribute: for settings configuration, not this one. """ - def __init__(self, value, priority): - self.value = value + def __init__(self, value: Any, priority: int): + self.value: Any = value + self.priority: int if isinstance(self.value, BaseSettings): self.priority = max(self.value.maxpriority(), priority) else: self.priority = priority - def set(self, value, priority): + def set(self, value: Any, priority: int) -> None: """Sets value if priority is higher or equal than current priority.""" if priority >= self.priority: if isinstance(self.value, BaseSettings): @@ -49,11 +79,11 @@ class SettingsAttribute: self.value = value self.priority = priority - def __repr__(self): + def __repr__(self) -> str: return f"" -class BaseSettings(MutableMapping): +class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -77,21 +107,23 @@ class BaseSettings(MutableMapping): __default = object() - def __init__(self, values=None, priority="project"): - self.frozen = False - self.attributes = {} + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): + self.frozen: bool = False + self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name): + def __getitem__(self, opt_name: _SettingsKeyT) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value - def __contains__(self, name): + def __contains__(self, name: Any) -> bool: return name in self.attributes - def get(self, name, default=None): + def get(self, name: _SettingsKeyT, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -103,7 +135,7 @@ class BaseSettings(MutableMapping): """ return self[name] if self[name] is not None else default - def getbool(self, name, default=False): + def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -133,7 +165,7 @@ class BaseSettings(MutableMapping): "'True'/'False' and 'true'/'false'" ) - def getint(self, name, default=0): + def getint(self, name: _SettingsKeyT, default: int = 0) -> int: """ Get a setting value as an int. @@ -145,7 +177,7 @@ class BaseSettings(MutableMapping): """ return int(self.get(name, default)) - def getfloat(self, name, default=0.0): + def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -157,7 +189,9 @@ class BaseSettings(MutableMapping): """ return float(self.get(name, default)) - def getlist(self, name, default=None): + def getlist( + self, name: _SettingsKeyT, default: Optional[List[Any]] = None + ) -> List[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by ",". @@ -176,7 +210,9 @@ class BaseSettings(MutableMapping): value = value.split(",") return list(value) - def getdict(self, name, default=None): + def getdict( + self, name: _SettingsKeyT, default: Optional[Dict[Any, Any]] = None + ) -> Dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a dictionary, a copy of it will be returned. If it is a string it will be @@ -197,7 +233,11 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) - def getdictorlist(self, name, default=None): + def getdictorlist( + self, + name: _SettingsKeyT, + default: Union[Dict[Any, Any], List[Any], None] = None, + ) -> Union[Dict[Any, Any], List[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be @@ -224,24 +264,29 @@ class BaseSettings(MutableMapping): return {} if isinstance(value, str): try: - return json.loads(value) + value_loaded = json.loads(value) + assert isinstance(value_loaded, (dict, list)) + return value_loaded except ValueError: return value.split(",") + assert isinstance(value, (dict, list)) return copy.deepcopy(value) - def getwithbase(self, name): + def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings": """Get a composition of a dictionary-like setting and its `_BASE` counterpart. :param name: name of the dictionary-like setting :type name: str """ + if not isinstance(name, str): + raise ValueError(f"Base setting key must be a string, got {name}") compbs = BaseSettings() compbs.update(self[name + "_BASE"]) compbs.update(self[name]) return compbs - def getpriority(self, name): + def getpriority(self, name: _SettingsKeyT) -> Optional[int]: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -253,7 +298,7 @@ class BaseSettings(MutableMapping): return None return self.attributes[name].priority - def maxpriority(self): + def maxpriority(self) -> int: """ Return the numerical value of the highest priority present throughout all settings, or the numerical value for ``default`` from @@ -261,13 +306,15 @@ class BaseSettings(MutableMapping): stored. """ if len(self) > 0: - return max(self.getpriority(name) for name in self) + return max(cast(int, self.getpriority(name)) for name in self) return get_settings_priority("default") - def __setitem__(self, name, value): + def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: self.set(name, value) - def set(self, name, value, priority="project"): + def set( + self, name: _SettingsKeyT, value: Any, priority: Union[int, str] = "project" + ) -> None: """ Store a key/value attribute with a given priority. @@ -295,17 +342,26 @@ class BaseSettings(MutableMapping): else: self.attributes[name].set(value, priority) - def setdefault(self, name, default=None, priority="project"): + def setdefault( + self, + name: _SettingsKeyT, + default: Any = None, + priority: Union[int, str] = "project", + ) -> Any: if name not in self: self.set(name, default, priority) return default return self.attributes[name].value - def setdict(self, values, priority="project"): + def setdict( + self, values: _SettingsInputT, priority: Union[int, str] = "project" + ) -> None: self.update(values, priority) - def setmodule(self, module, priority="project"): + def setmodule( + self, module: Union[ModuleType, str], priority: Union[int, str] = "project" + ) -> None: """ Store settings from a module with a given priority. @@ -327,7 +383,8 @@ class BaseSettings(MutableMapping): if key.isupper(): self.set(key, getattr(module, key), priority) - def update(self, values, priority="project"): + # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports + def update(self, values: _SettingsInputT, priority: Union[int, str] = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -351,30 +408,34 @@ class BaseSettings(MutableMapping): """ self._assert_mutability() if isinstance(values, str): - values = json.loads(values) + values = cast(dict, json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): - self.set(name, value, values.getpriority(name)) + self.set(name, value, cast(int, values.getpriority(name))) else: for name, value in values.items(): self.set(name, value, priority) - def delete(self, name, priority="project"): + def delete( + self, name: _SettingsKeyT, priority: Union[int, str] = "project" + ) -> None: + if name not in self: + raise KeyError(name) self._assert_mutability() priority = get_settings_priority(priority) - if priority >= self.getpriority(name): + if priority >= cast(int, self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name): + def __delitem__(self, name: _SettingsKeyT) -> None: self._assert_mutability() del self.attributes[name] - def _assert_mutability(self): + def _assert_mutability(self) -> None: if self.frozen: raise TypeError("Trying to modify an immutable Settings object") - def copy(self): + def copy(self) -> "Self": """ Make a deep copy of current settings. @@ -386,7 +447,7 @@ class BaseSettings(MutableMapping): """ return copy.deepcopy(self) - def freeze(self): + def freeze(self) -> None: """ Disable further changes to the current settings. @@ -396,7 +457,7 @@ class BaseSettings(MutableMapping): """ self.frozen = True - def frozencopy(self): + def frozencopy(self) -> "Self": """ Return an immutable copy of the current settings. @@ -406,26 +467,26 @@ class BaseSettings(MutableMapping): copy.freeze() return copy - def __iter__(self): + def __iter__(self) -> Iterator[_SettingsKeyT]: return iter(self.attributes) - def __len__(self): + def __len__(self) -> int: return len(self.attributes) - def _to_dict(self): + def _to_dict(self) -> Dict[_SettingsKeyT, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value): + def _get_key(self, key_value: Any) -> _SettingsKeyT: return ( key_value if isinstance(key_value, (bool, float, int, str, type(None))) else str(key_value) ) - def copy_to_dict(self): + def copy_to_dict(self) -> Dict[_SettingsKeyT, Any]: """ Make a copy of current settings and convert to a dict. @@ -441,13 +502,14 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() - def _repr_pretty_(self, p, cycle): + # https://ipython.readthedocs.io/en/stable/config/integrating.html#pretty-printing + def _repr_pretty_(self, p: Any, cycle: bool) -> None: if cycle: p.text(repr(self)) else: p.text(pformat(self.copy_to_dict())) - def pop(self, name, default=__default): + def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any: try: value = self.attributes[name].value except KeyError: @@ -471,7 +533,9 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__(self, values=None, priority="project"): + def __init__( + self, values: _SettingsInputT = None, priority: Union[int, str] = "project" + ): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user @@ -485,14 +549,16 @@ class Settings(BaseSettings): self.update(values, priority) -def iter_default_settings(): +def iter_default_settings() -> Iterable[Tuple[str, Any]]: """Return the default settings as an iterator of (name, value) tuples""" for name in dir(default_settings): if name.isupper(): yield name, getattr(default_settings, name) -def overridden_settings(settings): +def overridden_settings( + settings: Mapping[_SettingsKeyT, Any] +) -> Iterable[Tuple[str, Any]]: """Return a dict of the settings that have been overridden""" for name, defvalue in iter_default_settings(): value = settings[name] diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..0608527ae 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -28,6 +28,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): compbs = BaseSettings() for k, v in compdict.items(): prio = compdict.getpriority(k) + assert prio is not None if compbs.getpriority(convert(k)) == prio: raise ValueError( f"Some paths in {list(compdict.keys())!r} " diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 5fc825393..db000233e 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,6 +1,8 @@ import unittest from unittest import mock +import pytest + from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -199,6 +201,21 @@ class BaseSettingsTest(unittest.TestCase): settings.update({"key_lowprio": 3}, priority=20) self.assertEqual(settings["key_lowprio"], 1) + @pytest.mark.xfail( + raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" + ) + def test_update_kwargs(self): + settings = BaseSettings({"key": 0}) + settings.update(key=1) + + @pytest.mark.xfail( + raises=AttributeError, + reason="BaseSettings.update doesn't support iterable input", + ) + def test_update_iterable(self): + settings = BaseSettings({"key": 0}) + settings.update([("key", 1)]) + def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') @@ -217,6 +234,10 @@ class BaseSettingsTest(unittest.TestCase): self.assertIn("key_highprio", settings) del settings["key_highprio"] self.assertNotIn("key_highprio", settings) + with self.assertRaises(KeyError): + settings.delete("notkey") + with self.assertRaises(KeyError): + del settings["notkey"] def test_get(self): test_configuration = { diff --git a/tox.ini b/tox.ini index fe7df9782..1aeb94215 100644 --- a/tox.ini +++ b/tox.ini @@ -34,6 +34,7 @@ install_command = basepython = python3 deps = mypy==1.4.1 + typing-extensions==4.7.1 types-attrs==19.1.0 types-lxml==2023.3.28 types-Pillow==10.0.0.1 From 043a24410b2eeed554e8ce3a73b1e78fa53fa4d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 00:09:24 +0400 Subject: [PATCH 142/211] Disable pylint for broken code. --- tests/test_settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index db000233e..e7799737f 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -206,7 +206,7 @@ class BaseSettingsTest(unittest.TestCase): ) def test_update_kwargs(self): settings = BaseSettings({"key": 0}) - settings.update(key=1) + settings.update(key=1) # pylint: disable=unexpected-keyword-arg @pytest.mark.xfail( raises=AttributeError, From cdda8ad46dc064229b5d875fb7685fdb32ebfb3f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 21:05:58 +0400 Subject: [PATCH 143/211] Add docs about fallbacks in addons. --- docs/topics/addons.rst | 89 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 901a8bf8f..cc96207bd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -62,12 +62,53 @@ They can also have the following method: :type crawler: :class:`~scrapy.crawler.Crawler` The settings set by the addon should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`). This -allows users to override these settings in the project or spider configuration. -This is not possible with settings that are mutable objects, such as the dict -that is a value of :setting:`ITEM_PIPELINES`. In these cases you can provide an -addon-specific setting that governs whether the addon will modify -:setting:`ITEM_PIPELINES`. +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: + + class MyAddon: + def update_settings(self, settings): + settings.set("DNSCACHE_ENABLED", True, "addon") + +This allows users to override these settings in the project or spider +configuration. This is not possible with settings that are mutable objects, +such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases +you can provide an addon-specific setting that governs whether the addon will +modify :setting:`ITEM_PIPELINES`:: + + class MyAddon: + def update_settings(self, settings): + if settings.getbool("MYADDON_ENABLE_PIPELINE"): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + +Fallbacks +--------- + +Some components provided by addons need to fallback to "default" +implementations, e.g. a custom download handler needs to send the request that +it doesn't handle via the default download handler, or a stats collector that +includes some additional processing but otherwise uses the default stats +collector. And it's possible that a project needs to use several custom +components of the same type, e.g. two custom download handlers that support +different kinds of custom requests and still need to use the default download +handler for other requests. To make such use cases easier to configure, we +recommend that such custom components should be written in the following way: + +1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the + default Scrapy one (e.g. + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + be able to load the class of the fallback component from a special setting + (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use + it. +2. The addons that include these components should read the current value of + the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their + ``update_settings()`` methods, save that value into the fallback setting + (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default + setting to the component provided byt the addon (e.g. + ``MyDownloadHandler``). If the fallback setting is already set by the user, + they shouldn't change it. +3. This way, if there are several addons that want to modify the same setting, + all of them will fallback to the component from the previous one and then to + the Scrapy default. The order of that depends on the priority order in the + ``ADDONS`` setting. Add-on examples @@ -103,3 +144,39 @@ Access the crawler instance:: def update_settings(self, settings): ... + +Use a fallback component:: + + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + + + fallback_setting = "MY_FALLBACK_DOWNLOAD_HANDLER" + + + class MyHandler: + lazy = False + + def __init__(self, settings, crawler): + dhcls = load_object(settings.get(fallback_setting)) + self._fallback_handler = create_instance( + dhcls, + settings=None, + crawler=crawler, + ) + + def download_request(self, request, spider): + if request.meta.get("my_params"): + # handle the request + ... + else: + return self._fallback_handler.download_request(request, spider) + + + class MyAddon: + def update_settings(self, settings): + if not settings.get(fallback_setting): + settings.set( + fallback_setting, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) From db86f91789d25e45c43f964fff31d3016a451a1e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 23:17:11 +0400 Subject: [PATCH 144/211] Unbreak isort breakage. --- scrapy/settings/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 658c27f0a..b0adb5ba8 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -27,9 +27,10 @@ from scrapy.settings import default_settings _SettingsKeyT = Union[bool, float, int, str, None] if TYPE_CHECKING: - # typing.Self requires Python 3.11 # https://github.com/python/typing/issues/445#issuecomment-1131458824 from _typeshed import SupportsItems + + # typing.Self requires Python 3.11 from typing_extensions import Self _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] From 3f5bbe3a8fababac6dfcfcf1a22aa7f7dba199ba Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 18 Jul 2023 18:30:21 +0800 Subject: [PATCH 145/211] introduce CLOSESPIDER_TIMEOUT_NO_ITEM in CloseSpider --- docs/topics/extensions.rst | 13 ++++++++++ scrapy/extensions/closespider.py | 41 ++++++++++++++++++++++++++++++++ tests/test_closespider.py | 12 +++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 96e0216b8..8d4749ab3 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following settings: * :setting:`CLOSESPIDER_TIMEOUT` +* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` * :setting:`CLOSESPIDER_ERRORCOUNT` @@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by timeout. +.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM + +CLOSESPIDER_TIMEOUT_NO_ITEM +""""""""""""""""""""""""""" + +Default: ``0`` + +An integer which specifies a number of seconds. If the spider has not produced +any items in the last number of seconds, it will be closed with the reason +``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed +regardless if it hasn't produced any items. + .. setting:: CLOSESPIDER_ITEMCOUNT CLOSESPIDER_ITEMCOUNT diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index bb6f832f2..456470efd 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,11 +4,14 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +import logging from collections import defaultdict from scrapy import signals from scrapy.exceptions import NotConfigured +logger = logging.getLogger(__name__) + class CloseSpider: def __init__(self, crawler): @@ -19,6 +22,7 @@ class CloseSpider: "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"), + "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"), } if not any(self.close_on.values()): @@ -34,6 +38,15 @@ class CloseSpider: crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + if self.close_on.get("timeout_no_item"): + self.timeout_no_item = self.close_on["timeout_no_item"] + self.items_in_period = 0 + crawler.signals.connect( + self.spider_opened_no_item, signal=signals.spider_opened + ) + crawler.signals.connect( + self.item_scraped_no_item, signal=signals.item_scraped + ) crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod @@ -69,3 +82,31 @@ class CloseSpider: task = getattr(self, "task", False) if task and task.active(): task.cancel() + + task_no_item = getattr(self, "task_no_item", False) + if task_no_item.running: + task_no_item.stop() + + def spider_opened_no_item(self, spider): + from twisted.internet import task + + self.task_no_item = task.LoopingCall(self._count_items_produced, spider) + self.task_no_item.start(self.timeout_no_item, now=False) + + logger.info( + f"Spider will stop when no items are produced after " + f"{self.timeout_no_item} seconds." + ) + + def item_scraped_no_item(self, item, spider): + self.items_in_period += 1 + + def _count_items_produced(self, spider): + if self.items_in_period >= 1: + self.items_in_period = 0 + else: + logger.info( + f"Closing spider since no items were produced in the last " + f"{self.timeout_no_item} seconds." + ) + self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 9b39187d5..259a1a4ad 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider +from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider class TestCloseSpider(TestCase): @@ -54,3 +54,13 @@ class TestCloseSpider(TestCase): self.assertEqual(reason, "closespider_timeout") total_seconds = crawler.stats.get_value("elapsed_time_seconds") self.assertTrue(total_seconds >= close_on) + + @defer.inlineCallbacks + def test_closespider_timeout_no_item(self): + timeout = 1 + crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) + yield crawler.crawl(n=3, total=10, mockserver=self.mockserver) + reason = crawler.spider.meta["close_reason"] + self.assertEqual(reason, "closespider_timeout_no_item") + total_seconds = crawler.stats.get_value("elapsed_time_seconds") + self.assertTrue(total_seconds >= timeout) From 2f787a27dc1f7551b98d322b6b93b3cb8bad4e2e Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 18 Jul 2023 20:49:33 +0800 Subject: [PATCH 146/211] fix conditional on task_no_item --- scrapy/extensions/closespider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 456470efd..4307b4170 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -84,7 +84,7 @@ class CloseSpider: task.cancel() task_no_item = getattr(self, "task_no_item", False) - if task_no_item.running: + if task_no_item and task_no_item.running: task_no_item.stop() def spider_opened_no_item(self, spider): From 368ab29ffc32b13d440f8adefdf06431b7ba7c32 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Tue, 18 Jul 2023 22:29:15 +0800 Subject: [PATCH 147/211] improve tests by having SlowSpider --- tests/spiders.py | 16 ++++++++++++++++ tests/test_closespider.py | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 6ff48f471..f29dea2a1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -77,6 +77,22 @@ class DelaySpider(MetaSpider): self.t2_err = time.time() +class SlowSpider(DelaySpider): + name = "slow" + + def start_requests(self): + # 1st response is fast + url = self.mockserver.url("/delay?n=0&b=0") + yield Request(url, callback=self.parse, errback=self.errback) + + # 2nd response is slow + url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}") + yield Request(url, callback=self.parse, errback=self.errback) + + def parse(self, response): + yield Item() + + class SimpleSpider(MetaSpider): name = "simple" diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 259a1a4ad..38ede70e4 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import DelaySpider, ErrorSpider, FollowAllSpider, ItemSpider +from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider class TestCloseSpider(TestCase): @@ -58,8 +58,8 @@ class TestCloseSpider(TestCase): @defer.inlineCallbacks def test_closespider_timeout_no_item(self): timeout = 1 - crawler = get_crawler(DelaySpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) - yield crawler.crawl(n=3, total=10, mockserver=self.mockserver) + crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) + yield crawler.crawl(n=3, mockserver=self.mockserver) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_timeout_no_item") total_seconds = crawler.stats.get_value("elapsed_time_seconds") From 5c34f34ecbc18c0829cd04a621b1d8239c25642b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 19:50:08 +0400 Subject: [PATCH 148/211] Make AddonManager.add() private. --- scrapy/addons.py | 20 ++++++-------------- tests/test_addons.py | 6 ------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 612c7effc..812892613 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -17,15 +17,8 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def add(self, addon: Any) -> None: - """Store an add-on. - - :param addon: The add-on object (or path) to be stored - :type addon: Python object, class or ``str`` - - :param config: The add-on configuration dictionary - :type config: ``dict`` - """ + def _add(self, addon: Any) -> None: + """Store an add-on.""" if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): @@ -33,7 +26,7 @@ class AddonManager: self.addons.append(addon) def load_settings(self, settings) -> None: - """Load add-ons and configurations from settings object. + """Load add-ons and configurations from a settings object. This will load the addon for every add-on path in the ``ADDONS`` setting. @@ -42,10 +35,9 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings["ADDONS"]) - addons = [load_object(path) for path in paths] - for a in addons: - self.add(a) + addons = build_component_list(settings["ADDONS"]) + for addon in build_component_list(settings["ADDONS"]): + self._add(addon) logger.info( "Enabled addons:\n%(addons)s", { diff --git a/tests/test_addons.py b/tests/test_addons.py index d52665869..95377d718 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -43,12 +43,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def test_add(self): - crawler = get_crawler() - manager = crawler.addons - manager.add("tests.test_addons.GoodAddon") - self.assertIsInstance(manager.addons[0], GoodAddon) - def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.GoodAddon": 0}, From 90dae3ee60b1f5ab5ea83d82ac8c00f1be54723e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 19:52:27 +0400 Subject: [PATCH 149/211] Doc fixes. --- docs/topics/addons.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index cc96207bd..c432c64d2 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python objects that include the following method: +Add-ons are Python classes that include the following method: .. method:: update_settings(settings) @@ -54,9 +54,9 @@ They can also have the following method: If present, this class method is called to create an addon instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the addon. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. + of the addon. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the addon to access + them and hook its functionality into Scrapy. :param crawler: The crawler that uses this addon :type crawler: :class:`~scrapy.crawler.Crawler` @@ -82,7 +82,7 @@ modify :setting:`ITEM_PIPELINES`:: Fallbacks --------- -Some components provided by addons need to fallback to "default" +Some components provided by addons need to fall back to "default" implementations, e.g. a custom download handler needs to send the request that it doesn't handle via the default download handler, or a stats collector that includes some additional processing but otherwise uses the default stats @@ -102,7 +102,7 @@ recommend that such custom components should be written in the following way: the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their ``update_settings()`` methods, save that value into the fallback setting (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default - setting to the component provided byt the addon (e.g. + setting to the component provided by the addon (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, they shouldn't change it. 3. This way, if there are several addons that want to modify the same setting, @@ -114,14 +114,18 @@ recommend that such custom components should be written in the following way: Add-on examples =============== -Set some basic configuration:: +Set some basic configuration: + +.. code-block:: python class MyAddon: def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") -Check dependencies:: +Check dependencies: + +.. code-block:: python class MyAddon: def update_settings(self, settings): @@ -131,7 +135,9 @@ Check dependencies:: raise RuntimeError("MyAddon requires the boto library") ... -Access the crawler instance:: +Access the crawler instance: + +.. code-block:: python class MyAddon: def __init__(self, crawler) -> None: @@ -145,7 +151,9 @@ Access the crawler instance:: def update_settings(self, settings): ... -Use a fallback component:: +Use a fallback component: + +.. code-block:: python from scrapy.core.downloader.handlers.http import HTTPDownloadHandler From 0a25a300cfc8281c355a6cfcd8605418347ed324 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 20:29:42 +0400 Subject: [PATCH 150/211] Fix docs. --- docs/topics/addons.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index c432c64d2..36c7e0fa0 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -145,7 +145,7 @@ Access the crawler instance: self.crawler = crawler @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler): return cls(crawler) def update_settings(self, settings): From 005c8cc5f00f41ad7d836eccf45dbaf39c03ebca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jul 2023 13:15:35 +0400 Subject: [PATCH 151/211] Unify the "add-on" spelling. --- docs/topics/addons.rst | 22 +++++++++++----------- docs/topics/settings.rst | 8 ++++---- scrapy/addons.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 36c7e0fa0..8733f9bde 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -19,7 +19,7 @@ Add-ons and their configuration live in Scrapy's initialization the add-on manager will read a list of enabled add-ons from your ``ADDONS`` setting. -The ``ADDONS`` setting is a dict in which every key is an addon class or its +The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's @@ -52,16 +52,16 @@ They can also have the following method: .. classmethod:: from_crawler(cls, crawler) :noindex: - If present, this class method is called to create an addon instance + If present, this class method is called to create an add-on instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the addon. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the addon to access + of the add-on. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the add-on to access them and hook its functionality into Scrapy. - :param crawler: The crawler that uses this addon + :param crawler: The crawler that uses this add-on :type crawler: :class:`~scrapy.crawler.Crawler` -The settings set by the addon should use the ``addon`` priority (see +The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: class MyAddon: @@ -71,7 +71,7 @@ The settings set by the addon should use the ``addon`` priority (see This allows users to override these settings in the project or spider configuration. This is not possible with settings that are mutable objects, such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an addon-specific setting that governs whether the addon will +you can provide an add-on-specific setting that governs whether the add-on will modify :setting:`ITEM_PIPELINES`:: class MyAddon: @@ -82,7 +82,7 @@ modify :setting:`ITEM_PIPELINES`:: Fallbacks --------- -Some components provided by addons need to fall back to "default" +Some components provided by add-ons need to fall back to "default" implementations, e.g. a custom download handler needs to send the request that it doesn't handle via the default download handler, or a stats collector that includes some additional processing but otherwise uses the default stats @@ -98,14 +98,14 @@ recommend that such custom components should be written in the following way: be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. -2. The addons that include these components should read the current value of +2. The add-ons that include these components should read the current value of the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their ``update_settings()`` methods, save that value into the fallback setting (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default - setting to the component provided by the addon (e.g. + setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, they shouldn't change it. -3. This way, if there are several addons that want to modify the same setting, +3. This way, if there are several add-ons that want to modify the same setting, all of them will fallback to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 139e0a35f..602ab587d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,7 +40,7 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Settings set by addons + 4. Settings set by add-ons 5. Default settings per-command 6. Default global settings (less precedence) @@ -90,10 +90,10 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Settings set by addons -------------------------- +4. Settings set by add-ons +-------------------------- -:ref:`Addons ` can modify settings. They should do this with +:ref:`Add-ons ` can modify settings. They should do this with this priority, though this is not enforced. 5. Default settings per-command diff --git a/scrapy/addons.py b/scrapy/addons.py index 812892613..e72c5da98 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -28,7 +28,7 @@ class AddonManager: def load_settings(self, settings) -> None: """Load add-ons and configurations from a settings object. - This will load the addon for every add-on path in the + This will load the add-on for every add-on path in the ``ADDONS`` setting. :param settings: The :class:`~scrapy.settings.Settings` object from \ From 583df9f7d063ac0f48eafccdd463f3195a084d9b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:44:37 +0400 Subject: [PATCH 152/211] Simplify skipping uvloop tests. --- conftest.py | 16 ++++++++++++++ pytest.ini | 1 + tests/test_commands.py | 14 +----------- tests/test_crawler.py | 50 ++++-------------------------------------- 4 files changed, 22 insertions(+), 59 deletions(-) diff --git a/conftest.py b/conftest.py index e1d4b1213..fa5193470 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,10 @@ +import platform +import sys from pathlib import Path import pytest +from twisted import version as twisted_version +from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -73,6 +77,18 @@ def only_not_asyncio(request, reactor_pytest): pytest.skip("This test is only run without --reactor=asyncio") +@pytest.fixture(autouse=True) +def requires_uvloop(request): + if not request.node.get_closest_marker("requires_uvloop"): + return + if sys.implementation.name == "pypy": + pytest.skip("uvloop does not support pypy properly") + if platform.system() == "Windows": + pytest.skip("uvloop does not support Windows") + if twisted_version == Version("twisted", 21, 2, 0): + pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + + def pytest_configure(config): if config.getoption("--reactor") == "asyncio": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/pytest.ini b/pytest.ini index 866f0c950..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -20,6 +20,7 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed + requires_uvloop: marks tests as only enabled when uvloop is known to be working filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_commands.py b/tests/test_commands.py index 014f50e92..03d768d1a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,8 +18,6 @@ from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.trial import unittest import scrapy @@ -802,17 +800,7 @@ class MySpider(scrapy.Spider): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_asyncio_loop_enabled_true(self): log = self.get_log( self.debug_log_spider, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d54a2cb7e..68e58144b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -8,9 +8,7 @@ from pathlib import Path from packaging.version import parse as parse_version from pytest import mark, raises -from twisted import version as twisted_version from twisted.internet import defer -from twisted.python.versions import Version from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -466,17 +464,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): log, ) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") self.assertIn("Spider closed (finished)", log) @@ -485,17 +473,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_custom_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") self.assertIn("Spider closed (finished)", log) @@ -505,17 +483,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_same_loop(self): log = self.run_script("asyncio_enabled_reactor_same_loop.py") self.assertIn("Spider closed (finished)", log) @@ -524,17 +492,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): ) self.assertIn("Using asyncio event loop: uvloop.Loop", log) - @mark.skipif( - sys.implementation.name == "pypy", - reason="uvloop does not support pypy properly", - ) - @mark.skipif( - platform.system() == "Windows", reason="uvloop does not support Windows" - ) - @mark.skipif( - twisted_version == Version("twisted", 21, 2, 0), - reason="https://twistedmatrix.com/trac/ticket/10106", - ) + @mark.requires_uvloop def test_asyncio_enabled_reactor_different_loop(self): log = self.run_script("asyncio_enabled_reactor_different_loop.py") self.assertNotIn("Spider closed (finished)", log) From e058a05763b45086edaf8b52f067648c0c50ae21 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:51:13 +0400 Subject: [PATCH 153/211] Skip tests that don't work on Python 3.12. --- conftest.py | 2 ++ tests/requirements.txt | 10 ++++++---- tests/test_feedexport.py | 3 +++ tests/test_pipeline_files.py | 5 +++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index fa5193470..635935748 100644 --- a/conftest.py +++ b/conftest.py @@ -87,6 +87,8 @@ def requires_uvloop(request): pytest.skip("uvloop does not support Windows") if twisted_version == Version("twisted", 21, 2, 0): pytest.skip("https://twistedmatrix.com/trac/ticket/10106") + if sys.version_info >= (3, 12): + pytest.skip("uvloop doesn't support Python 3.12 yet") def pytest_configure(config): diff --git a/tests/requirements.txt b/tests/requirements.txt index 618949795..37186f3a7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,15 +1,17 @@ # Tests requirements attrs -pyftpdlib +# https://github.com/giampaolo/pyftpdlib/issues/560 +pyftpdlib; python_version < "3.12" pytest pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures -uvloop; platform_system != "Windows" +# uvloop currently doesn't build on 3.12 +uvloop; platform_system != "Windows" and python_version < "3.12" -# optional for shell wrapper tests -bpython +# bpython requires greenlet which currently doesn't build on 3.12 +bpython; python_version < "3.12" # optional for shell wrapper tests brotli # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8df86dbd7..eace59d37 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -125,6 +125,9 @@ class FileFeedStorageTest(unittest.TestCase): path.unlink() +@pytest.mark.skipif( + sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" +) class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 87f3a0295..fe7b26740 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,6 +1,7 @@ import dataclasses import os import random +import sys import time from datetime import datetime from io import BytesIO @@ -11,6 +12,7 @@ from unittest import mock from urllib.parse import urlparse import attr +import pytest from itemadapter import ItemAdapter from twisted.internet import defer from twisted.trial import unittest @@ -641,6 +643,9 @@ class TestGCSFilesStore(unittest.TestCase): store.bucket.get_blob.assert_called_with(expected_blob_path) +@pytest.mark.skipif( + sys.version_info >= (3, 12), reason="pyftpdlib doesn't support Python 3.12 yet" +) class TestFTPFileStore(unittest.TestCase): @defer.inlineCallbacks def test_persist(self): From a346732275b425e4fbebc3bdf133df961528df87 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:54:55 +0400 Subject: [PATCH 154/211] Skip more non-test files during discovery. --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index 635935748..2bfa46f5a 100644 --- a/conftest.py +++ b/conftest.py @@ -18,6 +18,10 @@ def _py_files(folder): collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + "tests/ftpserver.py", + "tests/mockserver.py", + "tests/pipelines.py", + "tests/spiders.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess From 21b6dc5f9fbd2607d6f4df20256dcdd9d5b9dae4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 17:55:32 +0400 Subject: [PATCH 155/211] Add 3.12 CI jobs. --- .github/workflows/tests-ubuntu.yml | 12 +++++++++++- setup.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 39e3b0af7..54b3fbaa2 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,6 +48,16 @@ jobs: env: TOXENV: botocore + - python-version: "3.12.0-beta.4" + env: + TOXENV: py + - python-version: "3.12.0-beta.4" + env: + TOXENV: asyncio + - python-version: "3.12.0-beta.4" + env: + TOXENV: extra-deps + steps: - uses: actions/checkout@v3 @@ -57,7 +67,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.12.0') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/setup.py b/setup.py index 1f214571b..405633f55 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet :: WWW/HTTP", From 53f8570786fbc7c90bc9990a22279d90a52025b3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 18:46:44 +0400 Subject: [PATCH 156/211] Add support for the new entry_points() interface. --- scrapy/cmdline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index efc9b36ea..6580ba9ce 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -48,7 +48,11 @@ def _get_commands_from_module(module, inproject): def _get_commands_from_entry_points(inproject, group="scrapy.commands"): cmds = {} - for entry_point in entry_points().get(group, {}): + if sys.version_info >= (3, 10): + eps = entry_points(group=group) + else: + eps = entry_points().get(group, ()) + for entry_point in eps: obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() From 5d91ea12d642503da95df43cf033eeeda3db1553 Mon Sep 17 00:00:00 2001 From: Adnan Awan Date: Sat, 22 Jul 2023 23:13:40 +0500 Subject: [PATCH 157/211] Fix JsonItemExporter puts lone comma in the output if encoder fails (#5952) * Partial fix for #3090 - only addresses JSON feeds. * Adding test case for #3090 to Json Exporter * Changing the deliberate-fail JSON example to a complex * Further tightening JsonItemExporter behaviour to prevent corruption. Based on Mikhail's observation that to_bytes can fail also, leading to the same dangling comma as the failure to encode to JSON. Added a new test case to avoid reversion. * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Add initial changes from cathal's PR - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Handle exception not to add empty item. - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Add comment for handling the exception - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Remove unused import - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Fix invalid json issue - https://github.com/scrapy/scrapy/issues/3090 * [scrapy] JsonItemExporter puts lone comma in the output if encoder fails - Perform CR changes - https://github.com/scrapy/scrapy/issues/3090 --------- Co-authored-by: Cathal Garvey --- scrapy/exporters.py | 17 ++++++++++------- tests/test_exporters.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 4538c9ee1..8254ea63e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -133,6 +133,13 @@ class JsonItemExporter(BaseItemExporter): if self.indent is not None: self.file.write(b"\n") + def _add_comma_after_first(self): + if self.first_item: + self.first_item = False + else: + self.file.write(b",") + self._beautify_newline() + def start_exporting(self): self.file.write(b"[") self._beautify_newline() @@ -142,14 +149,10 @@ class JsonItemExporter(BaseItemExporter): self.file.write(b"]") def export_item(self, item): - if self.first_item: - self.first_item = False - else: - self.file.write(b",") - self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) - data = self.encoder.encode(itemdict) - self.file.write(to_bytes(data, self.encoding)) + data = to_bytes(self.encoder.encode(itemdict), self.encoding) + self._add_comma_after_first() + self.file.write(data) class XmlItemExporter(BaseItemExporter): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 63bebcf7a..cb24ddd8e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -599,6 +599,20 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def test_two_dict_items(self): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph\xa3", age="22") + i2 = TestItem( + name="Maria", age=1j + ) # Invalid datetimes didn't consistently fail between Python versions + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(TypeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue())) + self.assertEqual(exported, [dict(i1), dict(i3)]) + def test_nested_item(self): i1 = self.item_class(name="Joseph\xa3", age="22") i2 = self.item_class(name="Maria", age=i1) @@ -637,6 +651,24 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [item]) +class JsonItemExporterToBytesTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + kwargs["encoding"] = "latin" + return JsonItemExporter(self.output, **kwargs) + + def test_two_items_with_failure_between(self): + i1 = TestItem(name="Joseph", age="22") + i2 = TestItem(name="\u263a", age="11") + i3 = TestItem(name="Jesus", age="44") + self.ie.start_exporting() + self.ie.export_item(i1) + self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2) + self.ie.export_item(i3) + self.ie.finish_exporting() + exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin")) + self.assertEqual(exported, [dict(i1), dict(i3)]) + + class JsonItemExporterDataclassTest(JsonItemExporterTest): item_class = TestDataClass custom_field_item_class = CustomFieldDataclass From 5e1582491bd3226afc69eb12287b951e78bfc4ba Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 22 Jul 2023 23:19:25 +0400 Subject: [PATCH 158/211] mypy --show-error-codes is the default now. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1aeb94215..ef7dd5854 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ deps = types-pyOpenSSL==23.2.0.1 types-setuptools==68.0.0.1 commands = - mypy --show-error-codes {posargs: scrapy tests} + mypy {posargs: scrapy tests} [testenv:pre-commit] basepython = python3 From 7522aeed357d90ed95ee10d3b5a506f1b1049d1f Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 24 Jul 2023 04:53:41 -0300 Subject: [PATCH 159/211] fix: -O/-o option does not work with absolute paths on Windows (#5971) --- scrapy/extensions/feedexport.py | 6 ++-- scrapy/utils/conf.py | 3 +- tests/test_commands.py | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6e391d3dc..c81f396cb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -9,7 +9,7 @@ import re import sys import warnings from datetime import datetime -from pathlib import Path +from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile from typing import IO, Any, Callable, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse @@ -615,7 +615,7 @@ class FeedExporter: def _storage_supported(self, uri, feed_options): scheme = urlparse(uri).scheme - if scheme in self.storages: + if scheme in self.storages or PureWindowsPath(uri).drive: try: self._get_storage(uri, feed_options) return True @@ -633,7 +633,7 @@ class FeedExporter: It supports not passing the *feed_options* parameters to classes that do not support it, and issuing a deprecation warning instead. """ - feedcls = self.storages[urlparse(uri).scheme] + feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) crawler = getattr(self, "crawler", None) def build_instance(builder, *preargs): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 3ade1d105..05d43e456 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -202,7 +202,8 @@ def feed_process_params_from_cli( for element in output: try: feed_uri, feed_format = element.rsplit(":", 1) - except ValueError: + check_valid_format(feed_format) + except (ValueError, UsageError): feed_uri = element feed_format = Path(element).suffix.replace(".", "") else: diff --git a/tests/test_commands.py b/tests/test_commands.py index 03d768d1a..b1d7be628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -907,6 +907,64 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json", + log, + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json", + log, + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json", + log, + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + self.assertIn( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json", + log, + ) + @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): From 9a1bf40c2f7ac0b5fdd4810336afddbfeb925981 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Thu, 20 Jul 2023 17:03:51 +0800 Subject: [PATCH 160/211] expose AWS_REGION_NAME in S3FeedStorage --- docs/topics/feed-exports.rst | 4 +++- scrapy/extensions/feedexport.py | 5 ++++ tests/test_feedexport.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2218f45e7..700775e4b 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -210,10 +210,12 @@ passed through the following settings: .. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys -You can also define a custom ACL and custom endpoint for exported feeds using this setting: +You can also define a custom ACL, custom endpoint, and region name for exported +feeds using these settings: - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +- :setting:`AWS_REGION_NAME` The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c81f396cb..84724640b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -169,6 +169,7 @@ class S3FeedStorage(BlockingFeedStorage): secret_key=None, acl=None, endpoint_url=None, + region_name=None, *, feed_options=None, session_token=None, @@ -183,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url + self.region_name = region_name if IS_BOTO3_AVAILABLE: import boto3.session @@ -195,6 +197,7 @@ class S3FeedStorage(BlockingFeedStorage): aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, endpoint_url=self.endpoint_url, + region_name=self.region_name, ) else: warnings.warn( @@ -213,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage): aws_secret_access_key=self.secret_key, aws_session_token=self.session_token, endpoint_url=self.endpoint_url, + region_name=self.region_name, ) if feed_options and feed_options.get("overwrite", True) is False: @@ -232,6 +236,7 @@ class S3FeedStorage(BlockingFeedStorage): session_token=crawler.settings["AWS_SESSION_TOKEN"], acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None, endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None, + region_name=crawler.settings["AWS_REGION_NAME"] or None, feed_options=feed_options, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 96bb688ab..9e4fb53bf 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -349,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_init_with_region_name(self): + region_name = "ap-east-1" + storage = S3FeedStorage( + "s3://mybucket/export.csv", + "access_key", + "secret_key", + region_name=region_name, + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + def test_from_crawler_without_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -377,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, None) + def test_without_region_name(self): + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler( + crawler, + "s3://mybucket/export.csv", + ) + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1") + def test_from_crawler_with_acl(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -404,6 +431,20 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.secret_key, "secret_key") self.assertEqual(storage.endpoint_url, "https://example.com") + def test_from_crawler_with_region_name(self): + region_name = "ap-east-1" + settings = { + "AWS_ACCESS_KEY_ID": "access_key", + "AWS_SECRET_ACCESS_KEY": "secret_key", + "AWS_REGION_NAME": region_name, + } + crawler = get_crawler(settings_dict=settings) + storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv") + self.assertEqual(storage.access_key, "access_key") + self.assertEqual(storage.secret_key, "secret_key") + self.assertEqual(storage.region_name, region_name) + self.assertEqual(storage.s3_client._client_config.region_name, region_name) + @defer.inlineCallbacks def test_store_without_acl(self): storage = S3FeedStorage( From a689fe5baf35e50d47270e9a402b1d67b6e95ffe Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Mon, 24 Jul 2023 13:08:39 +0800 Subject: [PATCH 161/211] move region_name param as kwargs --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 84724640b..4687c7382 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -169,10 +169,10 @@ class S3FeedStorage(BlockingFeedStorage): secret_key=None, acl=None, endpoint_url=None, - region_name=None, *, feed_options=None, session_token=None, + region_name=None, ): if not is_botocore_available(): raise NotConfigured("missing botocore library") From 3ba2dc4d682c459ca3fa4419a68ae18b52a47642 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Jul 2023 17:49:04 +0400 Subject: [PATCH 162/211] Fixes for addon docs. --- docs/topics/addons.rst | 17 ++++++++--------- docs/topics/api.rst | 11 ----------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8733f9bde..f02399671 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -14,10 +14,8 @@ developers. Activating and configuring add-ons ================================== -Add-ons and their configuration live in Scrapy's -:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` -initialization the add-on manager will read a list of enabled add-ons from your -``ADDONS`` setting. +During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled +add-ons is read from your ``ADDONS`` setting. The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. @@ -27,7 +25,7 @@ This is an example where two add-ons are enabled in a project's ADDONS = { 'path.to.someaddon': 0, - path.to.someaddon2: 1, + SomeAddonClass: 1, } @@ -158,14 +156,14 @@ Use a fallback component: from scrapy.core.downloader.handlers.http import HTTPDownloadHandler - fallback_setting = "MY_FALLBACK_DOWNLOAD_HANDLER" + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" class MyHandler: lazy = False def __init__(self, settings, crawler): - dhcls = load_object(settings.get(fallback_setting)) + dhcls = load_object(settings.get(FALLBACK_SETTING)) self._fallback_handler = create_instance( dhcls, settings=None, @@ -182,9 +180,10 @@ Use a fallback component: class MyAddon: def update_settings(self, settings): - if not settings.get(fallback_setting): + if not settings.get(FALLBACK_SETTING): settings.set( - fallback_setting, + FALLBACK_SETTING, settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) + settings["DOWNLOAD_HANDLERS"]["http"] = "path.to.MyHandler" diff --git a/docs/topics/api.rst b/docs/topics/api.rst index d1a5497fb..16c28405c 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -155,17 +155,6 @@ Settings API .. autoclass:: BaseSettings :members: -.. _topics-api-addonmanager: - -AddonManager API -================ - -.. module:: scrapy.addons - :synopsis: Add-on manager - -.. autoclass:: AddonManager - :members: - .. _topics-api-spiderloader: SpiderLoader API From c4f0aa4fdfff802c153b1eeda711957d063aed60 Mon Sep 17 00:00:00 2001 From: freddiewanah Date: Wed, 26 Jul 2023 21:09:03 +1000 Subject: [PATCH 163/211] Refactor test cases to improve unit test quality (#5986) --- tests/test_crawler.py | 8 ++++---- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware_httpcompression.py | 6 +++--- tests/test_pipeline_files.py | 9 +++++++-- tests/test_pipeline_images.py | 10 ++++++++-- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 68e58144b..f99606ccf 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -228,14 +228,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_bootstrap_successful(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, False) + self.assertFalse(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed(self): @@ -248,7 +248,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: self.fail("Exception should be raised from spider") - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): @@ -263,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): yield runner.crawl(NoRequestsSpider) - self.assertEqual(runner.bootstrap_failed, True) + self.assertTrue(runner.bootstrap_failed) @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9731b62c4..8459408ff 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -748,7 +748,7 @@ class Http11MockServerTestCase(unittest.TestCase): yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get("failure") - self.assertTrue(failure is None) + self.assertIsNone(failure) reason = crawler.spider.meta["close_reason"] self.assertTrue(reason, "finished") diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fac5588ff..9dad056de 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -229,7 +229,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_force_recalculate_encoding(self): headers = { @@ -254,7 +254,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): headers = { @@ -277,7 +277,7 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding("gb2312")) self.assertStatsEqual("httpcompression/response_count", 1) - self.assertStatsEqual("httpcompression/response_bytes", 104) + self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index fe7b26740..bf96f17b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -468,8 +468,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.files_result_field, "this") - self.assertEqual(pipeline.files_urls_field, "that") + self.assertEqual( + pipeline.files_result_field, + UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of FilesPipeline, diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6fd8e6308..8924875d1 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -598,8 +598,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): pipeline = UserDefinedImagePipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ) - self.assertEqual(pipeline.images_result_field, "something_else") - self.assertEqual(pipeline.images_urls_field, "something") + self.assertEqual( + pipeline.images_result_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD, + ) + self.assertEqual( + pipeline.images_urls_field, + UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD, + ) def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of ImagePipeline, From d8c5b415597f9a58e5bf10ab1d8838bd7f730949 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 19:09:21 +0400 Subject: [PATCH 164/211] Add more addon tests. --- docs/topics/addons.rst | 2 +- tests/test_addons.py | 107 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index f02399671..edb1e7883 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -186,4 +186,4 @@ Use a fallback component: settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) - settings["DOWNLOAD_HANDLERS"]["http"] = "path.to.MyHandler" + settings["DOWNLOAD_HANDLERS"]["https"] = "path.to.MyHandler" diff --git a/tests/test_addons.py b/tests/test_addons.py index 95377d718..4156762d3 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,18 +1,24 @@ +import itertools import unittest -from typing import Any, Dict, Optional +from typing import Any, Dict -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings +from scrapy import Spider +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler -class GoodAddon: - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: - super().__init__() - self.config = config or {} - +class SimpleAddon: def update_settings(self, settings): - settings.update(self.config, "addon") + pass + + +def get_addon_cls(config: Dict[str, Any]) -> type: + class AddonWithConfig: + def update_settings(self, settings: BaseSettings): + settings.update(config, priority="addon") + + return AddonWithConfig class CreateInstanceAddon: @@ -35,7 +41,7 @@ class AddonTest(unittest.TestCase): settings.set("KEY1", "default", priority="default") settings.set("KEY2", "project", priority="project") addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} - testaddon = GoodAddon(addon_config) + testaddon = get_addon_cls(addon_config)() testaddon.update_settings(settings) self.assertEqual(settings["KEY1"], "addon") self.assertEqual(settings["KEY2"], "project") @@ -45,11 +51,27 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): def test_load_settings(self): settings_dict = { - "ADDONS": {"tests.test_addons.GoodAddon": 0}, + "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], GoodAddon) + self.assertIsInstance(manager.addons[0], SimpleAddon) + + def test_load_settings_order(self): + # Get three addons with different settings + addonlist = [] + for i in range(3): + addon = get_addon_cls({"KEY1": i}) + addon.number = i + addonlist.append(addon) + # Test for every possible ordering + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.number for a in ordered_addons] + settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + crawler = get_crawler(settings_dict=settings) + manager = crawler.addons + self.assertEqual([a.number for a in manager.addons], expected_order) + self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) def test_create_instance(self): settings_dict = { @@ -60,3 +82,64 @@ class AddonManagerTest(unittest.TestCase): manager = crawler.addons self.assertIsInstance(manager.addons[0], CreateInstanceAddon) self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + + def test_settings_priority(self): + config = { + "KEY": 15, # priority=addon + } + settings_dict = { + "ADDONS": {get_addon_cls(config): 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings_dict = { + "KEY": 20, # priority=project + "ADDONS": {get_addon_cls(config): 1}, + } + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 20) + + def test_fallback_workflow(self): + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + class AddonWithFallback: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual( + crawler.settings.get(FALLBACK_SETTING), + "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ) + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") From b67a81b81dd7c2a1b2017482551b3d818026f04f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 20:25:56 +0400 Subject: [PATCH 165/211] Use the MyHandler class directly. --- docs/topics/addons.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index edb1e7883..8dddb7c91 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -186,4 +186,4 @@ Use a fallback component: settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) - settings["DOWNLOAD_HANDLERS"]["https"] = "path.to.MyHandler" + settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler From 41a4a163e3716bf0c14160373844bcded0e38abd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 21:09:18 +0400 Subject: [PATCH 166/211] Handle NotConfigured in add-ons. --- docs/topics/addons.rst | 6 +++++- scrapy/addons.py | 26 +++++++++++--------------- scrapy/crawler.py | 1 - tests/test_addons.py | 13 +++++++++++++ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8dddb7c91..32c085079 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -77,6 +77,10 @@ modify :setting:`ITEM_PIPELINES`:: if settings.getbool("MYADDON_ENABLE_PIPELINE"): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +If the ``update_settings`` method raises +:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This +makes it easy to enable an add-on only when some conditions are met. + Fallbacks --------- @@ -130,7 +134,7 @@ Check dependencies: try: import boto except ImportError: - raise RuntimeError("MyAddon requires the boto library") + raise NotConfigured("MyAddon requires the boto library") ... Access the crawler instance: diff --git a/scrapy/addons.py b/scrapy/addons.py index e72c5da98..072143107 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,6 +1,8 @@ import logging from typing import TYPE_CHECKING, Any, List +from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import create_instance, load_object @@ -17,19 +19,23 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def _add(self, addon: Any) -> None: + def _add(self, addon: Any, settings: Settings) -> None: """Store an add-on.""" if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): addon = create_instance(addon, settings=None, crawler=self.crawler) - self.addons.append(addon) + try: + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured: + pass - def load_settings(self, settings) -> None: + def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object. This will load the add-on for every add-on path in the - ``ADDONS`` setting. + ``ADDONS`` setting and execute their ``update_settings`` methods. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration @@ -37,7 +43,7 @@ class AddonManager: """ addons = build_component_list(settings["ADDONS"]) for addon in build_component_list(settings["ADDONS"]): - self._add(addon) + self._add(addon, settings) logger.info( "Enabled addons:\n%(addons)s", { @@ -45,13 +51,3 @@ class AddonManager: }, extra={"crawler": self.crawler}, ) - - def update_settings(self, settings) -> None: - """Call ``update_settings()`` of all held add-ons. - - :param settings: The :class:`~scrapy.settings.Settings` object to be \ - updated - :type settings: :class:`~scrapy.settings.Settings` - """ - for addon in self.addons: - addon.update_settings(settings) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 12256440b..bf69cee26 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,7 +71,6 @@ class Crawler: self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) - self.addons.update_settings(self.settings) self.signals: SignalManager = SignalManager(self) diff --git a/tests/test_addons.py b/tests/test_addons.py index 4156762d3..5d053ed52 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -4,6 +4,7 @@ from typing import Any, Dict from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler @@ -57,6 +58,18 @@ class AddonManagerTest(unittest.TestCase): manager = crawler.addons self.assertIsInstance(manager.addons[0], SimpleAddon) + def test_notconfigured(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured() + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertFalse(manager.addons) + def test_load_settings_order(self): # Get three addons with different settings addonlist = [] From cf55eb05f59d7003bfb1491198e46853976dfb9e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 13:30:56 +0400 Subject: [PATCH 167/211] Fix a typo. --- docs/topics/addons.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 32c085079..1bf2172bd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -78,8 +78,8 @@ modify :setting:`ITEM_PIPELINES`:: settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 If the ``update_settings`` method raises -:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This -makes it easy to enable an add-on only when some conditions are met. +:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes +it easy to enable an add-on only when some conditions are met. Fallbacks --------- From 7fdeb5c5c100fd94fd0fe2b8abc0c3fc0ba8c54d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 16:58:18 +0400 Subject: [PATCH 168/211] Improve NotConfigured handling in add-ons. --- scrapy/addons.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 072143107..cd0cfb7de 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -19,18 +19,6 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def _add(self, addon: Any, settings: Settings) -> None: - """Store an add-on.""" - if isinstance(addon, (type, str)): - addon = load_object(addon) - if isinstance(addon, type): - addon = create_instance(addon, settings=None, crawler=self.crawler) - try: - addon.update_settings(settings) - self.addons.append(addon) - except NotConfigured: - pass - def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object. @@ -41,13 +29,29 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - addons = build_component_list(settings["ADDONS"]) - for addon in build_component_list(settings["ADDONS"]): - self._add(addon, settings) + enabled = [] + for clspath in build_component_list(settings["ADDONS"]): + try: + addoncls = load_object(clspath) + addon = create_instance( + addoncls, settings=settings, crawler=self.crawler + ) + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured as e: + if e.args: + clsname = ( + clspath.split(".")[-1] if isinstance(clspath, str) else clspath + ) + logger.warning( + "Disabled %(clsname)s: %(eargs)s", + {"clsname": clsname, "eargs": e.args[0]}, + extra={"crawler": self.crawler}, + ) logger.info( "Enabled addons:\n%(addons)s", { - "addons": addons, + "addons": enabled, }, extra={"crawler": self.crawler}, ) From f803ad63f32b488b3b89a81c6069d3575910e28e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 17:23:09 +0400 Subject: [PATCH 169/211] Fix a typing issue. --- scrapy/addons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index cd0cfb7de..f1ddfd211 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -29,7 +29,7 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - enabled = [] + enabled: List[Any] = [] for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) From e58b8078f0d51148c1866d1da3f6e36609b5e2a5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 20:20:15 +0400 Subject: [PATCH 170/211] Improve NotConfigured handling when clspath is a class. --- scrapy/addons.py | 7 ++----- scrapy/middleware.py | 5 ++--- tests/test_middleware.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index f1ddfd211..02dd4fde8 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -40,12 +40,9 @@ class AddonManager: self.addons.append(addon) except NotConfigured as e: if e.args: - clsname = ( - clspath.split(".")[-1] if isinstance(clspath, str) else clspath - ) logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": self.crawler}, ) logger.info( diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 03e92b565..04b838d2d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -46,10 +46,9 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split(".")[-1] logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": crawler}, ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 00ff746ee..a42c7b3d1 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,7 +39,7 @@ class MOff: pass def __init__(self): - raise NotConfigured + raise NotConfigured("foo") class TestMiddlewareManager(MiddlewareManager): From 72462a53e2cfcac3ec6068fc94fec657c73e157c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 12:32:53 +0400 Subject: [PATCH 171/211] Add more docs for update_settings(). --- docs/topics/settings.rst | 20 ++++++++++++++++++-- docs/topics/spiders.rst | 17 +++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 219509c1e..d0f2acd89 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -66,8 +66,8 @@ Example:: ---------------------- Spiders (See the :ref:`topics-spiders` chapter for reference) can define their -own settings that will take precedence and override the project ones. They can -do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: +own settings that will take precedence and override the project ones. One way +to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: .. code-block:: python @@ -81,6 +81,22 @@ do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: "SOME_SETTING": "some value", } +It's often better to provide a :meth:`~scrapy.Spider.update_settings` instead, +and settings set there should use the "spider" priority explicitly: + +.. code-block:: python + + import scrapy + + + class MySpider(scrapy.Spider): + name = "myspider" + + @classmethod + def update_settings(cls, settings): + settings.set("SOME_SETTING", "some value", priority="spider") + super().update_settings(settings) + 3. Project settings module -------------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0f6c2b1ba..97b525bd6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -151,13 +151,18 @@ scrapy.Spider and is called during initialization of a spider instance. It takes a :class:`~scrapy.settings.Settings` object as a parameter and - can add or update the spider's configuration values. This method is a class method, - meaning that it is called on the :class:`~scrapy.Spider` class and allows all instances - of the spider to share the same configuration. + can add or update the spider's configuration values. This method is a + class method, meaning that it is called on the :class:`~scrapy.Spider` + class and allows all instances of the spider to share the same + configuration. - One of the main advantages of ``update_settings()`` is that it allows - you to dynamically add, remove or change settings based on other settings - or other external factors. + While per-spider settings can be set in + :attr:`~scrapy.Spider.custom_settings`, using ``update_settings()`` + allows you to dynamically add, remove or change settings based on other + settings, spider attributes or other factors and use setting priorities + other than ``'spider'``. Also, it's easy to extend ``update_settings()`` + in a subclass by overriding it, while doing the same with + :attr:`~scrapy.Spider.custom_settings` is hard or impossible. For example, suppose a spider needs to modify :setting:`FEEDS`: From 9f9a2292e08cb0944e1e88e64cef1f4b17486ec8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 16:21:06 +0400 Subject: [PATCH 172/211] Deprecate the custom attribute of build_component_list(). (#5993) --- scrapy/utils/conf.py | 14 ++++++--- tests/test_utils_conf.py | 64 +++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 43a8b65a5..1889f7571 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -50,11 +50,17 @@ def build_component_list(compdict, custom=None, convert=update_classpath): "please provide a real number or None instead" ) - if isinstance(custom, (list, tuple)): - _check_components(custom) - return type(custom)(convert(c) for c in custom) - if custom is not None: + warnings.warn( + "The 'custom' attribute of build_component_list() is deprecated. " + "Please merge its value into 'compdict' manually or change your " + "code to use Settings.getwithbase().", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if isinstance(custom, (list, tuple)): + _check_components(custom) + return type(custom)(convert(c) for c in custom) compdict.update(custom) _validate_values(compdict) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 78ed9a7c9..dc3f01d57 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,6 +1,8 @@ import unittest import warnings +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( @@ -21,44 +23,45 @@ class BuildComponentListTest(unittest.TestCase): def test_backward_compatible_build_dict(self): base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None} custom = {"two": None, "three": 8, "four": 4} - self.assertEqual( - build_component_list(base, custom, convert=lambda x: x), - ["one", "four", "five", "three"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(base, custom, convert=lambda x: x), + ["one", "four", "five", "three"], + ) def test_return_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, convert=lambda x: x), custom - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, convert=lambda x: x), custom + ) def test_map_dict(self): custom = {"one": 1, "two": 2, "three": 3} - self.assertEqual( - build_component_list({}, custom, convert=lambda x: x.upper()), - ["ONE", "TWO", "THREE"], - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list({}, custom, convert=lambda x: x.upper()), + ["ONE", "TWO", "THREE"], + ) def test_map_list(self): custom = ["a", "b", "c"] - self.assertEqual( - build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] - ) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + self.assertEqual( + build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"] + ) def test_duplicate_components_in_dict(self): duplicate_dict = {"one": 1, "two": 2, "ONE": 4} - self.assertRaises( - ValueError, - build_component_list, - {}, - duplicate_dict, - convert=lambda x: x.lower(), - ) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, duplicate_dict, convert=lambda x: x.lower()) def test_duplicate_components_in_list(self): duplicate_list = ["a", "b", "a"] with self.assertRaises(ValueError) as cm: - build_component_list(None, duplicate_list, convert=lambda x: x) + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list(None, duplicate_list, convert=lambda x: x) self.assertIn(str(duplicate_list), str(cm.exception)) def test_duplicate_components_in_basesettings(self): @@ -76,9 +79,8 @@ class BuildComponentListTest(unittest.TestCase): ) # Same priority raises ValueError duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20) - self.assertRaises( - ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower() - ) + with self.assertRaises(ValueError): + build_component_list(duplicate_bs, convert=lambda x: x.lower()) def test_valid_numbers(self): # work well with None and numeric values @@ -92,15 +94,9 @@ class BuildComponentListTest(unittest.TestCase): self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"]) # raise exception for invalid values d = {"one": "5"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "1.0"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": [1, 2, 3]} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": {"a": "a", "b": 2}} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {"one": "lorem ipsum"} - self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + with self.assertRaises(ValueError): + with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"): + build_component_list({}, d, convert=lambda x: x) class UtilsConfTestCase(unittest.TestCase): From af1be835e4a6c14634acb382568935c1a7e10445 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 19:46:16 +0400 Subject: [PATCH 173/211] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/settings.rst | 4 ++-- docs/topics/spiders.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d0f2acd89..0963f835f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -81,7 +81,7 @@ to do so is by setting their :attr:`~scrapy.Spider.custom_settings` attribute: "SOME_SETTING": "some value", } -It's often better to provide a :meth:`~scrapy.Spider.update_settings` instead, +It's often better to implement :meth:`~scrapy.Spider.update_settings` instead, and settings set there should use the "spider" priority explicitly: .. code-block:: python @@ -94,8 +94,8 @@ and settings set there should use the "spider" priority explicitly: @classmethod def update_settings(cls, settings): - settings.set("SOME_SETTING", "some value", priority="spider") super().update_settings(settings) + settings.set("SOME_SETTING", "some value", priority="spider") 3. Project settings module -------------------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 97b525bd6..5c3bf6e72 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -162,7 +162,7 @@ scrapy.Spider settings, spider attributes or other factors and use setting priorities other than ``'spider'``. Also, it's easy to extend ``update_settings()`` in a subclass by overriding it, while doing the same with - :attr:`~scrapy.Spider.custom_settings` is hard or impossible. + :attr:`~scrapy.Spider.custom_settings` can be hard. For example, suppose a spider needs to modify :setting:`FEEDS`: @@ -182,8 +182,8 @@ scrapy.Spider @classmethod def update_settings(cls, settings): - settings.setdefault("FEEDS", {}).update(cls.custom_feed) super().update_settings(settings) + settings.setdefault("FEEDS", {}).update(cls.custom_feed) .. method:: start_requests() From b9c32a0cfd13dea2a59d20735f33ceff18daac97 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 3 Aug 2023 12:06:55 -0300 Subject: [PATCH 174/211] Remove functions deprecated in 2.6.0 (#5996) --- scrapy/extensions/feedexport.py | 9 --------- scrapy/squeues.py | 30 ------------------------------ scrapy/utils/reqser.py | 27 --------------------------- tests/test_feedexport.py | 5 +---- tests/test_request_dict.py | 28 ---------------------------- 5 files changed, 1 insertion(+), 98 deletions(-) delete mode 100644 scrapy/utils/reqser.py diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4687c7382..c8022ff57 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -672,21 +672,12 @@ class FeedExporter: params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 - original_params = params.copy() uripar_function = ( load_object(uri_params_function) if uri_params_function else lambda params, _: params ) new_params = uripar_function(params, spider) - if new_params is None or original_params != params: - warnings.warn( - "Modifying the params dictionary in-place in the function defined in " - "the FEED_URI_PARAMS setting or in the uri_params key of the FEEDS " - "setting is deprecated. The function must return a new dictionary " - "instead.", - category=ScrapyDeprecationWarning, - ) return new_params if new_params is not None else params def _load_filter(self, feed_options): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 6afe0d636..f665ad88c 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -10,7 +10,6 @@ from typing import Union from queuelib import queue -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.request import request_from_dict @@ -143,32 +142,3 @@ MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDisk MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) - - -# deprecated queue classes -_subclass_warn_message = "{cls} inherits from deprecated class {old}" -_instance_warn_message = "{cls} is deprecated" -PickleFifoDiskQueueNonRequest = create_deprecated_class( - name="PickleFifoDiskQueueNonRequest", - new_class=_PickleFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -PickleLifoDiskQueueNonRequest = create_deprecated_class( - name="PickleLifoDiskQueueNonRequest", - new_class=_PickleLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalFifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalFifoDiskQueueNonRequest", - new_class=_MarshalFifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) -MarshalLifoDiskQueueNonRequest = create_deprecated_class( - name="MarshalLifoDiskQueueNonRequest", - new_class=_MarshalLifoSerializationDiskQueue, - subclass_warn_message=_subclass_warn_message, - instance_warn_message=_instance_warn_message, -) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py deleted file mode 100644 index 15705db83..000000000 --- a/scrapy/utils/reqser.py +++ /dev/null @@ -1,27 +0,0 @@ -import warnings -from typing import Optional - -import scrapy -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.request import request_from_dict as _from_dict - -warnings.warn( - ( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead" - ), - category=ScrapyDeprecationWarning, - stacklevel=2, -) - - -def request_to_dict( - request: "scrapy.Request", spider: Optional["scrapy.Spider"] = None -) -> dict: - return request.to_dict(spider=spider) - - -def request_from_dict( - d: dict, spider: Optional["scrapy.Spider"] = None -) -> "scrapy.Request": - return _from_dict(d, spider=spider) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9e4fb53bf..46bd5733a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -3067,10 +3067,7 @@ class URIParamsTest: spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with pytest.warns( - ScrapyDeprecationWarning, match="Modifying the params dictionary in-place" - ): - feed_exporter.open_spider(spider) + feed_exporter.open_spider(spider) self.assertEqual(feed_exporter.slots[0].uri, f"file:///tmp/{self.spider_name}") diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 8665a9205..7312eb036 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,10 +1,6 @@ -import sys import unittest -import warnings -from contextlib import suppress from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, JsonRequest from scrapy.utils.request import request_from_dict @@ -162,30 +158,6 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo")) -class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest): - def _assert_serializes_ok(self, request, spider=None): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - with suppress(KeyError): - del sys.modules[ - "scrapy.utils.reqser" - ] # delete module to reset the deprecation warning - - from scrapy.utils.reqser import request_from_dict as _from_dict - from scrapy.utils.reqser import request_to_dict as _to_dict - - request_copy = _from_dict(_to_dict(request, spider), spider) - self._assert_same_request(request, request_copy) - - self.assertEqual(len(caught), 1) - self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning)) - self.assertEqual( - "Module scrapy.utils.reqser is deprecated, please use request.to_dict method" - " and/or scrapy.utils.request.request_from_dict instead", - str(caught[0].message), - ) - - class TestSpiderMixin: def __mixin_callback(self, response): pass From 8a0a9e6d3e5398b884e92cb9336efe3fd19677d6 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:31:12 -0300 Subject: [PATCH 175/211] Enable Python 3.11 on Windows CI --- .github/workflows/tests-windows.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5bcf74d5e..c8d1928d7 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -20,13 +20,12 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio -# no binary package for lxml for 3.11 yet -# - python-version: "3.11" -# env: -# TOXENV: py -# - python-version: "3.11" -# env: -# TOXENV: asyncio + - python-version: "3.11" + env: + TOXENV: py + - python-version: "3.11" + env: + TOXENV: asyncio steps: - uses: actions/checkout@v3 From 09c63a178bf28a66b564562125497721bea371b3 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:53:04 -0300 Subject: [PATCH 176/211] Remove the deprecated spider parameter from the engine (#5998) --- scrapy/core/engine.py | 93 ++++++++----------------------------------- tests/test_engine.py | 87 +--------------------------------------- 2 files changed, 18 insertions(+), 162 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 3e5a281b2..dad384ddc 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -5,7 +5,6 @@ For more information see docs/topics/architecture.rst """ import logging -import warnings from time import time from typing import ( TYPE_CHECKING, @@ -14,7 +13,6 @@ from typing import ( Generator, Iterable, Iterator, - List, Optional, Set, Type, @@ -29,7 +27,7 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider, DontCloseSpider from scrapy.http import Request, Response from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings @@ -213,7 +211,7 @@ class ExecutionEngine: if request is None: return None - d = self._download(request, self.spider) + d = self._download(request) d.addBoth(self._handle_downloader_output, request) d.addErrback( lambda f: logger.info( @@ -266,13 +264,7 @@ class ExecutionEngine: ) return d - def spider_is_idle(self, spider: Optional[Spider] = None) -> bool: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.spider_is_idle is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + def spider_is_idle(self) -> bool: if self.slot is None: raise RuntimeError("Engine slot not assigned") if not self.scraper.slot.is_idle(): # type: ignore[union-attr] @@ -285,18 +277,8 @@ class ExecutionEngine: return False return True - def crawl(self, request: Request, spider: Optional[Spider] = None) -> None: + def crawl(self, request: Request) -> None: """Inject the request into the spider <-> downloader pipeline""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - raise RuntimeError( - f"The spider {spider.name!r} does not match the open spider" - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") self._schedule_request(request, self.spider) @@ -311,39 +293,24 @@ class ExecutionEngine: signals.request_dropped, request=request, spider=spider ) - def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: + def download(self, request: Request) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to ExecutionEngine.download is deprecated", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if spider is not self.spider: - logger.warning( - "The spider '%s' does not match the open spider", spider.name - ) if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - return self._download(request, spider).addBoth( - self._downloaded, request, spider - ) + return self._download(request).addBoth(self._downloaded, request) def _downloaded( - self, result: Union[Response, Request], request: Request, spider: Spider + self, result: Union[Response, Request], request: Request ) -> Union[Deferred, Response]: assert self.slot is not None # typing self.slot.remove_request(request) - return self.download(result, spider) if isinstance(result, Request) else result + return self.download(result) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: + def _download(self, request: Request) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) - if spider is None: - spider = self.spider - def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError( @@ -352,15 +319,17 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request - assert spider is not None - logkws = self.logformatter.crawled(result.request, result, spider) + assert self.spider is not None + logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.spider} + ) self.signals.send_catch_log( signal=signals.response_received, response=result, request=result.request, - spider=spider, + spider=self.spider, ) return result @@ -369,8 +338,8 @@ class ExecutionEngine: self.slot.nextcall.schedule() return _ - assert spider is not None - dwld = self.downloader.fetch(request, spider) + assert self.spider is not None + dwld = self.downloader.fetch(request, self.spider) dwld.addCallbacks(_on_success) dwld.addBoth(_on_complete) return dwld @@ -485,31 +454,3 @@ class ExecutionEngine: dfd.addBoth(lambda _: self._spider_closed_callback(spider)) return dfd - - @property - def open_spiders(self) -> List[Spider]: - warnings.warn( - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return [self.spider] if self.spider is not None else [] - - def has_capacity(self) -> bool: - warnings.warn( - "ExecutionEngine.has_capacity is deprecated", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return not bool(self.slot) - - def schedule(self, request: Request, spider: Spider) -> None: - warnings.warn( - "ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if self.slot is None: - raise RuntimeError("Engine slot not assigned") - self._schedule_request(request, spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 410eba921..8d7afb6a1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -20,7 +20,6 @@ from threading import Timer from urllib.parse import urlparse import attr -import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer, reactor @@ -29,7 +28,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider from scrapy.http import Request from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor @@ -436,90 +435,6 @@ class EngineTest(unittest.TestCase): finally: yield e.stop() - @defer.inlineCallbacks - def test_close_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_crawl_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.crawl is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.crawl(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_download_deprecated_spider_arg(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.download is deprecated", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.download(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_schedule(self): - with pytest.warns( - ScrapyDeprecationWarning, - match="ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.schedule(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_has_capacity(self): - with pytest.warns( - ScrapyDeprecationWarning, match="ExecutionEngine.has_capacity is deprecated" - ): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - self.assertTrue(e.has_capacity()) - spider = TestSpider() - yield e.open_spider(spider, []) - self.assertFalse(e.has_capacity()) - e.start() - yield e.close() - self.assertTrue(e.has_capacity()) - def test_short_timeout(self): args = ( sys.executable, From 72de48be6d351cc504ec901e39bf649d84214665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 4 Aug 2023 09:56:30 +0200 Subject: [PATCH 177/211] asyncio: cover accidental bad reactor installation, sort sections, reword the Windows section --- docs/topics/asyncio.rst | 100 ++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..07baea071 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -27,54 +27,43 @@ reactor manually. You can do that using install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') -.. _using-custom-loops: +.. _asyncio-preinstalled-reactor: -Using custom asyncio loops -========================== +Handling a pre-installed reactor +================================ -You can also use custom asyncio event loops with the asyncio reactor. Set the -:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to -use it instead of the default asyncio event loop. +``twisted.internet.reactor`` and some other Twisted imports install the default +Twisted reactor as a side effect. Once a Twisted reactor is installed, it is +not possible to switch to a different reactor at run time. + +If you :ref:`configure the asyncio Twisted reactor ` and, at +run time, Scrapy complains that a different reactor is already installed, +chances are you have some such imports in your code. + +You can usually fix the issue by moving those offending module-level Twisted +imports to the method or function definitions where they are used. For example, +if you have something like: + +.. code-block:: python + + from twisted.internet import reactor -.. _asyncio-windows: + def my_function(): + reactor.callLater(...) -Windows-specific notes -====================== +Switch to something like: -The Windows implementation of :mod:`asyncio` can use two event loop -implementations: +.. code-block:: python -- :class:`~asyncio.SelectorEventLoop`, default before Python 3.8, required - when using Twisted. + def my_function(): + from twisted.internet import reactor -- :class:`~asyncio.ProactorEventLoop`, default since Python 3.8, cannot work - with Twisted. + reactor.callLater(...) -So on Python 3.8+ the event loop class needs to be changed. - -.. versionchanged:: 2.6.0 - The event loop class is changed automatically when you change the - :setting:`TWISTED_REACTOR` setting or call - :func:`~scrapy.utils.reactor.install_reactor`. - -To change the event loop class manually, call the following code before -installing the reactor:: - - import asyncio - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - -You can put this in the same function that installs the reactor, if you do that -yourself, or in some code that runs before the reactor is installed, e.g. -``settings.py``. - -.. note:: Other libraries you use may require - :class:`~asyncio.ProactorEventLoop`, e.g. because it supports - subprocesses (this is the case with `playwright`_), so you cannot use - them together with Scrapy on Windows (but you should be able to use - them on WSL or native Linux). - -.. _playwright: https://github.com/microsoft/playwright-python +Alternatively, you can try to :ref:`manually install the asyncio reactor +`, with :func:`~scrapy.utils.reactor.install_reactor`, before +those imports happen. .. _asyncio-await-dfd: @@ -122,3 +111,36 @@ example: f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) + + +.. _asyncio-windows: + +Windows-specific notes +====================== + +The Windows implementation of :mod:`asyncio` can use two event loop +implementations, :class:`~asyncio.ProactorEventLoop` (default) and +:class:`~asyncio.SelectorEventLoop`. However, only +:class:`~asyncio.SelectorEventLoop` works with Twisted. + +Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` +automatically when you change the :setting:`TWISTED_REACTOR` setting or call +:func:`~scrapy.utils.reactor.install_reactor`. + +.. note:: Other libraries you use may require + :class:`~asyncio.ProactorEventLoop`, e.g. because it supports + subprocesses (this is the case with `playwright`_), so you cannot use + them together with Scrapy on Windows (but you should be able to use + them on WSL or native Linux). + +.. _playwright: https://github.com/microsoft/playwright-python + + +.. _using-custom-loops: + +Using custom asyncio loops +========================== + +You can also use custom asyncio event loops with the asyncio reactor. Set the +:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event +loop class to use it instead of the default asyncio event loop. From c14a0a9d5d0443165e1581a039803dde648b9ee6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 16:36:27 +0400 Subject: [PATCH 178/211] Add release notes for 2.10.0. --- docs/news.rst | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c7ad11862..ae51d05dc 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,176 @@ Release notes ============= +.. _release-2.10.0: + +Scrapy 2.10.0 (YYYY-MM-DD) +-------------------------- + +Highlights: + +- Added Python 3.12 support, dropped Python 3.7 support. + +- The add-ons framework that simplifies configuring 3rd-party components that + support it. + +- Exceptions to retry can now be configured. + +- Many fixes and improvements for feed exports. + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.7. (:issue:`5953`) + +- Added support for the upcoming Python 3.12. (:issue:`5984`) + +- Minimum versions increased for these dependencies: + + - lxml_: 4.3.0 → 4.4.1 + + - cryptography_: 3.4.6 → 36.0.0 + +- ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`) + +- boto3_ is now recommended for exporting to S3 instead of botocore_. + (:issue:`5833`). + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The value of the :setting:`FEED_STORE_EMPTY` is now ``True`` instead of + ``False``. In earlier Scrapy versions empty files were created even when + this setting was ``False`` (which was a bug that is now fixed), so the new + default should keep the old behavior. (:issue:`872`, :issue:`5847`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- When a function is assigned to the :setting:`FEED_URI_PARAMS` setting, + returning ``None`` or modifying the ``params`` input parameter, deprecated + in Scrapy 2.6, is no longer supported. (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.utils.reqser`` module, deprecated in Scrapy 2.6, is removed. + (:issue:`5994`, :issue:`5996`) + +- The ``scrapy.squeues`` classes ``PickleFifoDiskQueueNonRequest``, + ``PickleLifoDiskQueueNonRequest``, ``MarshalFifoDiskQueueNonRequest``, + and ``MarshalLifoDiskQueueNonRequest``, deprecated in + Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5996`) + +- The property ``open_spiders`` and the methods ``has_capacity`` and + ``schedule`` of :class:`scrapy.core.engine.ExecutionEngine`, + deprecated in Scrapy 2.6, are removed. (:issue:`5994`, :issue:`5998`) + +- Passing a ``spider`` argument to the + :meth:`~scrapy.core.engine.ExecutionEngine.spider_is_idle`, + :meth:`~scrapy.core.engine.ExecutionEngine.crawl` and + :meth:`~scrapy.core.engine.ExecutionEngine.download` methods of + :class:`scrapy.core.engine.ExecutionEngine`, deprecated in Scrapy 2.6, is + no longer supported. (:issue:`5994`, :issue:`5998`) + +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.utils.datatypes.CaselessDict` is deprecated, use + :class:`scrapy.utils.datatypes.CaseInsensitiveDict` instead. + (:issue:`5146`) + +- Passing the ``custom`` argument to + :func:`scrapy.utils.conf.build_component_list` is deprecated, it was used + in the past to merge ``FOO`` and ``FOO_BASE`` setting values but now Scrapy + uses :func:`scrapy.settings.BaseSettings.getwithbase` to do the same. + Code that uses this argument and cannot be switched to ``getwithbase()`` + can be switched to merging the values explicitly. (:issue:`5726`, + :issue:`5923`) + +New features +~~~~~~~~~~~~ + +- Added support for :ref:`Scrapy add-ons `. (:issue:`5950`) + +- Added the :setting:`RETRY_EXCEPTIONS` setting that configures which + exceptions will be retried by + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`. + (:issue:`2701`, :issue:`5929`) + +- Added the possiiblity to close the spider if no items were produced in the + specified time, configured by :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + (:issue:`5979`) + +- Added support for the :setting:`AWS_REGION_NAME` setting to feed exports. + (:issue:`5980`) + +- Added support for using :class:`pathlib.Path` objects that refer to + absolute Windows paths in the :setting:`FEEDS` setting. (:issue:`5939`) + +Bug fixes +~~~~~~~~~ + +- Fixed creating empty feeds even with ``FEED_STORE_EMPTY=False``. + (:issue:`872`, :issue:`5847`) + +- Fixed using absolute Windows paths when specifying output files. + (:issue:`5969`, :issue:`5971`) + +- Fixed problems with uploading large files to S3 by switching to multipart + uploads (requires boto3_). (:issue:`960`, :issue:`5735`, :issue:`5833`) + +- Fixed the JSON exporter writing extra commas when some exceptions occur. + (:issue:`3090`, :issue:`5952`) + +- Fixed the "read of closed file" error in the CSV exporter. (:issue:`5043`, + :issue:`5705`) + +- Fixed an error when a component added by the class object throws + :exc:`~scrapy.exceptions.NotConfigured` with a message. (:issue:`5950`, + :issue:`5992`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.pop` method. + (:issue:`5959`, :issue:`5960`, :issue:`5963`) + +- Added :class:`~scrapy.utils.datatypes.CaseInsensitiveDict` as a replacement + for :class:`~scrapy.utils.datatypes.CaselessDict` that fixes some API + inconsistencies. (:issue:`5146`) + +Documentation +~~~~~~~~~~~~~ + +- Documented :meth:`scrapy.Spider.update_settings`. (:issue:`5745`, + :issue:`5846`) + +- Documented possible problems with early Twisted reactor installation and + their solutions. (:issue:`5981`, :issue:`6000`) + +- Added examples of making additional requests in callbacks. (:issue:`5927`) + +- Improved the feed export docs. (:issue:`5579`, :issue:`5931`) + +- Clarified the docs about request objects on redirection. (:issue:`5707`, + :issue:`5937`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added support for running tests against the installed Scrapy version. + (:issue:`4914`, :issue:`5949`) + +- Extended typing hints. (:issue:`5925`, :issue:`5977`) + +- Fixed the ``test_utils_asyncio.AsyncioTest.test_set_asyncio_event_loop`` + test. (:issue:`5951`) + +- Fixed the ``test_feedexport.BatchDeliveriesTest.test_batch_path_differ`` + test on Windows. (:issue:`5847`) + +- Enabled CI runs for Python 3.11 on Windows. (:issue:`5999`) + +- Simplified skipping tests that depend on ``uvloop``. (:issue:`5984`) + +- Fixed the ``extra-deps-pinned`` tox env. (:issue:`5948`) + +- Implemented cleanups. (:issue:`5965`, :issue:`5986`) + .. _release-2.9.0: Scrapy 2.9.0 (2023-05-08) @@ -5748,6 +5918,7 @@ First release of Scrapy. .. _AJAX crawlable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 +.. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ From 7fe4c0c9f7e58d8099c6ec47560c2ffd181bebb8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 16:56:39 +0400 Subject: [PATCH 179/211] Update docs/news.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index ae51d05dc..80ad0fa45 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,7 @@ Highlights: - Added Python 3.12 support, dropped Python 3.7 support. -- The add-ons framework that simplifies configuring 3rd-party components that +- The new add-ons framework simplifies configuring 3rd-party components that support it. - Exceptions to retry can now be configured. From 022ef0f86b2a5b09cd1044d6a381f025e821ccf3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 17:05:09 +0400 Subject: [PATCH 180/211] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/news.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 80ad0fa45..940093e58 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -34,16 +34,17 @@ Modified requirements - ``pkg_resources`` is no longer used. (:issue:`5956`, :issue:`5958`) -- boto3_ is now recommended for exporting to S3 instead of botocore_. +- boto3_ is now recommended instead of botocore_ for exporting to S3. (:issue:`5833`). Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` is now ``True`` instead of - ``False``. In earlier Scrapy versions empty files were created even when - this setting was ``False`` (which was a bug that is now fixed), so the new - default should keep the old behavior. (:issue:`872`, :issue:`5847`) +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, + :issue:`5847`) Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From 88327c7c58928b5e1e07921ca055c8579c197234 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 4 Aug 2023 17:23:30 +0400 Subject: [PATCH 181/211] =?UTF-8?q?Bump=20version:=202.9.0=20=E2=86=92=202?= =?UTF-8?q?.10.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a00b7cfb3..53e873427 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.9.0 +current_version = 2.10.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 940093e58..c55c0b222 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.10.0: -Scrapy 2.10.0 (YYYY-MM-DD) +Scrapy 2.10.0 (2023-08-04) -------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index c8e38b614..10c2c0c3d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.9.0 +2.10.0 From d31829b72f6238a92a42c0990953d5056e8f5778 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 23:13:15 +0400 Subject: [PATCH 182/211] More typing for scrapy/utils. --- scrapy/exceptions.py | 3 +- scrapy/extensions/feedexport.py | 25 +++++++++++------ scrapy/utils/conf.py | 49 ++++++++++++++++++++++----------- scrapy/utils/ftp.py | 15 ++++++++-- scrapy/utils/job.py | 2 +- scrapy/utils/ossignal.py | 3 +- scrapy/utils/project.py | 8 +++--- scrapy/utils/sitemap.py | 14 ++++++---- scrapy/utils/ssl.py | 7 ++--- scrapy/utils/trackref.py | 19 +++++++++---- 10 files changed, 93 insertions(+), 52 deletions(-) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index fedd02805..6e83e4a00 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ +from typing import Any # Internal @@ -77,7 +78,7 @@ class NotSupported(Exception): class UsageError(Exception): """To indicate a command-line usage error""" - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any): self.print_help = kw.pop("print_help", True) super().__init__(*a, **kw) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c8022ff57..2bbcaf3ad 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,7 +11,7 @@ import warnings from datetime import datetime from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -282,15 +282,22 @@ class GCSFeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri, use_active_mode=False, *, feed_options=None): + def __init__( + self, + uri: str, + use_active_mode: bool = False, + *, + feed_options: Optional[Dict[str, Any]] = None, + ): u = urlparse(uri) - self.host = u.hostname - self.port = int(u.port or "21") - self.username = u.username - self.password = unquote(u.password or "") - self.path = u.path - self.use_active_mode = use_active_mode - self.overwrite = not feed_options or feed_options.get("overwrite", True) + assert u.hostname + self.host: str = u.hostname + self.port: int = int(u.port or "21") + self.username: str = u.username or "" + self.password: str = unquote(u.password or "") + self.path: str = u.path + self.use_active_mode: bool = use_active_mode + self.overwrite: bool = not feed_options or feed_options.get("overwrite", True) @classmethod def from_crawler(cls, crawler, uri, *, feed_options=None): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 1889f7571..641dfa4a2 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,18 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import ( + Any, + Callable, + Collection, + Dict, + Iterable, + List, + Mapping, + MutableMapping, + Optional, + Union, +) from scrapy.exceptions import ScrapyDeprecationWarning, UsageError from scrapy.settings import BaseSettings @@ -13,17 +24,21 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values -def build_component_list(compdict, custom=None, convert=update_classpath): +def build_component_list( + compdict: MutableMapping[Any, Any], + custom: Any = None, + convert: Callable[[Any], Any] = update_classpath, +) -> List[Any]: """Compose a component list from a { class: order } dictionary.""" - def _check_components(complist): + def _check_components(complist: Collection[Any]) -> None: if len({convert(c) for c in complist}) != len(complist): raise ValueError( f"Some paths in {complist!r} convert to the same object, " "please update your settings" ) - def _map_keys(compdict): + def _map_keys(compdict: Mapping[Any, Any]) -> Union[BaseSettings, Dict[Any, Any]]: if isinstance(compdict, BaseSettings): compbs = BaseSettings() for k, v in compdict.items(): @@ -41,7 +56,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in compdict.items()} - def _validate_values(compdict): + def _validate_values(compdict: Mapping[Any, Any]) -> None: """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): @@ -60,7 +75,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): ) if isinstance(custom, (list, tuple)): _check_components(custom) - return type(custom)(convert(c) for c in custom) + return type(custom)(convert(c) for c in custom) # type: ignore[return-value] compdict.update(custom) _validate_values(compdict) @@ -68,7 +83,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] -def arglist_to_dict(arglist): +def arglist_to_dict(arglist: List[str]) -> Dict[str, str]: """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a dict """ @@ -91,7 +106,7 @@ def closest_scrapy_cfg( return closest_scrapy_cfg(path.parent, path) -def init_env(project="default", set_syspath=True): +def init_env(project: str = "default", set_syspath: bool = True) -> None: """Initialize environment to use command-line tool from inside a project dir. This sets the Scrapy settings module and modifies the Python path to be able to locate the project module. @@ -106,7 +121,7 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): +def get_config(use_closest: bool = True) -> ConfigParser: """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) cfg = ConfigParser() @@ -114,7 +129,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> List[str]: +def get_sources(use_closest: bool = True) -> List[str]: xdg_config_home = ( os.environ.get("XDG_CONFIG_HOME") or Path("~/.config").expanduser() ) @@ -129,7 +144,9 @@ def get_sources(use_closest=True) -> List[str]: return sources -def feed_complete_default_values_from_settings(feed, settings): +def feed_complete_default_values_from_settings( + feed: Dict[str, Any], settings: BaseSettings +) -> Dict[str, Any]: out = feed.copy() out.setdefault("batch_item_count", settings.getint("FEED_EXPORT_BATCH_ITEM_COUNT")) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) @@ -145,21 +162,21 @@ def feed_complete_default_values_from_settings(feed, settings): def feed_process_params_from_cli( - settings, + settings: BaseSettings, output: List[str], - output_format=None, + output_format: Optional[str] = None, overwrite_output: Optional[List[str]] = None, -): +) -> Dict[str, Dict[str, Any]]: """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary suitable to be used as the FEEDS setting. """ - valid_output_formats = without_none_values( + valid_output_formats: Iterable[str] = without_none_values( settings.getwithbase("FEED_EXPORTERS") ).keys() - def check_valid_format(output_format): + def check_valid_format(output_format: str) -> None: if output_format not in valid_output_formats: raise UsageError( f"Unrecognized output format '{output_format}'. " diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6bf6e9195..c77681a53 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,9 +1,10 @@ import posixpath from ftplib import FTP, error_perm from posixpath import dirname +from typing import IO -def ftp_makedirs_cwd(ftp, path, first_call=True): +def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None: """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they don't exist. The ftplib.FTP object must be already connected and logged in. @@ -18,8 +19,16 @@ def ftp_makedirs_cwd(ftp, path, first_call=True): def ftp_store_file( - *, path, file, host, port, username, password, use_active_mode=False, overwrite=True -): + *, + path: str, + file: IO, + host: str, + port: int, + username: str, + password: str, + use_active_mode: bool = False, + overwrite: bool = True, +) -> None: """Opens a FTP connection with passed credentials,sets current directory to the directory extracted from given path, then uploads the file to server """ diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 858affc03..c49f7d758 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -5,7 +5,7 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: - path = settings["JOBDIR"] + path: str = settings["JOBDIR"] if path and not Path(path).exists(): Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 7646264a8..f835a2221 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,4 +1,5 @@ import signal +from typing import Callable signal_names = {} for signame in dir(signal): @@ -8,7 +9,7 @@ for signame in dir(signal): signal_names[signum] = signame -def install_shutdown_handlers(function, override_sigint=True): +def install_shutdown_handlers(function: Callable, override_sigint: bool = True) -> None: """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the SIGINT handler won't be install if there is already a handler in place diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 652b74759..a2c224b90 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -11,7 +11,7 @@ ENVVAR = "SCRAPY_SETTINGS_MODULE" DATADIR_CFG_SECTION = "datadir" -def inside_project(): +def inside_project() -> bool: scrapy_module = os.environ.get(ENVVAR) if scrapy_module: try: @@ -25,7 +25,7 @@ def inside_project(): return bool(closest_scrapy_cfg()) -def project_data_dir(project="default") -> str: +def project_data_dir(project: str = "default") -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") @@ -44,7 +44,7 @@ def project_data_dir(project="default") -> str: return str(d) -def data_path(path: str, createdir=False) -> str: +def data_path(path: str, createdir: bool = False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. @@ -60,7 +60,7 @@ def data_path(path: str, createdir=False) -> str: return str(path_obj) -def get_project_settings(): +def get_project_settings() -> Settings: if ENVVAR not in os.environ: project = os.environ.get("SCRAPY_PROJECT", "default") init_env(project) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2622c2775..3d2ecc9a7 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,7 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ - +from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin import lxml.etree @@ -14,7 +14,7 @@ class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" - def __init__(self, xmltext): + def __init__(self, xmltext: str): xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) @@ -22,9 +22,9 @@ class Sitemap: rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt - def __iter__(self): + def __iter__(self) -> Iterator[Dict[str, Any]]: for elem in self._root.getchildren(): - d = {} + d: Dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag name = tag.split("}", 1)[1] if "}" in tag else tag @@ -39,11 +39,13 @@ class Sitemap: yield d -def sitemap_urls_from_robots(robots_text, base_url=None): +def sitemap_urls_from_robots( + robots_text: str, base_url: Optional[str] = None +) -> Generator[str, Any, None]: """Return an iterator over all sitemap urls contained in the given robots.txt file """ for line in robots_text.splitlines(): if line.lstrip().lower().startswith("sitemap:"): url = line.split(":", 1)[1].strip() - yield urljoin(base_url, url) + yield urljoin(base_url or "", url) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 03ae4ba9e..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, cast +from typing import Any, Optional import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL @@ -58,9 +58,6 @@ def get_temp_key_info(ssl_object: Any) -> Optional[str]: def get_openssl_version() -> str: - # https://github.com/python/typeshed/issues/10024 - system_openssl_bytes = cast( - bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) - ) + system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 01b980c93..9ff9a273f 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -12,9 +12,14 @@ alias to object in that case). from collections import defaultdict from operator import itemgetter from time import time -from typing import DefaultDict +from typing import TYPE_CHECKING, Any, DefaultDict, Iterable from weakref import WeakKeyDictionary +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + NoneType = type(None) live_refs: DefaultDict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) @@ -24,13 +29,14 @@ class object_ref: __slots__ = () - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: Any, **kwargs: Any) -> "Self": obj = object.__new__(cls) live_refs[cls][obj] = time() return obj -def format_live_refs(ignore=NoneType): +# using Any as it's hard to type type(None) +def format_live_refs(ignore: Any = NoneType) -> str: """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() @@ -44,12 +50,12 @@ def format_live_refs(ignore=NoneType): return s -def print_live_refs(*a, **kw): +def print_live_refs(*a: Any, **kw: Any) -> None: """Print tracked objects""" print(format_live_refs(*a, **kw)) -def get_oldest(class_name): +def get_oldest(class_name: str) -> Any: """Get the oldest object for a specific class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: @@ -58,8 +64,9 @@ def get_oldest(class_name): return min(wdict.items(), key=itemgetter(1))[0] -def iter_all(class_name): +def iter_all(class_name: str) -> Iterable[Any]: """Iterate over all objects of the same class by its class name""" for cls, wdict in live_refs.items(): if cls.__name__ == class_name: return wdict.keys() + return [] From d015329d759dda72586b856ee92d787dc730f06e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 01:54:28 +0400 Subject: [PATCH 183/211] Add more typing for scrapy/utils/log.py. --- scrapy/utils/log.py | 60 +++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2ce4725f4..2013bfc43 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,8 +1,11 @@ +from __future__ import annotations + import logging import sys import warnings from logging.config import dictConfig -from typing import Tuple +from types import TracebackType +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast from twisted.python import log as twisted_log from twisted.python.failure import Failure @@ -12,13 +15,25 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions +if TYPE_CHECKING: + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) -def failure_to_exc_info(failure: Failure): +def failure_to_exc_info( + failure: Failure, +) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]: """Extract exc_info from Failure instances""" if isinstance(failure, Failure): - return (failure.type, failure.value, failure.getTracebackObject()) + assert failure.type + assert failure.value + return ( + failure.type, + failure.value, + cast(Optional[TracebackType], failure.getTracebackObject()), + ) + return None class TopLevelFormatter(logging.Filter): @@ -33,10 +48,10 @@ class TopLevelFormatter(logging.Filter): ``loggers`` list where it should act. """ - def __init__(self, loggers=None): - self.loggers = loggers or [] + def __init__(self, loggers: Optional[List[str]] = None): + self.loggers: List[str] = loggers or [] - def filter(self, record): + def filter(self, record: logging.LogRecord) -> bool: if any(record.name.startswith(logger + ".") for logger in self.loggers): record.name = record.name.split(".", 1)[0] return True @@ -62,7 +77,9 @@ DEFAULT_LOGGING = { } -def configure_logging(settings=None, install_root_handler=True): +def configure_logging( + settings: Union[Settings, dict, None] = None, install_root_handler: bool = True +) -> None: """ Initialize logging defaults for Scrapy. @@ -99,13 +116,13 @@ def configure_logging(settings=None, install_root_handler=True): settings = Settings(settings) if settings.getbool("LOG_STDOUT"): - sys.stdout = StreamLogger(logging.getLogger("stdout")) + sys.stdout = StreamLogger(logging.getLogger("stdout")) # type: ignore[assignment] if install_root_handler: install_scrapy_root_handler(settings) -def install_scrapy_root_handler(settings): +def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler if ( @@ -118,16 +135,17 @@ def install_scrapy_root_handler(settings): logging.root.addHandler(_scrapy_root_handler) -def get_scrapy_root_handler(): +def get_scrapy_root_handler() -> Optional[logging.Handler]: return _scrapy_root_handler -_scrapy_root_handler = None +_scrapy_root_handler: Optional[logging.Handler] = None -def _get_handler(settings): +def _get_handler(settings: Settings) -> logging.Handler: """Return a log handler object according to settings""" filename = settings.get("LOG_FILE") + handler: logging.Handler if filename: mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w" encoding = settings.get("LOG_ENCODING") @@ -181,16 +199,16 @@ class StreamLogger: https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ - def __init__(self, logger, log_level=logging.INFO): - self.logger = logger - self.log_level = log_level - self.linebuf = "" + def __init__(self, logger: logging.Logger, log_level: int = logging.INFO): + self.logger: logging.Logger = logger + self.log_level: int = log_level + self.linebuf: str = "" - def write(self, buf): + def write(self, buf: str) -> None: for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) - def flush(self): + def flush(self) -> None: for h in self.logger.handlers: h.flush() @@ -198,11 +216,11 @@ class StreamLogger: class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" - def __init__(self, crawler, *args, **kwargs): + def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.crawler = crawler + self.crawler: Crawler = crawler - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: sname = f"log_count/{record.levelname}" self.crawler.stats.inc_value(sname) From c43798cb9bee99ccf96047f3dfcc6debb65973d4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 22:15:37 +0400 Subject: [PATCH 184/211] More typing for scrapy/utils/defer.py and scrapy/utils/spider.py. --- scrapy/spiderloader.py | 9 +++++---- scrapy/spiders/__init__.py | 2 +- scrapy/utils/defer.py | 24 ++++++++++++++++++++---- scrapy/utils/spider.py | 31 +++++++++++++++++++++++++++---- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index ea5a26e77..13d6f9f87 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,11 +1,12 @@ import traceback import warnings from collections import defaultdict +from types import ModuleType from typing import DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Spider +from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules @@ -45,12 +46,12 @@ class SpiderLoader: category=UserWarning, ) - def _load_spiders(self, module): + def _load_spiders(self, module: ModuleType) -> None: for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): + def _load_all_spiders(self) -> None: for name in self.spider_modules: try: for module in walk_modules(name): @@ -81,7 +82,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request): + def find_by_request(self, request: Request) -> List[str]: """ Return the list of spider names that can handle the given request. """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 3502f8b27..388439f4f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -82,7 +82,7 @@ class Spider(object_ref): settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod - def handles_request(cls, request): + def handles_request(cls, request: Request) -> bool: return url_is_from_spider(request.url, cls) @staticmethod diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 03f026ce9..bf3c5ef5b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,11 +5,13 @@ import asyncio import inspect from asyncio import Future from functools import wraps +from types import CoroutineType from typing import ( Any, AsyncGenerator, AsyncIterable, AsyncIterator, + Awaitable, Callable, Coroutine, Dict, @@ -19,8 +21,10 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, + overload, ) from twisted.internet import defer @@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = cast( - Deferred, deferred_from_coro(self.aiterator.__anext__()) - ) + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -297,7 +299,21 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o: Any) -> Any: +_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +_T = TypeVar("_T") + + +@overload +def deferred_from_coro(o: _CT) -> Deferred: + ... + + +@overload +def deferred_from_coro(o: _T) -> _T: + ... + + +def deferred_from_coro(o: _T) -> Union[Deferred, _T]: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 86449eeb2..3228eda49 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,14 +1,33 @@ +from __future__ import annotations + import inspect import logging +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Generator, + Iterable, + Optional, + Type, + Union, +) +from twisted.internet.defer import Deferred + +from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) -def iterate_spider_output(result): +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -18,7 +37,7 @@ def iterate_spider_output(result): return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module): +def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -37,8 +56,12 @@ def iter_spider_classes(module): def spidercls_for_request( - spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False -): + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Optional[Type[Spider]] = None, + log_none: bool = False, + log_multiple: bool = False, +) -> Optional[Type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using From d1f87e4f088c0757dd833d2a0841be0b830deb57 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 23:11:15 +0400 Subject: [PATCH 185/211] More typing for scrapy/utils/iterators.py. --- scrapy/http/response/text.py | 6 +- scrapy/selector/unified.py | 18 ++++-- scrapy/utils/iterators.py | 106 ++++++++++++++++++++++++++-------- tests/test_utils_iterators.py | 7 ++- 4 files changed, 103 insertions(+), 34 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5289f014a..f228e11c1 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst import json from contextlib import suppress -from typing import Generator, Tuple +from typing import Generator, Optional, Tuple from urllib.parse import urljoin import parsel @@ -37,7 +37,7 @@ class TextResponse(Response): def __init__(self, *args, **kwargs): self._encoding = kwargs.pop("encoding", None) self._cached_benc = None - self._cached_ubody = None + self._cached_ubody: Optional[str] = None self._cached_selector = None super().__init__(*args, **kwargs) @@ -82,7 +82,7 @@ class TextResponse(Response): return self._cached_decoded_json @property - def text(self): + def text(self) -> str: """Body as unicode""" # access self.encoding before _cached_ubody to make sure # _body_inferred_encoding is called diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index caff79e9c..863fb6032 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,10 +1,11 @@ """ XPath selectors based on lxml """ +from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector -from scrapy.http import HtmlResponse, XmlResponse +from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref @@ -13,14 +14,14 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response, st): +def _st(response: Optional[TextResponse], st: Optional[str]) -> str: if st is None: return "xml" if isinstance(response, XmlResponse) else "html" return st -def _response_from_text(text, st): - rt = XmlResponse if st == "xml" else HtmlResponse +def _response_from_text(text: Union[str, bytes], st: Optional[str]) -> TextResponse: + rt: Type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -65,7 +66,14 @@ class Selector(_ParselSelector, object_ref): __slots__ = ["response"] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs): + def __init__( + self, + response: Optional[TextResponse] = None, + text: Optional[str] = None, + type: Optional[str] = None, + root: Optional[Any] = _NOT_SET, + **kwargs: Any, + ): if response is not None and text is not None: raise ValueError( f"{self.__class__.__name__}.__init__() received " diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 170055d5e..58850b843 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,16 +1,37 @@ +from __future__ import annotations + import csv import logging import re from io import StringIO +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + Iterable, + List, + Literal, + Optional, + Union, + cast, + overload, +) from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode +if TYPE_CHECKING: + from lxml._types import SupportsReadClose + logger = logging.getLogger(__name__) -def xmliter(obj, nodename): +def xmliter( + obj: Union[Response, str, bytes], nodename: str +) -> Generator[Selector, Any, None]: """Return a iterator of Selector's over all nodes of a XML document, given the name of the node to iterate. Useful for parsing XML feeds. @@ -27,20 +48,22 @@ def xmliter(obj, nodename): NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) - document_header = re.search(DOCUMENT_HEADER_RE, text) - document_header = document_header.group().strip() if document_header else "" + document_header_match = re.search(DOCUMENT_HEADER_RE, text) + document_header = ( + document_header_match.group().strip() if document_header_match else "" + ) header_end_idx = re_rsearch(HEADER_END_RE, text) header_end = text[header_end_idx[1] :].strip() if header_end_idx else "" - namespaces = {} + namespaces: Dict[str, str] = {} if header_end: for tagname in reversed(re.findall(END_TAG_RE, header_end)): + assert header_end_idx tag = re.search( rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: - namespaces.update( - reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()) - ) + for x in re.findall(NAMESPACE_RE, tag.group()): + namespaces[x[1]] = x[0] r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): @@ -54,12 +77,19 @@ def xmliter(obj, nodename): yield Selector(text=nodetext, type="xml") -def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): +def xmliter_lxml( + obj: Union[TextResponse, str, bytes], + nodename: str, + namespace: Optional[str] = None, + prefix: str = "x", +) -> Generator[Selector, Any, None]: from lxml import etree reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename - iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) + iterable = etree.iterparse( + cast(SupportsReadClose[bytes], reader), tag=tag, encoding=reader.encoding + ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: nodetext = etree.tostring(node, encoding="unicode") @@ -71,30 +101,39 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix="x"): class _StreamReader: - def __init__(self, obj): - self._ptr = 0 - if isinstance(obj, Response): + def __init__(self, obj: Union[TextResponse, str, bytes]): + self._ptr: int = 0 + self._text: Union[str, bytes] + if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, "utf-8" - self._is_unicode = isinstance(self._text, str) + self._is_unicode: bool = isinstance(self._text, str) - def read(self, n=65535): - self.read = self._read_unicode if self._is_unicode else self._read_string + def read(self, n: int = 65535) -> bytes: + self.read: Callable[[int], bytes] = ( # type: ignore[method-assign] + self._read_unicode if self._is_unicode else self._read_string + ) return self.read(n).lstrip() - def _read_string(self, n=65535): + def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e] + return cast(bytes, self._text)[s:e] - def _read_unicode(self, n=65535): + def _read_unicode(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n self._ptr = e - return self._text[s:e].encode("utf-8") + return cast(str, self._text)[s:e].encode("utf-8") -def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): +def csviter( + obj: Union[Response, str, bytes], + delimiter: Optional[str] = None, + headers: Optional[List[str]] = None, + encoding: Optional[str] = None, + quotechar: Optional[str] = None, +) -> Generator[Dict[str, str], Any, None]: """Returns an iterator of dictionaries from the given csv object obj can be: @@ -112,12 +151,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8" - def row_to_unicode(row_): + def row_to_unicode(row_: Iterable) -> List[str]: return [to_unicode(field, encoding) for field in row_] lines = StringIO(_body_or_str(obj, unicode=True)) - kwargs = {} + kwargs: Dict[str, Any] = {} if delimiter: kwargs["delimiter"] = delimiter if quotechar: @@ -147,7 +186,24 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): yield dict(zip(headers, row)) -def _body_or_str(obj, unicode=True): +@overload +def _body_or_str(obj: Union[Response, str, bytes]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: + ... + + +@overload +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: + ... + + +def _body_or_str( + obj: Union[Response, str, bytes], unicode: bool = True +) -> Union[str, bytes]: expected_types = (Response, str, bytes) if not isinstance(obj, expected_types): expected_types_str = " or ".join(t.__name__ for t in expected_types) @@ -156,10 +212,10 @@ def _body_or_str(obj, unicode=True): ) if isinstance(obj, Response): if not unicode: - return obj.body + return cast(bytes, obj.body) if isinstance(obj, TextResponse): return obj.text - return obj.body.decode("utf-8") + return cast(bytes, obj.body).decode("utf-8") if isinstance(obj, str): return obj if unicode else obj.encode("utf-8") return obj.decode("utf-8") if unicode else obj diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..5dfd7e7ac 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,13 +1,18 @@ +from typing import Callable, Iterable, Union + from pytest import mark from twisted.trial import unittest +from scrapy import Selector from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) + xmliter: Callable[ + [Union[TextResponse, str, bytes], str], Iterable[Selector] + ] = staticmethod(xmliter) def test_xmliter(self): body = b""" From 9fe662d856a6b2496379585143aa2f4d023039f8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 23:42:59 +0400 Subject: [PATCH 186/211] Add typing for scrapy/utils/testproc.py. --- scrapy/utils/testproc.py | 45 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 5f9bdef37..5f7a7db14 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,7 +1,13 @@ +from __future__ import annotations + import os import sys +from typing import Iterable, Optional, Tuple, cast -from twisted.internet import defer, protocol +from twisted.internet.defer import Deferred +from twisted.internet.error import ProcessTerminated +from twisted.internet.protocol import ProcessProtocol +from twisted.python.failure import Failure class ProcessTest: @@ -9,7 +15,12 @@ class ProcessTest: prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir - def execute(self, args, check_code=True, settings=None): + def execute( + self, + args: Iterable[str], + check_code: bool = True, + settings: Optional[str] = None, + ) -> Deferred: from twisted.internet import reactor env = os.environ.copy() @@ -21,29 +32,31 @@ class ProcessTest: reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd) return pp.deferred - def _process_finished(self, pp, cmd, check_code): + def _process_finished( + self, pp: TestProcessProtocol, cmd: str, check_code: bool + ) -> Tuple[int, bytes, bytes]: if pp.exitcode and check_code: msg = f"process {cmd} exit with code {pp.exitcode}" - msg += f"\n>>> stdout <<<\n{pp.out}" + msg += f"\n>>> stdout <<<\n{pp.out.decode()}" msg += "\n" - msg += f"\n>>> stderr <<<\n{pp.err}" + msg += f"\n>>> stderr <<<\n{pp.err.decode()}" raise RuntimeError(msg) - return pp.exitcode, pp.out, pp.err + return cast(int, pp.exitcode), pp.out, pp.err -class TestProcessProtocol(protocol.ProcessProtocol): - def __init__(self): - self.deferred = defer.Deferred() - self.out = b"" - self.err = b"" - self.exitcode = None +class TestProcessProtocol(ProcessProtocol): + def __init__(self) -> None: + self.deferred: Deferred = Deferred() + self.out: bytes = b"" + self.err: bytes = b"" + self.exitcode: Optional[int] = None - def outReceived(self, data): + def outReceived(self, data: bytes) -> None: self.out += data - def errReceived(self, data): + def errReceived(self, data: bytes) -> None: self.err += data - def processEnded(self, status): - self.exitcode = status.value.exitCode + def processEnded(self, status: Failure) -> None: + self.exitcode = cast(ProcessTerminated, status.value).exitCode self.deferred.callback(self) From 66bad1150cf0a72b39d1af8a63d7b4eb7c1f42fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 3 Aug 2023 00:08:28 +0400 Subject: [PATCH 187/211] Add more typing for scrapy/utils/signal.py. --- scrapy/utils/signal.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 9e7ddd827..21a12a19e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -23,7 +23,10 @@ logger = logging.getLogger(__name__) def send_catch_log( - signal=Any, sender=Anonymous, *arguments, **named + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -65,13 +68,18 @@ def send_catch_log( return responses -def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): +def send_catch_log_deferred( + signal: TypingAny = Any, + sender: TypingAny = Anonymous, + *arguments: TypingAny, + **named: TypingAny +) -> Deferred: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. """ - def logerror(failure, recv): + def logerror(failure: Failure, recv: Any) -> Failure: if dont_log is None or not isinstance(failure.value, dont_log): logger.error( "Error caught on signal handler: %(receiver)s", @@ -96,7 +104,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): return d -def disconnect_all(signal=Any, sender=Any): +def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: """Disconnect all signal handlers. Useful for cleaning up after running tests """ From 518e56046e45d38c505db9e2bc00677ad08ac58c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 17:28:34 +0400 Subject: [PATCH 188/211] Check for async callbacks in contracts. --- scrapy/contracts/__init__.py | 13 ++++++++++--- tests/test_contracts.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 86098edca..1ec2a0234 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -2,7 +2,8 @@ import re import sys from functools import wraps from inspect import getmembers -from typing import Dict +from types import CoroutineType +from typing import AsyncGenerator, Dict from unittest import TestCase from scrapy.http import Request @@ -37,7 +38,10 @@ class Contract: else: results.addSuccess(self.testcase_pre) finally: - return list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + return list(iterate_spider_output(cb_result)) request.callback = wrapper @@ -49,7 +53,10 @@ class Contract: @wraps(cb) def wrapper(response, **cb_kwargs): - output = list(iterate_spider_output(cb(response, **cb_kwargs))) + cb_result = cb(response, **cb_kwargs) + if isinstance(cb_result, (AsyncGenerator, CoroutineType)): + raise TypeError("Contracts don't support async callbacks") + output = list(iterate_spider_output(cb_result)) try: results.startTest(self.testcase_post) self.post_process(output) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 813927fc5..1459e0b5f 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -63,6 +63,13 @@ class TestSpider(Spider): """ return Request("http://scrapy.org", callback=self.returns_item) + async def returns_request_async(self, response): + """async method which returns request + @url http://scrapy.org + @returns requests 1 + """ + return Request("http://scrapy.org", callback=self.returns_item) + def returns_item(self, response): """method which returns item @url http://scrapy.org @@ -337,6 +344,14 @@ class ContractsManagerTest(unittest.TestCase): request.callback(response) self.should_fail() + def test_returns_async(self): + spider = TestSpider() + response = ResponseMock() + + request = self.conman.from_method(spider.returns_request_async, self.results) + request.callback(response) + self.should_error() + def test_scrapes(self): spider = TestSpider() response = ResponseMock() From e2adec629b63e9d7735efd58b4353dcfe7ab2863 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 17:31:11 +0400 Subject: [PATCH 189/211] Fix regressions in typing. --- scrapy/commands/__init__.py | 4 +-- scrapy/commands/fetch.py | 8 +++-- scrapy/commands/shell.py | 8 +++-- scrapy/http/response/text.py | 12 +++++--- scrapy/spiderloader.py | 2 +- scrapy/utils/ossignal.py | 14 +++++++-- scrapy/utils/spider.py | 60 ++++++++++++++++++++++++++++++++++-- 7 files changed, 91 insertions(+), 17 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9baee3a48..2aa569cdd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -4,7 +4,7 @@ Base class for Scrapy commands import argparse import os from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from twisted.python import failure @@ -116,7 +116,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: """ Entry point for running commands """ diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1359e445f..cdb7ad4ae 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,7 +1,10 @@ import sys +from argparse import Namespace +from typing import List, Type from w3lib.url import is_url +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.http import Request @@ -57,7 +60,7 @@ class Command(ScrapyCommand): def _print_bytes(self, bytes_): sys.stdout.buffer.write(bytes_ + b"\n") - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: if len(args) != 1 or not is_url(args[0]): raise UsageError() request = Request( @@ -73,7 +76,8 @@ class Command(ScrapyCommand): else: request.meta["handle_httpstatus_all"] = True - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: spidercls = spider_loader.load(opts.spider) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 63c23d04c..0a5e61f7a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,8 +3,11 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ +from argparse import Namespace from threading import Thread +from typing import List, Type +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -54,15 +57,16 @@ class Command(ScrapyCommand): """ pass - def run(self, args, opts): + def run(self, args: List[str], opts: Namespace) -> None: url = args[0] if args else None if url: # first argument may be a local file url = guess_scheme(url) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader - spidercls = DefaultSpider + spidercls: Type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) elif url: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index f228e11c1..7fc54b5d3 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,10 +4,11 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations import json from contextlib import suppress -from typing import Generator, Optional, Tuple +from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple from urllib.parse import urljoin import parsel @@ -25,6 +26,9 @@ from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url +if TYPE_CHECKING: + from scrapy.selector import Selector + _NONE = object() @@ -34,11 +38,11 @@ class TextResponse(Response): attributes: Tuple[str, ...] = Response.attributes + ("encoding",) - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): self._encoding = kwargs.pop("encoding", None) - self._cached_benc = None + self._cached_benc: Optional[str] = None self._cached_ubody: Optional[str] = None - self._cached_selector = None + self._cached_selector: Optional[Selector] = None super().__init__(*args, **kwargs) def _set_url(self, url): diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 13d6f9f87..f6bb93ddc 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader: self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() - def _check_name_duplicates(self): + def _check_name_duplicates(self) -> None: dupes = [] for name, locations in self._found.items(): dupes.extend( diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index f835a2221..2334ea792 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,7 +1,13 @@ import signal -from typing import Callable +from types import FrameType +from typing import Any, Callable, Dict, Optional, Union -signal_names = {} +# copy of _HANDLER from typeshed/stdlib/signal.pyi +SignalHandlerT = Union[ + Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None +] + +signal_names: Dict[int, str] = {} for signame in dir(signal): if signame.startswith("SIG") and not signame.startswith("SIG_"): signum = getattr(signal, signame) @@ -9,7 +15,9 @@ for signame in dir(signal): signal_names[signum] = signame -def install_shutdown_handlers(function: Callable, override_sigint: bool = True) -> None: +def install_shutdown_handlers( + function: SignalHandlerT, override_sigint: bool = True +) -> None: """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the SIGINT handler won't be install if there is already a handler in place diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 3228eda49..704df8657 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,16 +2,19 @@ from __future__ import annotations import inspect import logging -from types import ModuleType +from types import CoroutineType, ModuleType from typing import ( TYPE_CHECKING, Any, - AsyncIterable, + AsyncGenerator, Generator, Iterable, + Literal, Optional, Type, + TypeVar, Union, + overload, ) from twisted.internet.defer import Deferred @@ -26,8 +29,26 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") -def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]: + +# https://stackoverflow.com/questions/60222982 +@overload +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] + ... + + +@overload +def iterate_spider_output(result: CoroutineType) -> Deferred: + ... + + +@overload +def iterate_spider_output(result: _T) -> Iterable: + ... + + +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -55,6 +76,39 @@ def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None yield obj +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Type[Spider], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Type[Spider]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Literal[None], + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + +@overload +def spidercls_for_request( + spider_loader: SpiderLoader, + request: Request, + *, + log_none: bool = ..., + log_multiple: bool = ..., +) -> Optional[Type[Spider]]: + ... + + def spidercls_for_request( spider_loader: SpiderLoader, request: Request, From f5f593e5f5e2b0c216e9d6fd41f4260f70c74d34 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 17:46:28 +0400 Subject: [PATCH 190/211] Remove a workaround for a w3lib typing bug. --- scrapy/utils/response.py | 3 +-- tox.ini | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 794678c48..c540d6278 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -42,8 +42,7 @@ def get_meta_refresh( """Parse the http-equiv refresh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - # a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211 - _metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment] + _metaref_cache[response] = html.get_meta_refresh( text, response.url, response.encoding, ignore_tags=ignore_tags ) return _metaref_cache[response] diff --git a/tox.ini b/tox.ini index ef7dd5854..8b2d207c7 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,8 @@ deps = types-Pygments==2.15.0.1 types-pyOpenSSL==23.2.0.1 types-setuptools==68.0.0.1 + # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 + w3lib >= 2.1.2 commands = mypy {posargs: scrapy tests} From 471281d29e5c7b8e293f59ac3c3330ecda687d19 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 23:05:02 +0400 Subject: [PATCH 191/211] Fixes for scrapy/utils/iterators.py typing. --- scrapy/utils/iterators.py | 11 ++++++----- tests/test_utils_iterators.py | 7 +------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 58850b843..40af68dec 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import csv import logging import re @@ -78,7 +76,7 @@ def xmliter( def xmliter_lxml( - obj: Union[TextResponse, str, bytes], + obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", @@ -87,8 +85,9 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename + # technically, etree.iterparse only needs .read() AFAICS, but this is how it's typed iterable = etree.iterparse( - cast(SupportsReadClose[bytes], reader), tag=tag, encoding=reader.encoding + cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: @@ -101,11 +100,13 @@ def xmliter_lxml( class _StreamReader: - def __init__(self, obj: Union[TextResponse, str, bytes]): + def __init__(self, obj: Union[Response, str, bytes]): self._ptr: int = 0 self._text: Union[str, bytes] if isinstance(obj, TextResponse): self._text, self.encoding = obj.body, obj.encoding + elif isinstance(obj, Response): + self._text, self.encoding = obj.body, "utf-8" else: self._text, self.encoding = obj, "utf-8" self._is_unicode: bool = isinstance(self._text, str) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 5dfd7e7ac..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,18 +1,13 @@ -from typing import Callable, Iterable, Union - from pytest import mark from twisted.trial import unittest -from scrapy import Selector from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata class XmliterTestCase(unittest.TestCase): - xmliter: Callable[ - [Union[TextResponse, str, bytes], str], Iterable[Selector] - ] = staticmethod(xmliter) + xmliter = staticmethod(xmliter) def test_xmliter(self): body = b""" From 644a71bfd40be3ce8a465ad49891d34c47516f56 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:17:52 +0400 Subject: [PATCH 192/211] Use ftp:// URLs in FTP tests. --- tests/test_feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 46bd5733a..42fa25b1d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2978,8 +2978,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_init(self): settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptions}, + "FEED_URI": "ftp://localhost/foo", + "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptions}, } with pytest.warns( ScrapyDeprecationWarning, @@ -3000,8 +3000,8 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): def test_from_crawler(self): settings_dict = { - "FEED_URI": "file:///tmp/foobar", - "FEED_STORAGES": {"file": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, + "FEED_URI": "ftp://localhost/foo", + "FEED_STORAGES": {"ftp": FTPFeedStorageWithoutFeedOptionsWithFromCrawler}, } with pytest.warns( ScrapyDeprecationWarning, From 23af21491d526072873ab7ec3d1429560861964f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:21:06 +0400 Subject: [PATCH 193/211] Move definitions around to woark around a pypy3.8 bug. --- scrapy/utils/log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 2013bfc43..0d17f6153 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -122,6 +122,9 @@ def configure_logging( install_scrapy_root_handler(settings) +_scrapy_root_handler: Optional[logging.Handler] = None + + def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler @@ -139,9 +142,6 @@ def get_scrapy_root_handler() -> Optional[logging.Handler]: return _scrapy_root_handler -_scrapy_root_handler: Optional[logging.Handler] = None - - def _get_handler(settings: Settings) -> logging.Handler: """Return a log handler object according to settings""" filename = settings.get("LOG_FILE") From 110d5fffb4eb034cb876f5339eb2c95b89d8630b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 12:57:48 +0400 Subject: [PATCH 194/211] Update tool versions. (#6002) --- .pre-commit-config.yaml | 8 ++++---- tox.ini | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 31e9ed1ad..5998ebef8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,11 +5,11 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -17,8 +17,8 @@ repos: hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs - rev: 1.13.0 + rev: 1.15.0 hooks: - id: blacken-docs additional_dependencies: - - black==23.3.0 + - black==23.7.0 diff --git a/tox.ini b/tox.ini index ef7dd5854..b22ef404d 100644 --- a/tox.ini +++ b/tox.ini @@ -37,10 +37,10 @@ deps = typing-extensions==4.7.1 types-attrs==19.1.0 types-lxml==2023.3.28 - types-Pillow==10.0.0.1 - types-Pygments==2.15.0.1 - types-pyOpenSSL==23.2.0.1 - types-setuptools==68.0.0.1 + types-Pillow==10.0.0.2 + types-Pygments==2.15.0.2 + types-pyOpenSSL==23.2.0.2 + types-setuptools==68.0.0.3 commands = mypy {posargs: scrapy tests} @@ -55,7 +55,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.17.2 + pylint==2.17.5 commands = pylint conftest.py docs extras scrapy setup.py tests From 53539483c32dec537064628c0c6e407eb8ce1c2f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 15:13:20 +0400 Subject: [PATCH 195/211] Refactor _StreamReader.read(). --- scrapy/utils/iterators.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 40af68dec..baf92681a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -110,12 +110,17 @@ class _StreamReader: else: self._text, self.encoding = obj, "utf-8" self._is_unicode: bool = isinstance(self._text, str) + self._is_first_read: bool = True def read(self, n: int = 65535) -> bytes: - self.read: Callable[[int], bytes] = ( # type: ignore[method-assign] + method: Callable[[int], bytes] = ( self._read_unicode if self._is_unicode else self._read_string ) - return self.read(n).lstrip() + result = method(n) + if self._is_first_read: + self._is_first_read = False + result = result.lstrip() + return result def _read_string(self, n: int = 65535) -> bytes: s, e = self._ptr, self._ptr + n From 8050257c1495fe405767dcd37c4db0f6f29c38aa Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 9 Aug 2023 23:17:32 +0400 Subject: [PATCH 196/211] Small cleanup. --- scrapy/extensions/feedexport.py | 3 ++- scrapy/utils/iterators.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2bbcaf3ad..c1b77f4fb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -290,7 +290,8 @@ class FTPFeedStorage(BlockingFeedStorage): feed_options: Optional[Dict[str, Any]] = None, ): u = urlparse(uri) - assert u.hostname + if not u.hostname: + raise ValueError(f"Got a storage URI without a hostname: {uri}") self.host: str = u.hostname self.port: int = int(u.port or "21") self.username: str = u.username or "" diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index baf92681a..03d779afb 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -85,7 +85,6 @@ def xmliter_lxml( reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename - # technically, etree.iterparse only needs .read() AFAICS, but this is how it's typed iterable = etree.iterparse( cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding ) From 084a9ba0768c34e2c0b82e13c7182da33460a19b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 6 Aug 2023 23:51:24 +0400 Subject: [PATCH 197/211] Full typing for scrapy/crawler.py and scrapy/spiders/__init__.py. --- scrapy/crawler.py | 77 +++++++++++++++++++++++--------------- scrapy/logformatter.py | 13 ++++++- scrapy/middleware.py | 29 ++++++++++++-- scrapy/spiderloader.py | 10 ++++- scrapy/spiders/__init__.py | 37 ++++++++++-------- 5 files changed, 114 insertions(+), 52 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c5b3e1903..531798893 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,9 +4,14 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union -from twisted.internet import defer +from twisted.internet.defer import ( + Deferred, + DeferredList, + inlineCallbacks, + maybeDeferred, +) from zope.interface.exceptions import DoesNotImplement try: @@ -24,7 +29,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter -from scrapy.settings import Settings, overridden_settings +from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector @@ -123,8 +128,8 @@ class Crawler: self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None - @defer.inlineCallbacks - def crawl(self, *args, **kwargs): + @inlineCallbacks + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") self.crawling = True @@ -134,26 +139,27 @@ class Crawler: self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) - yield defer.maybeDeferred(self.engine.start) + yield maybeDeferred(self.engine.start) except Exception: self.crawling = False if self.engine is not None: yield self.engine.close() raise - def _create_spider(self, *args, **kwargs): + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) - def _create_engine(self): + def _create_engine(self) -> ExecutionEngine: return ExecutionEngine(self, lambda _: self.stop()) - @defer.inlineCallbacks - def stop(self): + @inlineCallbacks + def stop(self) -> Generator[Deferred, Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False - yield defer.maybeDeferred(self.engine.stop) + assert self.engine + yield maybeDeferred(self.engine.stop) class CrawlerRunner: @@ -176,10 +182,10 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings) -> SpiderLoader: + def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls = load_object(cls_path) + loader_cls: Type[SpiderLoader] = load_object(cls_path) excs = ( (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement ) @@ -201,11 +207,11 @@ class CrawlerRunner: self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() - self._active: Set[defer.Deferred] = set() + self._active: Set[Deferred] = set() self.bootstrap_failed = False @property - def spiders(self): + def spiders(self) -> SpiderLoader: warnings.warn( "CrawlerRunner.spiders attribute is renamed to " "CrawlerRunner.spider_loader.", @@ -214,7 +220,12 @@ class CrawlerRunner: ) return self.spider_loader - def crawl(self, crawler_or_spidercls, *args, **kwargs): + def crawl( + self, + crawler_or_spidercls: Union[Type[Spider], str, Crawler], + *args: Any, + **kwargs: Any, + ) -> Deferred: """ Run a crawler with the provided arguments. @@ -244,12 +255,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler, *args, **kwargs): + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result): + def _done(result: Any) -> Any: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -284,16 +295,16 @@ class CrawlerRunner: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) - def stop(self): + def stop(self) -> Deferred: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return defer.DeferredList([c.stop() for c in list(self.crawlers)]) + return DeferredList([c.stop() for c in list(self.crawlers)]) - @defer.inlineCallbacks - def join(self): + @inlineCallbacks + def join(self) -> Generator[Deferred, Any, None]: """ join() @@ -301,7 +312,7 @@ class CrawlerRunner: completed their executions. """ while self._active: - yield defer.DeferredList(self._active) + yield DeferredList(self._active) class CrawlerProcess(CrawlerRunner): @@ -328,13 +339,17 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True): + def __init__( + self, + settings: Union[Dict[str, Any], Settings, None] = None, + install_root_handler: bool = True, + ): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False - def _signal_shutdown(self, signum, _): + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(self._signal_kill) @@ -345,7 +360,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._graceful_stop_reactor) - def _signal_kill(self, signum, _): + def _signal_kill(self, signum: int, _: Any) -> None: from twisted.internet import reactor install_shutdown_handlers(signal.SIG_IGN) @@ -355,14 +370,16 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) - def _create_crawler(self, spidercls): + def _create_crawler(self, spidercls: Union[Type[Spider], str]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) - def start(self, stop_after_crawl=True, install_signal_handlers=True): + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -396,12 +413,12 @@ class CrawlerProcess(CrawlerRunner): reactor.addSystemEventTrigger("before", "shutdown", self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _graceful_stop_reactor(self): + def _graceful_stop_reactor(self) -> Deferred: d = self.stop() d.addBoth(self._stop_reactor) return d - def _stop_reactor(self, _=None): + def _stop_reactor(self, _: Any = None) -> None: from twisted.internet import reactor try: diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 7cb379b46..600da0d40 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging import os -from typing import Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Union from twisted.python.failure import Failure @@ -8,6 +10,13 @@ from scrapy import Request, Spider from scrapy.http import Response from scrapy.utils.request import referer_str +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" @@ -161,5 +170,5 @@ class LogFormatter: } @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls() diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 04b838d2d..090588130 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,21 @@ +from __future__ import annotations + import logging import pprint from collections import defaultdict, deque -from typing import Any, Callable, Deque, Dict, Iterable, List, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) from twisted.internet.defer import Deferred @@ -11,6 +25,13 @@ from scrapy.settings import Settings from scrapy.utils.defer import process_chain, process_parallel from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + logger = logging.getLogger(__name__) @@ -34,7 +55,9 @@ class MiddlewareManager: raise NotImplementedError @classmethod - def from_settings(cls, settings: Settings, crawler=None): + def from_settings( + cls, settings: Settings, crawler: Optional[Crawler] = None + ) -> Self: mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] enabled = [] @@ -63,7 +86,7 @@ class MiddlewareManager: return cls(*middlewares) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls.from_settings(crawler.settings, crawler) def _add_middleware(self, mw: Any) -> None: diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index f6bb93ddc..cd60fce9d 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import traceback import warnings from collections import defaultdict from types import ModuleType -from typing import DefaultDict, Dict, List, Tuple, Type +from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer @@ -12,6 +14,10 @@ from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + @implementer(ISpiderLoader) class SpiderLoader: @@ -69,7 +75,7 @@ class SpiderLoader: self._check_name_duplicates() @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls(settings) def load(self, spider_name: str) -> Type[Spider]: diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 388439f4f..e16d71727 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -6,15 +6,21 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast + +from twisted.internet.defer import Deferred from scrapy import signals -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings class Spider(object_ref): @@ -25,21 +31,21 @@ class Spider(object_ref): name: str custom_settings: Optional[dict] = None - def __init__(self, name=None, **kwargs): + def __init__(self, name: Optional[str] = None, **kwargs: Any): if name is not None: self.name = name elif not getattr(self, "name", None): raise ValueError(f"{type(self).__name__} must have a name") self.__dict__.update(kwargs) if not hasattr(self, "start_urls"): - self.start_urls = [] + self.start_urls: List[str] = [] @property - def logger(self): + def logger(self) -> logging.LoggerAdapter: logger = logging.getLogger(self.name) return logging.LoggerAdapter(logger, {"spider": self}) - def log(self, message, level=logging.DEBUG, **kw): + def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None: """Log the given message at the given log level This helper wraps a log call to the logger within the spider, but you @@ -49,17 +55,17 @@ class Spider(object_ref): self.logger.log(level, message, **kw) @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = cls(*args, **kwargs) spider._set_crawler(crawler) return spider - def _set_crawler(self, crawler: Crawler): + def _set_crawler(self, crawler: Crawler) -> None: self.crawler = crawler self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - def start_requests(self): + def start_requests(self) -> Iterable[Request]: if not self.start_urls and hasattr(self, "start_url"): raise AttributeError( "Crawling could not start: 'start_urls' not found " @@ -69,16 +75,16 @@ class Spider(object_ref): for url in self.start_urls: yield Request(url, dont_filter=True) - def _parse(self, response, **kwargs): + def _parse(self, response: Response, **kwargs: Any) -> Any: return self.parse(response, **kwargs) - def parse(self, response, **kwargs): + def parse(self, response: Response, **kwargs: Any) -> Any: raise NotImplementedError( f"{self.__class__.__name__}.parse callback is not defined" ) @classmethod - def update_settings(cls, settings): + def update_settings(cls, settings: BaseSettings) -> None: settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod @@ -86,12 +92,13 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider, reason): + def close(spider: Spider, reason: str) -> Union[Deferred, None]: closed = getattr(spider, "closed", None) if callable(closed): - return closed(reason) + return cast(Union[Deferred, None], closed(reason)) + return None - def __repr__(self): + def __repr__(self) -> str: return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" From dc6e142096e77a99e5340f36309f40e70f4145cd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:35:44 +0400 Subject: [PATCH 198/211] Full typing for scrapy/spiderloader.py. --- scrapy/spiderloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index cd60fce9d..d855c962c 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,8 +27,8 @@ class SpiderLoader: """ def __init__(self, settings: BaseSettings): - self.spider_modules = settings.getlist("SPIDER_MODULES") - self.warn_only = settings.getbool("SPIDER_LOADER_WARN_ONLY") + self.spider_modules: List[str] = settings.getlist("SPIDER_MODULES") + self.warn_only: bool = settings.getbool("SPIDER_LOADER_WARN_ONLY") self._spiders: Dict[str, Type[Spider]] = {} self._found: DefaultDict[str, List[Tuple[str, str]]] = defaultdict(list) self._load_all_spiders() @@ -96,7 +96,7 @@ class SpiderLoader: name for name, cls in self._spiders.items() if cls.handles_request(request) ] - def list(self): + def list(self) -> List[str]: """ Return a list with the names of all spiders available in the project. """ From 89503ae3f1228e9406428a19bc6c49c1f8e9e19b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:41:11 +0400 Subject: [PATCH 199/211] Full typing for scrapy/dupefilters.py. --- scrapy/dupefilters.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d796e5cbb..bc912268c 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import logging from pathlib import Path -from typing import Optional, Set, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Set from warnings import warn from twisted.internet.defer import Deferred @@ -12,14 +14,16 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir from scrapy.utils.request import RequestFingerprinter, referer_str -BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler class BaseDupeFilter: @classmethod - def from_settings( - cls: Type[BaseDupeFilterTV], settings: BaseSettings - ) -> BaseDupeFilterTV: + def from_settings(cls, settings: BaseSettings) -> Self: return cls() def request_seen(self, request: Request) -> bool: @@ -36,9 +40,6 @@ class BaseDupeFilter: pass -RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter") - - class RFPDupeFilter(BaseDupeFilter): """Request Fingerprint duplicates filter""" @@ -47,10 +48,12 @@ class RFPDupeFilter(BaseDupeFilter): path: Optional[str] = None, debug: bool = False, *, - fingerprinter=None, + fingerprinter: Optional[RequestFingerprinter] = None, ) -> None: self.file = None - self.fingerprinter = fingerprinter or RequestFingerprinter() + self.fingerprinter: RequestFingerprinter = ( + fingerprinter or RequestFingerprinter() + ) self.fingerprints: Set[str] = set() self.logdupes = True self.debug = debug @@ -62,8 +65,11 @@ class RFPDupeFilter(BaseDupeFilter): @classmethod def from_settings( - cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None - ) -> RFPDupeFilterTV: + cls, + settings: BaseSettings, + *, + fingerprinter: Optional[RequestFingerprinter] = None, + ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") try: return cls(job_dir(settings), debug, fingerprinter=fingerprinter) @@ -75,11 +81,11 @@ class RFPDupeFilter(BaseDupeFilter): ScrapyDeprecationWarning, ) result = cls(job_dir(settings), debug) - result.fingerprinter = fingerprinter + result.fingerprinter = fingerprinter or RequestFingerprinter() return result @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: try: return cls.from_settings( crawler.settings, From 9960c62b871a4b76d72c57abdead553b6e7178e1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:42:06 +0400 Subject: [PATCH 200/211] Full typing for scrapy/logformatter.py. --- scrapy/core/scraper.py | 1 + scrapy/logformatter.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index a85f6a661..a54929712 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -364,6 +364,7 @@ class Scraper: spider=spider, exception=output.value, ) + assert ex logkws = self.logformatter.item_error(item, ex, response, spider) logger.log( *logformatter_adapter(logkws), diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 600da0d40..9b05e1153 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -114,7 +114,7 @@ class LogFormatter: } def item_error( - self, item: Any, exception, response: Response, spider: Spider + self, item: Any, exception: BaseException, response: Response, spider: Spider ) -> dict: """Logs a message when an item causes an error while it is passing through the item pipeline. From c5885fc13b7d44ddaf403e8ccdd8dccb14082eee Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:44:46 +0400 Subject: [PATCH 201/211] Full typing for scrapy/exceptions.py. --- scrapy/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6e83e4a00..6d188c489 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -52,7 +52,7 @@ class StopDownload(Exception): should be handled by the request errback. Note that 'fail' is a keyword-only argument. """ - def __init__(self, *, fail=True): + def __init__(self, *, fail: bool = True): super().__init__() self.fail = fail From 7dca18e2e7e92e7dcc0fb8ae769f82287dd64386 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:45:16 +0400 Subject: [PATCH 202/211] Full typing for scrapy/responsetypes.py. --- scrapy/responsetypes.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 58884f21a..9e411d4aa 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -5,6 +5,7 @@ based on different criteria. from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data +from typing import Dict, Mapping, Optional, Type, Union from scrapy.http import Response from scrapy.utils.misc import load_object @@ -29,15 +30,19 @@ class ResponseTypes: "text/*": "scrapy.http.TextResponse", } - def __init__(self): - self.classes = {} - self.mimetypes = MimeTypes() - mimedata = get_data("scrapy", "mime.types").decode("utf8") - self.mimetypes.readfp(StringIO(mimedata)) + def __init__(self) -> None: + self.classes: Dict[str, Type[Response]] = {} + self.mimetypes: MimeTypes = MimeTypes() + mimedata = get_data("scrapy", "mime.types") + if not mimedata: + raise ValueError( + "The mime.types file is not found in the Scrapy installation" + ) + self.mimetypes.readfp(StringIO(mimedata.decode("utf8"))) for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) - def from_mimetype(self, mimetype): + def from_mimetype(self, mimetype: str) -> Type[Response]: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response @@ -46,7 +51,9 @@ class ResponseTypes: basetype = f"{mimetype.split('/')[0]}/*" return self.classes.get(basetype, Response) - def from_content_type(self, content_type, content_encoding=None): + def from_content_type( + self, content_type: Union[str, bytes], content_encoding: Optional[bytes] = None + ) -> Type[Response]: """Return the most appropriate Response class from an HTTP Content-Type header""" if content_encoding: @@ -56,7 +63,9 @@ class ResponseTypes: ) return self.from_mimetype(mimetype) - def from_content_disposition(self, content_disposition): + def from_content_disposition( + self, content_disposition: Union[str, bytes] + ) -> Type[Response]: try: filename = ( to_unicode(content_disposition, encoding="latin-1", errors="replace") @@ -68,7 +77,7 @@ class ResponseTypes: except IndexError: return Response - def from_headers(self, headers): + def from_headers(self, headers: Mapping[bytes, bytes]) -> Type[Response]: """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response @@ -81,14 +90,14 @@ class ResponseTypes: cls = self.from_content_disposition(headers[b"Content-Disposition"]) return cls - def from_filename(self, filename): + def from_filename(self, filename: str) -> Type[Response]: """Return the most appropriate Response class from a file name""" mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) return Response - def from_body(self, body): + def from_body(self, body: bytes) -> Type[Response]: """Try to guess the appropriate response based on the body content. This method is a bit magic and could be improved in the future, but it's not meant to be used except for special cases where response types @@ -106,7 +115,13 @@ class ResponseTypes: return self.from_mimetype("text/html") return self.from_mimetype("text") - def from_args(self, headers=None, url=None, filename=None, body=None): + def from_args( + self, + headers: Optional[Mapping[bytes, bytes]] = None, + url: Optional[str] = None, + filename: Optional[str] = None, + body: Optional[bytes] = None, + ) -> Type[Response]: """Guess the most appropriate Response class based on the given arguments.""" cls = Response From e6d919497d08f5c1ddcdd0c210fcc17bd78f2fe9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:52:30 +0400 Subject: [PATCH 203/211] More typing for scrapy/core/scheduler.py. --- scrapy/core/scheduler.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 3fb0bbaff..70b6dc8a1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,8 +1,10 @@ +from __future__ import annotations + import json import logging from abc import abstractmethod from pathlib import Path -from typing import Any, Optional, Type, TypeVar, cast +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast from twisted.internet.defer import Deferred @@ -14,6 +16,11 @@ from scrapy.statscollectors import StatsCollector from scrapy.utils.job import job_dir from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + logger = logging.getLogger(__name__) @@ -54,7 +61,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): """ @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler: Crawler) -> Self: """ Factory method which receives the current :class:`~scrapy.crawler.Crawler` object as argument. """ @@ -325,6 +332,7 @@ class Scheduler(BaseScheduler): def _dq(self): """Create a new priority queue instance, with disk storage""" + assert self.dqdir state = self._read_dqs_state(self.dqdir) q = create_instance( self.pqclass, From b50268c100c6ad70ea81f0606aedcb720a12692b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 7 Aug 2023 00:55:40 +0400 Subject: [PATCH 204/211] Full typing for scrapy/link.py. --- scrapy/link.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scrapy/link.py b/scrapy/link.py index 704649731..0868ae5ef 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ +from typing import Any class Link: @@ -26,16 +27,20 @@ class Link: __slots__ = ["url", "text", "fragment", "nofollow"] - def __init__(self, url, text="", fragment="", nofollow=False): + def __init__( + self, url: str, text: str = "", fragment: str = "", nofollow: bool = False + ): if not isinstance(url, str): got = url.__class__.__name__ raise TypeError(f"Link urls must be str objects, got {got}") - self.url = url - self.text = text - self.fragment = fragment - self.nofollow = nofollow + self.url: str = url + self.text: str = text + self.fragment: str = fragment + self.nofollow: bool = nofollow - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Link): + raise NotImplementedError return ( self.url == other.url and self.text == other.text @@ -43,12 +48,12 @@ class Link: and self.nofollow == other.nofollow ) - def __hash__(self): + def __hash__(self) -> int: return ( hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) ) - def __repr__(self): + def __repr__(self) -> str: return ( f"Link(url={self.url!r}, text={self.text!r}, " f"fragment={self.fragment!r}, nofollow={self.nofollow!r})" From 4a090d951a721881d0c434ebf4207e78a4eadfe1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Aug 2023 02:36:42 -0300 Subject: [PATCH 205/211] Remove deprecated PythonItemExporter.binary (#6007) --- scrapy/exporters.py | 14 +------------- tests/test_exporters.py | 12 +----------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 8254ea63e..f85f1dad8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -7,13 +7,11 @@ import io import marshal import pickle import pprint -import warnings from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import ItemAdapter, is_item -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -330,13 +328,7 @@ class PythonItemExporter(BaseItemExporter): """ def _configure(self, options, dont_fail=False): - self.binary = options.pop("binary", True) super()._configure(options, dont_fail) - if self.binary: - warnings.warn( - "PythonItemExporter will drop support for binary export in the future", - ScrapyDeprecationWarning, - ) if not self.encoding: self.encoding = "utf-8" @@ -351,18 +343,14 @@ class PythonItemExporter(BaseItemExporter): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): - return encode_func(value, encoding=self.encoding) + return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item): for key, value in ItemAdapter(item).items(): - key = to_bytes(key) if self.binary else key yield key, self._serialize_value(value) def export_item(self, item): result = dict(self._get_serialized_fields(item)) - if self.binary: - result = dict(self._serialize_item(result)) return result diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cb24ddd8e..f4e82705a 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -7,12 +7,10 @@ import tempfile import unittest from datetime import datetime from io import BytesIO -from warnings import catch_warnings, filterwarnings import lxml.etree from itemadapter import ItemAdapter -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exporters import ( BaseItemExporter, CsvItemExporter, @@ -143,7 +141,7 @@ class BaseItemExporterDataclassTest(BaseItemExporterTest): class PythonItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): - return PythonItemExporter(binary=False, **kwargs) + return PythonItemExporter(**kwargs) def test_invalid_option(self): with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): @@ -198,14 +196,6 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported["age"][0]), dict) self.assertEqual(type(exported["age"][0]["age"][0]), dict) - def test_export_binary(self): - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - exporter = PythonItemExporter(binary=True) - value = self.item_class(name="John\xa3", age="22") - expected = {b"name": b"John\xc2\xa3", b"age": b"22"} - self.assertEqual(expected, exporter.export_item(value)) - def test_nonstring_types_item(self): item = self._get_nonstring_types_item() ie = self._get_exporter() From 9e74748fca94e6b5c2c70346cd8789d80bc04507 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Aug 2023 08:48:43 -0300 Subject: [PATCH 206/211] Remove extra spider parameter in item pipeline docs (#6009) --- docs/topics/item-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index bc26bbebe..a5f6e07b8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -215,7 +215,7 @@ item. screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) response = await maybe_deferred_to_future( - spider.crawler.engine.download(request, spider) + spider.crawler.engine.download(request) ) if response.status != 200: From 9df67a554e542b3ec8f92491c3840fca5ea182d2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 10 Aug 2023 22:53:22 +0400 Subject: [PATCH 207/211] Add RequestFingerprinterProtocol. --- scrapy/dupefilters.py | 12 ++++++++---- scrapy/utils/request.py | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index bc912268c..d2639104b 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -12,7 +12,11 @@ from scrapy.settings import BaseSettings from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.job import job_dir -from scrapy.utils.request import RequestFingerprinter, referer_str +from scrapy.utils.request import ( + RequestFingerprinter, + RequestFingerprinterProtocol, + referer_str, +) if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -48,10 +52,10 @@ class RFPDupeFilter(BaseDupeFilter): path: Optional[str] = None, debug: bool = False, *, - fingerprinter: Optional[RequestFingerprinter] = None, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> None: self.file = None - self.fingerprinter: RequestFingerprinter = ( + self.fingerprinter: RequestFingerprinterProtocol = ( fingerprinter or RequestFingerprinter() ) self.fingerprints: Set[str] = set() @@ -68,7 +72,7 @@ class RFPDupeFilter(BaseDupeFilter): cls, settings: BaseSettings, *, - fingerprinter: Optional[RequestFingerprinter] = None, + fingerprinter: Optional[RequestFingerprinterProtocol] = None, ) -> Self: debug = settings.getbool("DUPEFILTER_DEBUG") try: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 6c7f3b345..24fcbd85e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -14,6 +14,7 @@ from typing import ( Iterable, List, Optional, + Protocol, Tuple, Type, Union, @@ -230,6 +231,11 @@ def fingerprint( return cache[cache_key] +class RequestFingerprinterProtocol(Protocol): + def fingerprint(self, request: Request) -> bytes: + ... + + class RequestFingerprinter: """Default fingerprinter. From 44b15c3004d950021460293c391f2283f2418726 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 10 Aug 2023 23:01:54 +0400 Subject: [PATCH 208/211] Remove typing for CrawlerRunner.spider_loader. --- scrapy/crawler.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 531798893..ee845a831 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast from twisted.internet.defer import ( Deferred, @@ -31,7 +31,6 @@ from scrapy.interfaces import ISpiderLoader from scrapy.logformatter import LogFormatter from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.spiderloader import SpiderLoader from scrapy.statscollectors import StatsCollector from scrapy.utils.log import ( LogCounterHandler, @@ -182,10 +181,10 @@ class CrawlerRunner: ) @staticmethod - def _get_spider_loader(settings: BaseSettings) -> SpiderLoader: + def _get_spider_loader(settings: BaseSettings): """Get SpiderLoader instance from settings""" cls_path = settings.get("SPIDER_LOADER_CLASS") - loader_cls: Type[SpiderLoader] = load_object(cls_path) + loader_cls = load_object(cls_path) excs = ( (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement ) @@ -211,7 +210,7 @@ class CrawlerRunner: self.bootstrap_failed = False @property - def spiders(self) -> SpiderLoader: + def spiders(self): warnings.warn( "CrawlerRunner.spiders attribute is renamed to " "CrawlerRunner.spider_loader.", @@ -293,7 +292,8 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + # temporary cast until self.spider_loader is typed + return Crawler(cast(Type[Spider], spidercls), self.settings) def stop(self) -> Deferred: """ @@ -375,7 +375,10 @@ class CrawlerProcess(CrawlerRunner): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - return Crawler(spidercls, self.settings, init_reactor=init_reactor) + # temporary cast until self.spider_loader is typed + return Crawler( + cast(Type[Spider], spidercls), self.settings, init_reactor=init_reactor + ) def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True From 34d050cfe5aec94a15f00950fcecbc49cb470fb8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 11 Aug 2023 12:41:05 +0400 Subject: [PATCH 209/211] Remove deprecated CrawlerRunner.spiders. (#6010) --- scrapy/crawler.py | 10 ---------- tests/test_crawler.py | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c5b3e1903..bc0ab02df 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -204,16 +204,6 @@ class CrawlerRunner: self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False - @property - def spiders(self): - warnings.warn( - "CrawlerRunner.spiders attribute is renamed to " - "CrawlerRunner.spider_loader.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return self.spider_loader - def crawl(self, crawler_or_spidercls, *args, **kwargs): """ Run a crawler with the provided arguments. diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f99606ccf..4c5c48e6d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -20,7 +20,6 @@ from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler -from scrapy.utils.misc import load_object from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env @@ -182,16 +181,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_deprecated_attribute_spiders(self): - with warnings.catch_warnings(record=True) as w: - runner = CrawlerRunner(Settings()) - spiders = runner.spiders - self.assertEqual(len(w), 1) - self.assertIn("CrawlerRunner.spiders", str(w[0].message)) - self.assertIn("CrawlerRunner.spider_loader", str(w[0].message)) - sl_cls = load_object(runner.settings["SPIDER_LOADER_CLASS"]) - self.assertIsInstance(spiders, sl_cls) - class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): From b06936f111741f16aae8338ea8006435890bb10d Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 14 Aug 2023 10:33:48 -0300 Subject: [PATCH 210/211] Handle Tuple type on getdictorlist method, bump 3.12 python version --- .github/workflows/tests-ubuntu.yml | 6 +++--- scrapy/settings/__init__.py | 4 +++- tests/test_feedexport.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 54b3fbaa2..c2b686628 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -48,13 +48,13 @@ jobs: env: TOXENV: botocore - - python-version: "3.12.0-beta.4" + - python-version: "3.12.0-rc.1" env: TOXENV: py - - python-version: "3.12.0-beta.4" + - python-version: "3.12.0-rc.1" env: TOXENV: asyncio - - python-version: "3.12.0-beta.4" + - python-version: "3.12.0-rc.1" env: TOXENV: extra-deps diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index bc82cc098..ba9727bac 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -238,7 +238,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, name: _SettingsKeyT, - default: Union[Dict[Any, Any], List[Any], None] = None, + default: Union[Dict[Any, Any], List[Any], Tuple[Any], None] = None, ) -> Union[Dict[Any, Any], List[Any]]: """Get a setting value as either a :class:`dict` or a :class:`list`. @@ -271,6 +271,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return value_loaded except ValueError: return value.split(",") + if isinstance(value, tuple): + return list(value) assert isinstance(value, (dict, list)) return copy.deepcopy(value) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 42fa25b1d..6b82974fa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1321,6 +1321,17 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) + @defer.inlineCallbacks + def test_export_tuple(self): + items = [ + {"foo": "bar1", "egg": "spam1"}, + {"foo": "bar2", "egg": "spam2", "baz": "quux"}, + ] + + settings = {"FEED_EXPORT_FIELDS": ("foo", "baz")} + rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] + yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) + @defer.inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields From df2163ce6aded0ed56eed8b125739193b953ff17 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:51:49 -0300 Subject: [PATCH 211/211] Remove datetime.utcnow() usage (#6014) --- scrapy/extensions/corestats.py | 6 +++--- scrapy/extensions/feedexport.py | 4 ++-- tests/keys/__init__.py | 6 +++--- tests/test_spiderstate.py | 4 ++-- tests/test_stats.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 30c987253..302a615f2 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,7 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime +from datetime import datetime, timezone from scrapy import signals @@ -22,11 +22,11 @@ class CoreStats: return o def spider_opened(self, spider): - self.start_time = datetime.utcnow() + self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) def spider_closed(self, spider, reason): - finish_time = datetime.utcnow() + finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() self.stats.set_value( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c1b77f4fb..4e846d1bd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -8,7 +8,7 @@ import logging import re import sys import warnings -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union @@ -676,7 +676,7 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - utc_now = datetime.utcnow() + utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") params["batch_id"] = slot.batch_id + 1 if slot is not None else 1 diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index 5cc65a903..9b73ca4f0 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from cryptography.hazmat.backends import default_backend @@ -50,8 +50,8 @@ def generate_keys(): .issuer_name(issuer) .public_key(key.public_key()) .serial_number(random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after(datetime.utcnow() + timedelta(days=10)) + .not_valid_before(datetime.now(tz=timezone.utc)) + .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10)) .add_extension( SubjectAlternativeName([DNSName("localhost")]), critical=False, diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f645f4cce..f97125b76 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,5 @@ import shutil -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from twisted.trial import unittest @@ -16,7 +16,7 @@ class SpiderStateTest(unittest.TestCase): Path(jobdir).mkdir() try: spider = Spider(name="default") - dt = datetime.now() + dt = datetime.now(tz=timezone.utc) ss = SpiderState(jobdir) ss.spider_opened(spider) diff --git a/tests/test_stats.py b/tests/test_stats.py index 7a8adf638..3d4c7e88e 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -16,7 +16,7 @@ class CoreStatsExtensionTest(unittest.TestCase): @mock.patch("scrapy.extensions.corestats.datetime") def test_core_stats_default_stats_collector(self, mock_datetime): fixed_datetime = datetime(2019, 12, 1, 11, 38) - mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime) + mock_datetime.now = mock.Mock(return_value=fixed_datetime) self.crawler.stats = StatsCollector(self.crawler) ext = CoreStats.from_crawler(self.crawler) ext.spider_opened(self.spider)