diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b949d81c4..4cfba674d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.7.1 +current_version = 2.8.0 commit = True tag = True tag_name = {new_version} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eee9a4f02..22b8996b6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,31 +1,21 @@ name: Publish -on: [push] +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+' jobs: publish: runs-on: ubuntu-latest - if: startsWith(github.event.ref, 'refs/tags/') - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Check Tag - id: check-release-tag - run: | - if [[ ${{ github.event.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$ ]]; then - echo ::set-output name=release_tag::true - fi - - - name: Publish to PyPI - if: steps.check-release-tag.outputs.release_tag == 'true' - run: | - pip install --upgrade build twine - python -m build - export TWINE_USERNAME=__token__ - export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} - twine upload dist/* + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + - run: | + pip install --upgrade build twine + python -m build + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.6.4 + with: + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 729682392..cfb5a46dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,15 +5,15 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 5.0.4 # 6.0.0 drops Python 3.7 support hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.11.5 # 5.12 drops Python 3.7 support hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs diff --git a/docs/contributing.rst b/docs/contributing.rst index edc6c2179..6b1a41339 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -49,7 +49,7 @@ guidelines when you're going to report a new bug. (use "scrapy" tag). * check the `open issues`_ to see if the issue has already been reported. If it - has, don't dismiss the report, but check the ticket history and comments. If + has, don't dismiss the report, but check the ticket history and comments. If you have additional useful information, please leave a comment, or consider :ref:`sending a pull request ` with a fix. @@ -169,7 +169,7 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `black `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also run black manually with ``tox -e black``. @@ -179,29 +179,31 @@ Scrapy: See https://help.github.com/en/github/using-git/setting-your-username-in-git for setup instructions. +.. _scrapy-pre-commit: + Pre-commit ========== -We use `pre-commit`_ to automatically address simple code issues before every +We use `pre-commit`_ to automatically address simple code issues before every commit. .. _pre-commit: https://pre-commit.com/ -Before you start writing a patch: +After your create a local clone of your fork of the Scrapy repository: #. `Install pre-commit `_. -#. On the root of your local clone of the Scrapy repository, run the following +#. On the root of your local clone of the Scrapy repository, run the following command: .. code-block:: bash pre-commit install -Now pre-commit will check your changes every time you create a Git commit. Upon -finding issues, pre-commit aborts your commit, and either fixes those issues -automatically, or only reports them to you. If it fixes those issues -automatically, creating your commit again should succeed. Otherwise, you may +Now pre-commit will check your changes every time you create a Git commit. Upon +finding issues, pre-commit aborts your commit, and either fixes those issues +automatically, or only reports them to you. If it fixes those issues +automatically, creating your commit again should succeed. Otherwise, you may need to address the corresponding issues manually first. .. _documentation-policies: diff --git a/docs/news.rst b/docs/news.rst index dd1574e26..9b9eeac71 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,212 @@ Release notes ============= +.. _release-2.8.0: + +Scrapy 2.8.0 (2023-02-02) +------------------------- + +This is a maintenance release, with minor features, bug fixes, and cleanups. + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``scrapy.utils.gz.read1`` function, deprecated in Scrapy 2.0, has now + been removed. Use the :meth:`~io.BufferedIOBase.read1` method of + :class:`~gzip.GzipFile` instead. + (:issue:`5719`) + +- The ``scrapy.utils.python.to_native_str`` function, deprecated in Scrapy + 2.0, has now been removed. Use :func:`scrapy.utils.python.to_unicode` + instead. + (:issue:`5719`) + +- The ``scrapy.utils.python.MutableChain.next`` method, deprecated in Scrapy + 2.0, has now been removed. Use + :meth:`~scrapy.utils.python.MutableChain.__next__` instead. + (:issue:`5719`) + +- The ``scrapy.linkextractors.FilteringLinkExtractor`` class, deprecated + in Scrapy 2.0, has now been removed. Use + :class:`LinkExtractor ` + instead. + (:issue:`5720`) + +- Support for using environment variables prefixed with ``SCRAPY_`` to + override settings, deprecated in Scrapy 2.0, has now been removed. + (:issue:`5724`) + +- Support for the ``noconnect`` query string argument in proxy URLs, + deprecated in Scrapy 2.0, has now been removed. We expect proxies that used + to need it to work fine without it. + (:issue:`5731`) + +- The ``scrapy.utils.python.retry_on_eintr`` function, deprecated in Scrapy + 2.3, has now been removed. + (:issue:`5719`) + +- The ``scrapy.utils.python.WeakKeyCache`` class, deprecated in Scrapy 2.4, + has now been removed. + (:issue:`5719`) + + +Deprecations +~~~~~~~~~~~~ + +- :exc:`scrapy.pipelines.images.NoimagesDrop` is now deprecated. + (:issue:`5368`, :issue:`5489`) + +- :meth:`ImagesPipeline.convert_image + ` must now accept a + ``response_body`` parameter. + (:issue:`3055`, :issue:`3689`, :issue:`4753`) + + +New features +~~~~~~~~~~~~ + +- Applied black_ coding style to files generated with the + :command:`genspider` and :command:`startproject` commands. + (:issue:`5809`, :issue:`5814`) + + .. _black: https://black.readthedocs.io/en/stable/ + +- :setting:`FEED_EXPORT_ENCODING` is now set to ``"utf-8"`` in the + ``settings.py`` file that the :command:`startproject` command generates. + With this value, JSON exports won’t force the use of escape sequences for + non-ASCII characters. + (:issue:`5797`, :issue:`5800`) + +- The :class:`~scrapy.extensions.memusage.MemoryUsage` extension now logs the + peak memory usage during checks, and the binary unit MiB is now used to + avoid confusion. + (:issue:`5717`, :issue:`5722`, :issue:`5727`) + +- The ``callback`` parameter of :class:`~scrapy.http.Request` can now be set + to :func:`scrapy.http.request.NO_CALLBACK`, to distinguish it from + ``None``, as the latter indicates that the default spider callback + (:meth:`~scrapy.Spider.parse`) is to be used. + (:issue:`5798`) + + +Bug fixes +~~~~~~~~~ + +- Enabled unsafe legacy SSL renegotiation to fix access to some outdated + websites. + (:issue:`5491`, :issue:`5790`) + +- Fixed STARTTLS-based email delivery not working with Twisted 21.2.0 and + better. + (:issue:`5386`, :issue:`5406`) + +- Fixed the :meth:`finish_exporting` method of :ref:`item exporters + ` not being called for empty files. + (:issue:`5537`, :issue:`5758`) + +- Fixed HTTP/2 responses getting only the last value for a header when + multiple headers with the same name are received. + (:issue:`5777`) + +- Fixed an exception raised by the :command:`shell` command on some cases + when :ref:`using asyncio `. + (:issue:`5740`, :issue:`5742`, :issue:`5748`, :issue:`5759`, :issue:`5760`, + :issue:`5771`) + +- When using :class:`~scrapy.spiders.CrawlSpider`, callback keyword arguments + (``cb_kwargs``) added to a request in the ``process_request`` callback of a + :class:`~scrapy.spiders.Rule` will no longer be ignored. + (:issue:`5699`) + +- The :ref:`images pipeline ` no longer re-encodes JPEG + files. + (:issue:`3055`, :issue:`3689`, :issue:`4753`) + +- Fixed the handling of transparent WebP images by the :ref:`images pipeline + `. + (:issue:`3072`, :issue:`5766`, :issue:`5767`) + +- :func:`scrapy.shell.inspect_response` no longer inhibits ``SIGINT`` + (Ctrl+C). + (:issue:`2918`) + +- :class:`LinkExtractor ` + with ``unique=False`` no longer filters out links that have identical URL + *and* text. + (:issue:`3798`, :issue:`3799`, :issue:`4695`, :issue:`5458`) + +- :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` now + ignores URL protocols that do not support ``robots.txt`` (``data://``, + ``file://``). + (:issue:`5807`) + +- Silenced the ``filelock`` debug log messages introduced in Scrapy 2.6. + (:issue:`5753`, :issue:`5754`) + +- Fixed the output of ``scrapy -h`` showing an unintended ``**commands**`` + line. + (:issue:`5709`, :issue:`5711`, :issue:`5712`) + +- Made the active project indication in the output of :ref:`commands + ` more clear. + (:issue:`5715`) + + +Documentation +~~~~~~~~~~~~~ + +- Documented how to :ref:`debug spiders from Visual Studio Code + `. + (:issue:`5721`) + +- Documented how :setting:`DOWNLOAD_DELAY` affects per-domain concurrency. + (:issue:`5083`, :issue:`5540`) + +- Improved consistency. + (:issue:`5761`) + +- Fixed typos. + (:issue:`5714`, :issue:`5744`, :issue:`5764`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Applied :ref:`black coding style `, sorted import statements, + and introduced :ref:`pre-commit `. + (:issue:`4654`, :issue:`4658`, :issue:`5734`, :issue:`5737`, :issue:`5806`, + :issue:`5810`) + +- Switched from :mod:`os.path` to :mod:`pathlib`. + (:issue:`4916`, :issue:`4497`, :issue:`5682`) + +- Addressed many issues reported by Pylint. + (:issue:`5677`) + +- Improved code readability. + (:issue:`5736`) + +- Improved package metadata. + (:issue:`5768`) + +- Removed direct invocations of ``setup.py``. + (:issue:`5774`, :issue:`5776`) + +- Removed unnecessary :class:`~collections.OrderedDict` usages. + (:issue:`5795`) + +- Removed unnecessary ``__str__`` definitions. + (:issue:`5150`) + +- Removed obsolete code and comments. + (:issue:`5725`, :issue:`5729`, :issue:`5730`, :issue:`5732`) + +- Fixed test and CI issues. + (:issue:`5749`, :issue:`5750`, :issue:`5756`, :issue:`5762`, :issue:`5765`, + :issue:`5780`, :issue:`5781`, :issue:`5782`, :issue:`5783`, :issue:`5785`, + :issue:`5786`) + + .. _release-2.7.1: Scrapy 2.7.1 (2022-11-02) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index b133fcc1e..6ecba110c 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -164,6 +164,8 @@ For more information, check the :ref:`topics-logging` section. .. _base tag: https://www.w3schools.com/tags/tag_base.asp +.. _debug-vscode: + Visual Studio Code ================== diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 554fc2976..eef0bb5ca 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -516,7 +516,7 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``utf-8`` if you want UTF-8 for JSON too. -.. versionchanged:: VERSION +.. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to ``utf-8`` in the generated ``settings.py`` file. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index fd38eb504..b516a6bef 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -39,9 +39,7 @@ Request objects - ``None`` (default), which indicates that the spider's :meth:`~scrapy.Spider.parse` method must be used. - - :py:data:`scrapy.http.request.NO_CALLBACK` - - .. autodata:: scrapy.http.request.NO_CALLBACK + - :func:`~scrapy.http.request.NO_CALLBACK` For more information, see :ref:`topics-request-response-ref-request-callback-arguments`. @@ -253,6 +251,8 @@ Request objects Other functions related to requests ----------------------------------- +.. autofunction:: scrapy.http.request.NO_CALLBACK + .. autofunction:: scrapy.utils.request.request_from_dict diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index efb4c202e..1f270837c 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -18,7 +18,6 @@ from pathlib import Path def main(): - # Used for remembering the file (and its contents) # so we don't have to open the same file again. _filename = None @@ -50,7 +49,6 @@ def main(): else: # If this is a new file if newfilename != _filename: - # Update the previous file if _filename: Path(_filename).write_text(_contents, encoding="utf-8") diff --git a/extras/qpsclient.py b/extras/qpsclient.py index acad71e07..119dfdabb 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -13,7 +13,6 @@ from scrapy.spiders import Spider class QPSSpider(Spider): - name = "qps" benchurl = "http://localhost:8880/" diff --git a/pylintrc b/pylintrc index 94465fca4..c8654b8d3 100644 --- a/pylintrc +++ b/pylintrc @@ -12,6 +12,7 @@ disable=abstract-method, bad-mcs-classmethod-argument, bare-except, broad-except, + broad-exception-raised, c-extension-no-member, catching-non-exception, cell-var-from-loop, @@ -46,12 +47,14 @@ disable=abstract-method, method-hidden, missing-docstring, no-else-raise, + no-else-return, no-member, no-method-argument, no-name-in-module, no-self-argument, no-value-for-parameter, not-callable, + pointless-exception-statement, pointless-statement, pointless-string-statement, protected-access, @@ -87,6 +90,7 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, + use-dict-literal, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, diff --git a/scrapy/VERSION b/scrapy/VERSION index 860487ca1..834f26295 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.7.1 +2.8.0 diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index de68c43a5..9baee3a48 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -14,7 +14,6 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: - requires_project = False crawler_process: Optional[CrawlerProcess] = None diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 911e5afe6..e1ccdc451 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -9,7 +9,6 @@ from scrapy.linkextractors import LinkExtractor class Command(ScrapyCommand): - default_settings = { "LOG_LEVEL": "INFO", "LOGSTATS_INTERVAL": 1, diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index df8006f36..2f0f1c7b9 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommand): - requires_project = True def syntax(self): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index ca591011c..03a8ed5c7 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -6,7 +6,6 @@ from scrapy.exceptions import UsageError class Command(ScrapyCommand): - requires_project = True default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a9076c5b1..1359e445f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -10,7 +10,6 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): - requires_project = False def syntax(self): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 90dd0874e..c1565a138 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -32,7 +32,6 @@ def extract_domain(url): class Command(ScrapyCommand): - requires_project = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2a81bd182..2f5032360 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -2,7 +2,6 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): - requires_project = True default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 8a75f9270..58ed89a81 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -24,7 +24,6 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType: class Command(BaseRunSpiderCommand): - requires_project = False default_settings = {"SPIDER_LOADER_WARN_ONLY": True} diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index c43298ac7..318187204 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -5,7 +5,6 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): - requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 05c76d1eb..63c23d04c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -13,7 +13,6 @@ from scrapy.utils.url import guess_scheme class Command(ScrapyCommand): - requires_project = False default_settings = { "KEEP_ALIVE": True, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 88bd5bb33..fde609c6f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -28,7 +28,6 @@ def _make_writable(path): class Command(ScrapyCommand): - requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index faa0e242c..47582866b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -4,7 +4,6 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} def syntax(self): diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7e0b62bb0..7d27a5193 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -69,7 +69,6 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader: - DOWNLOAD_SLOT = "download_slot" def __init__(self, crawler): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8de5459e9..c3704de3d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -292,7 +292,6 @@ class ScrapyProxyAgent(Agent): class ScrapyAgent: - _Agent = Agent _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 5a94e66a6..56df48b2e 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -17,7 +17,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferred class DownloaderMiddlewareManager(MiddlewareManager): - component_name = "downloader middleware" @classmethod diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 4558402b2..3d103652b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -40,7 +40,6 @@ def _parse(url): class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b"\n" def connectionMade(self): @@ -103,7 +102,6 @@ class ScrapyHTTPPageGetter(HTTPClient): # Twisted (https://github.com/twisted/twisted/pull/643), we merged its # non-overridden code into this class. class ScrapyHTTPClientFactory(ClientFactory): - protocol = ScrapyHTTPPageGetter waiting = 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index ba9c37e38..971cf52e4 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -46,7 +46,6 @@ def _isiterable(o) -> bool: class SpiderMiddlewareManager(MiddlewareManager): - component_name = "spider middleware" def __init__(self, *middlewares): diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 137ed5b18..04ae719de 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -30,7 +30,6 @@ class AjaxCrawlMiddleware: return cls(crawler.settings) def process_response(self, request, response, spider): - if not isinstance(response, HtmlResponse) or response.status != 200: return response diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 74c55f6e2..b9316c43a 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -27,7 +27,6 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlew class HttpCacheMiddleware: - DOWNLOAD_EXCEPTIONS = ( defer.TimeoutError, TimeoutError, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 7b5f4824a..ead426951 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -53,7 +53,6 @@ class HttpCompressionMiddleware: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): - if request.method == "HEAD": return response if isinstance(response, Response): diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index f442a3012..65f1d2224 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -26,7 +26,6 @@ def _build_redirect_request(source_request, *, url, **kwargs): class BaseRedirectMiddleware: - enabled_setting = "REDIRECT_ENABLED" def __init__(self, settings): @@ -115,7 +114,6 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): - enabled_setting = "METAREFRESH_ENABLED" def __init__(self, settings): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 11a30911c..081642a4b 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -122,7 +122,6 @@ def get_retry_request( class RetryMiddleware: - # IOError is raised by the HttpCompression middleware when trying to # decompress an empty response EXCEPTIONS_TO_RETRY = ( diff --git a/scrapy/extension.py b/scrapy/extension.py index 89f6a694d..4e365cfa1 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -8,7 +8,6 @@ from scrapy.utils.conf import build_component_list class ExtensionManager(MiddlewareManager): - component_name = "extension" @classmethod diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 2d120a6ed..dfe843974 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -41,7 +41,6 @@ class DummyPolicy: class RFC2616Policy: - MAXAGE = 3600 * 24 * 365 # one year def __init__(self, settings): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7afb28db5..9ba6ddf20 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -25,6 +25,12 @@ def NO_CALLBACK(*args, **kwargs): :class:`~scrapy.http.Request`, it indicates that the request is not meant to have a spider callback at all. + For example: + + .. code-block:: python + + Request("https://example.com", callback=NO_CALLBACK) + This value should be used by :ref:`components ` that create and handle their own requests, e.g. through :meth:`scrapy.core.engine.ExecutionEngine.download`, so that downloader diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 35e2808c2..510c903db 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -15,7 +15,6 @@ from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): - attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index e45d95602..73bb811de 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -29,7 +29,6 @@ _NONE = object() class TextResponse(Response): - _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 66cf58d89..dd8dcdf7c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -194,7 +194,6 @@ class LxmlLinkExtractor: return True def matches(self, url): - if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 536341fc6..df574a0a1 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -10,7 +10,6 @@ from scrapy.utils.defer import deferred_f_from_coro_f class ItemPipelineManager(MiddlewareManager): - component_name = "item pipeline" @classmethod diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 3430603ab..0be28933e 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -187,7 +187,6 @@ class S3FilesStore: class GCSFilesStore: - GCS_PROJECT_ID = None CACHE_CONTROL = "max-age=172800" @@ -253,7 +252,6 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME = None FTP_PASSWORD = None USE_ACTIVE_MODE = None diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 31f51c004..401029439 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -23,7 +23,6 @@ def _DUMMY_CALLBACK(response): class MediaPipeline: - LOG_FAILED_RESULTS = True class SpiderInfo: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 6af8915c2..f01e9096c 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -12,7 +12,6 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicode class ResponseTypes: - CLASSES = { "text/html": "scrapy.http.HtmlResponse", "application/atom+xml": "scrapy.http.XmlResponse", diff --git a/scrapy/shell.py b/scrapy/shell.py index ae6e641fd..bb3b1461c 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -25,7 +25,6 @@ from scrapy.utils.response import open_in_browser class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) def __init__(self, crawler, update_vars=None, code=None): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d86f55a40..fd91e658b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -34,7 +34,6 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES name: str diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 05c425948..31e845716 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -60,7 +60,6 @@ class Rule: class CrawlSpider(Spider): - rules: Sequence[Rule] = () def __init__(self, *a, **kw): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 5ec0504a8..6afadc577 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -89,7 +89,7 @@ class XMLFeedSpider(Spider): yield node def _register_namespaces(self, selector): - for (prefix, uri) in self.namespaces: + for prefix, uri in self.namespaces: selector.register_namespace(prefix, uri) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c3cca9699..aaf75a519 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -10,7 +10,6 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): - sitemap_urls = () sitemap_rules = [("", "parse")] sitemap_follow = [""] diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index bd09890fe..8c9a86dce 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -53,7 +53,7 @@ class ${ProjectName}SpiderMiddleware: yield r def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) + spider.logger.info("Spider opened: %s" % spider.name) class ${ProjectName}DownloaderMiddleware: @@ -100,4 +100,4 @@ class ${ProjectName}DownloaderMiddleware: pass def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) + spider.logger.info("Spider opened: %s" % spider.name) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 2f6df5abc..ecb1e5e5c 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -7,14 +7,14 @@ # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html # https://docs.scrapy.org/en/latest/topics/spider-middleware.html -BOT_NAME = '$project_name' +BOT_NAME = "$project_name" -SPIDER_MODULES = ['$project_name.spiders'] -NEWSPIDER_MODULE = '$project_name.spiders' +SPIDER_MODULES = ["$project_name.spiders"] +NEWSPIDER_MODULE = "$project_name.spiders" # Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = '$project_name (+http://www.yourdomain.com)' +#USER_AGENT = "$project_name (+http://www.yourdomain.com)" # Obey robots.txt rules ROBOTSTXT_OBEY = True @@ -38,32 +38,32 @@ ROBOTSTXT_OBEY = True # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { -# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -# 'Accept-Language': 'en', +# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +# "Accept-Language": "en", #} # Enable or disable spider middlewares # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, +# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543, #} # Enable or disable downloader middlewares # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, +# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543, #} # Enable or disable extensions # See https://docs.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { -# 'scrapy.extensions.telnet.TelnetConsole': None, +# "scrapy.extensions.telnet.TelnetConsole": None, #} # Configure item pipelines # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { -# '$project_name.pipelines.${ProjectName}Pipeline': 300, +# "$project_name.pipelines.${ProjectName}Pipeline": 300, #} # Enable and configure the AutoThrottle extension (disabled by default) @@ -83,11 +83,11 @@ ROBOTSTXT_OBEY = True # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 -#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_DIR = "httpcache" #HTTPCACHE_IGNORE_HTTP_CODES = [] -#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' +#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' -TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' -FEED_EXPORT_ENCODING = 'utf-8' +REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7" +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" +FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index e9112bc95..d3ba19553 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -2,9 +2,9 @@ import scrapy class $classname(scrapy.Spider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/"] def parse(self, response): pass diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 356496487..2e467e632 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -4,17 +4,15 @@ from scrapy.spiders import CrawlSpider, Rule class $classname(CrawlSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/"] - rules = ( - Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), - ) + rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),) def parse_item(self, response): item = {} - #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() - #item['name'] = response.xpath('//div[@id="name"]').get() - #item['description'] = response.xpath('//div[@id="description"]').get() + #item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get() + #item["name"] = response.xpath('//div[@id="name"]').get() + #item["description"] = response.xpath('//div[@id="description"]').get() return item diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index cbcbe9e2c..ce9c1dd20 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -2,11 +2,11 @@ from scrapy.spiders import CSVFeedSpider class $classname(CSVFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.csv'] - # headers = ['id', 'name', 'description', 'image_link'] - # delimiter = '\t' + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/feed.csv"] + #headers = ["id", "name", "description", "image_link"] + #delimiter = "\t" # Do any adaptations you need here #def adapt_response(self, response): @@ -14,7 +14,7 @@ class $classname(CSVFeedSpider): def parse_row(self, response, row): i = {} - #i['url'] = row['url'] - #i['name'] = row['name'] - #i['description'] = row['description'] + #i["url"] = row["url"] + #i["name"] = row["name"] + #i["description"] = row["description"] return i diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 5aa2aa8b0..6b50e4cf4 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -2,15 +2,15 @@ from scrapy.spiders import XMLFeedSpider class $classname(XMLFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.xml'] - iterator = 'iternodes' # you can change this; see the docs - itertag = 'item' # change it accordingly + name = "$name" + allowed_domains = ["$domain"] + start_urls = ["http://$domain/feed.xml"] + iterator = "iternodes" # you can change this; see the docs + itertag = "item" # change it accordingly def parse_node(self, response, selector): item = {} - #item['url'] = selector.select('url').get() - #item['name'] = selector.select('name').get() - #item['description'] = selector.select('description').get() + #item["url"] = selector.select("url").get() + #item["name"] = selector.select("name").get() + #item["description"] = selector.select("description").get() return item diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 1089ba7b8..38884a9f0 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -6,7 +6,6 @@ from twisted.web.server import Site class Root(Resource): - isLeaf = True def getChild(self, name, request): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 10497e9ed..fa57a4f26 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -11,7 +11,6 @@ from collections.abc import Mapping class CaselessDict(dict): - __slots__ = () def __init__(self, seq=None): diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 61a4347ea..f4d6e0451 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -54,7 +54,6 @@ def create_deprecated_class( """ class DeprecatedClass(new_class.__class__): - deprecated_class = None warned_on_subclass = False diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 358f41679..414658944 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -9,7 +9,6 @@ from scrapy.http import Request, Response class ScrapyJSONEncoder(json.JSONEncoder): - DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index ecb2e31bf..5f9bdef37 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -5,7 +5,6 @@ from twisted.internet import defer, protocol class ProcessTest: - command = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 0a27ccd6d..833aa3e20 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -144,7 +144,6 @@ def strip_url( origin_only=False, strip_fragment=True, ): - """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" diff --git a/tests/mockserver.py b/tests/mockserver.py index 7991da9dc..d541f3f7c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -95,7 +95,6 @@ class BrokenDownloadResource(resource.Resource): class LeafResource(resource.Resource): - isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): diff --git a/tests/requirements.txt b/tests/requirements.txt index d9373dfa8..618949795 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,7 +2,7 @@ attrs pyftpdlib pytest -pytest-cov==3.0.0 +pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures diff --git a/tests/spiders.py b/tests/spiders.py index 2036e8cf3..6ff48f471 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -25,7 +25,6 @@ class MockServerSpider(Spider): class MetaSpider(MockServerSpider): - name = "meta" def __init__(self, *args, **kwargs): @@ -37,7 +36,6 @@ class MetaSpider(MockServerSpider): class FollowAllSpider(MetaSpider): - name = "follow" link_extractor = LinkExtractor() @@ -59,7 +57,6 @@ class FollowAllSpider(MetaSpider): class DelaySpider(MetaSpider): - name = "delay" def __init__(self, n=1, b=0, *args, **kwargs): @@ -81,7 +78,6 @@ class DelaySpider(MetaSpider): class SimpleSpider(MetaSpider): - name = "simple" def __init__(self, url="http://localhost:8998", *args, **kwargs): @@ -93,7 +89,6 @@ class SimpleSpider(MetaSpider): class AsyncDefSpider(SimpleSpider): - name = "asyncdef" async def parse(self, response): @@ -102,7 +97,6 @@ class AsyncDefSpider(SimpleSpider): class AsyncDefAsyncioSpider(SimpleSpider): - name = "asyncdef_asyncio" async def parse(self, response): @@ -112,7 +106,6 @@ class AsyncDefAsyncioSpider(SimpleSpider): class AsyncDefAsyncioReturnSpider(SimpleSpider): - name = "asyncdef_asyncio_return" async def parse(self, response): @@ -123,7 +116,6 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): - name = "asyncdef_asyncio_return_single_element" async def parse(self, response): @@ -134,7 +126,6 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): - name = "asyncdef_asyncio_reqs_return" async def parse(self, response): @@ -191,7 +182,6 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): class AsyncDefAsyncioGenSpider(SimpleSpider): - name = "asyncdef_asyncio_gen" async def parse(self, response): @@ -201,7 +191,6 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - name = "asyncdef_asyncio_gen_loop" async def parse(self, response): @@ -212,7 +201,6 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - name = "asyncdef_asyncio_gen_complex" initial_reqs = 4 following_reqs = 3 @@ -246,7 +234,6 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): class ItemSpider(FollowAllSpider): - name = "item" def parse(self, response): @@ -261,7 +248,6 @@ class DefaultError(Exception): class ErrorSpider(FollowAllSpider): - name = "error" exception_cls = DefaultError @@ -275,7 +261,6 @@ class ErrorSpider(FollowAllSpider): class BrokenStartRequestsSpider(FollowAllSpider): - fail_before_yield = False fail_yielding = False @@ -305,7 +290,6 @@ class BrokenStartRequestsSpider(FollowAllSpider): class SingleRequestSpider(MetaSpider): - seed = None callback_func = None errback_func = None @@ -451,7 +435,6 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse class BytesReceivedCallbackSpider(MetaSpider): - full_response_length = 2**18 @classmethod diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 1f299587f..129ef0121 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -2,7 +2,6 @@ from tests.test_commands import CommandTest class CheckCommandTest(CommandTest): - command = "check" def setUp(self): diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 124c968c2..d2027d1c2 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -6,7 +6,6 @@ from scrapy.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): - command = "fetch" @defer.inlineCallbacks diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 8ce82db86..6589381f3 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -9,7 +9,6 @@ from tests import NON_EXISTING_RESOLVABLE, tests_datadir class ShellTest(ProcessTest, SiteTest, unittest.TestCase): - command = "shell" @defer.inlineCallbacks diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 3bf6019b5..a52d0d13c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -8,7 +8,6 @@ from scrapy.utils.testproc import ProcessTest class VersionTest(ProcessTest, unittest.TestCase): - command = "version" @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index 5ff2dd482..00ddcdd3e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -223,7 +223,6 @@ def get_permissions_dict( class StartprojectTemplatesTest(ProjectTest): - maxDiff = None def setUp(self): @@ -505,7 +504,7 @@ class GenspiderCommandTest(CommandTest): # change name of spider but not its file name with file_path.open("r+", encoding="utf-8") as spider_file: file_data = spider_file.read() - file_data = file_data.replace("name = 'example'", "name = 'renamed'") + file_data = file_data.replace('name = "example"', 'name = "renamed"') spider_file.seek(0) spider_file.write(file_data) spider_file.truncate() @@ -538,14 +537,14 @@ class GenspiderCommandTest(CommandTest): domain, self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[\'(.+)\'\]", + r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", ).group(1), ) self.assertEqual( f"http://{domain}/", self.find_in_file( Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[\'(.+)\'\]", + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", ).group(1), ) @@ -604,7 +603,6 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): - spider_filename = "myspider.py" debug_log_spider = """ @@ -873,7 +871,6 @@ class MySpider(scrapy.Spider): @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): - spider_filename = "myspider.pyw" def setUp(self): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8090d50b9..31fa1430d 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -25,7 +25,6 @@ from tests.test_downloader_handlers import ( @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): - scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 2be32e37b..060cfe08b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -16,7 +16,6 @@ from scrapy.utils.test import get_crawler, get_from_asyncio_queue class ManagerTestCase(TestCase): - settings_dict = None def setUp(self): diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 412c20a78..95739414e 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -17,7 +17,6 @@ def _test_data(formats): class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] uncompressed_body, test_responses = _test_data(test_formats) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a355a9b5b..f80eff3e6 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -14,7 +14,6 @@ from scrapy.utils.test import get_crawler class _BaseTest(unittest.TestCase): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" @@ -146,12 +145,10 @@ class DefaultStorageTest(_BaseTest): class DbmStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): - dbm_module = "tests.mocks.dummydbm" def _get_settings(self, **new_settings): @@ -165,7 +162,6 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): class FilesystemStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" @@ -176,7 +172,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): class DummyPolicyTest(_BaseTest): - policy_class = "scrapy.extensions.httpcache.DummyPolicy" def test_middleware(self): @@ -270,7 +265,6 @@ class DummyPolicyTest(_BaseTest): class RFC2616PolicyTest(DefaultStorageTest): - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def _process_requestresponse(self, mw, request, response): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index ca125ba36..9ec51f75b 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,6 @@ spider = Spider("foo") class TestHttpProxyMiddleware(TestCase): - failureException = AssertionError def setUp(self): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 02854c2a7..63bd61848 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -127,7 +127,6 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): - invalid_url = "http://www.scrapytest.org/invalid_url" def get_spider_and_middleware(self, settings=None): diff --git a/tests/test_engine.py b/tests/test_engine.py index 02b59f448..410eba921 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,7 +243,6 @@ class CrawlerRun: class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_crawler(self): - for spider in ( TestSpider, DictItemsSpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8e0999348..1491e788e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -55,7 +55,6 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class = TestItem custom_field_item_class = CustomFieldItem @@ -513,13 +512,11 @@ class XmlItemExporterTest(BaseItemExporterTest): class XmlItemExporterDataclassTest(XmlItemExporterTest): - item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, @@ -559,13 +556,11 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): - item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class JsonItemExporterTest(JsonLinesItemExporterTest): - _expected_nested = [JsonLinesItemExporterTest._expected_nested] def _get_exporter(self, **kwargs): @@ -627,13 +622,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): class JsonItemExporterDataclassTest(JsonItemExporterTest): - item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class CustomExporterItemTest(unittest.TestCase): - item_class = TestItem def setUp(self): @@ -664,7 +657,6 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): - item_class = TestDataClass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 96f97ca99..eafe1b334 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1068,7 +1068,6 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1711,7 +1710,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin(self): - filename = self._named_tempfile("gzip_file") settings = { @@ -1731,7 +1729,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin_compresslevel(self): - filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( self.expected, compresslevel=0 @@ -1839,7 +1836,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin(self): - filename = self._named_tempfile("lzma_file") settings = { @@ -1859,7 +1855,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_format(self): - filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( self.expected, format=lzma.FORMAT_XZ @@ -1893,7 +1888,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_check(self): - filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( self.expected, check=lzma.CHECK_NONE @@ -1927,7 +1921,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_preset(self): - filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( self.expected, preset=0 @@ -1986,7 +1979,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_bz2_plugin(self): - filename = self._named_tempfile("bz2_file") settings = { @@ -2006,7 +1998,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_bz2_plugin_compresslevel(self): - filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( self.expected, compresslevel=1 @@ -2056,7 +2047,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin_with_parameter(self): - expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2075,7 +2065,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin_with_compression(self): - expected = b"foo\r\n\nbar\r\n\n" filename_to_decompressor = { @@ -2555,7 +2544,6 @@ class BatchDeliveriesTest(FeedExportTestBase): ] class CustomS3FeedStorage(S3FeedStorage): - stubs = [] def open(self, *args, **kwargs): @@ -2828,7 +2816,6 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): class URIParamsTest: - spider_name = "uri_params_spider" deprecated_options = False diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5a7413b32..2bd68f846 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -19,7 +19,6 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): - request_class = Request default_method = "GET" default_headers = {} @@ -424,7 +423,6 @@ class RequestTest(unittest.TestCase): class FormRequestTest(RequestTest): - request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): @@ -1447,7 +1445,6 @@ def _qs(req, encoding="utf-8", to_unicode=False): class XmlRpcRequestTest(RequestTest): - request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} diff --git a/tests/test_http_response.py b/tests/test_http_response.py index beb0b712b..dbc9f1fef 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -23,7 +23,6 @@ from tests import get_testdata class BaseResponseTest(unittest.TestCase): - response_class = Response def test_init(self): @@ -349,7 +348,6 @@ class BaseResponseTest(unittest.TestCase): class TextResponseTest(BaseResponseTest): - response_class = TextResponse def test_replace(self): @@ -835,11 +833,9 @@ class TextResponseTest(BaseResponseTest): class HtmlResponseTest(TextResponseTest): - response_class = HtmlResponse def test_html_encoding(self): - body = b"""Some page Price: \xa3100' @@ -878,7 +874,6 @@ class HtmlResponseTest(TextResponseTest): class XmlResponseTest(TextResponseTest): - response_class = XmlResponse def test_xml_encoding(self): diff --git a/tests/test_loader.py b/tests/test_loader.py index 5f4750ff3..0dc2de172 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -87,7 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class = None def test_keep_single_value(self): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 638af825b..3eae87599 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -469,7 +469,6 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationFromDictTest(unittest.TestCase): - item_class = dict def test_keep_single_value(self): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 0971a5a38..56810f2ff 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -16,7 +16,6 @@ from tests.spiders import ItemSpider class CustomItem(Item): - name = Field() def __str__(self): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 8f5d87ebf..cb9464949 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -116,7 +116,6 @@ class FileDownloadCrawlTestCase(TestCase): self.assertTrue((self.tmpmediastore / i["path"]).exists()) def _assert_files_download_failure(self, crawler, items, code, logs): - # check that the item does NOT have the "images/files" field populated self.assertEqual(len(items), 1) self.assertIn(self.media_key, items[0]) @@ -205,7 +204,6 @@ else: class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): - skip = skip_pillow pipeline_class = "scrapy.pipelines.images.ImagesPipeline" diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index cfdac1ab8..6fd8e6308 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -33,7 +33,6 @@ else: class ImagesPipelineTestCase(unittest.TestCase): - skip = skip_pillow def setUp(self): @@ -325,7 +324,6 @@ class DeprecatedImagesPipeline(ImagesPipeline): class ImagesPipelineTestCaseFieldsMixin: - skip = skip_pillow def test_item_fields_default(self): @@ -420,7 +418,6 @@ class ImagesPipelineTestCaseFieldsAttrsItem( class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): - skip = skip_pillow img_cls_attribute_names = [ diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 66a492217..d655eb128 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -37,7 +37,6 @@ def _mocked_download_func(request, info): class BaseMediaPipelineTestCase(unittest.TestCase): - pipeline_class = MediaPipeline settings = None @@ -213,7 +212,6 @@ class MockedMediaPipeline(MediaPipeline): class MediaPipelineTestCase(BaseMediaPipelineTestCase): - pipeline_class = MockedMediaPipeline def _callback(self, result): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 577522c6c..adce6cda7 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -154,7 +154,6 @@ class KeywordArgumentsSpider(MockServerSpider): class CallbackKeywordArgumentsTestCase(TestCase): - maxDiff = None def setUp(self): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 5acc412e5..bfb370373 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -43,7 +43,6 @@ class MockDownloader: class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict( SCHEDULER_DEBUG=False, SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", @@ -325,7 +324,6 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): @defer.inlineCallbacks def test_integration_downloader_aware_priority_queue(self): with MockServer() as mockserver: - url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 2a3b2d529..3e8187096 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -93,7 +93,6 @@ class BaseSettingsTest(unittest.TestCase): with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( attr, "set" ) as mock_set: - self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): diff --git a/tests/test_spider.py b/tests/test_spider.py index eb8a1f9f0..57c2b79cb 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -26,7 +26,6 @@ from tests import get_testdata class SpiderTest(unittest.TestCase): - spider_class = Spider def setUp(self): @@ -115,12 +114,10 @@ class SpiderTest(unittest.TestCase): class InitSpiderTest(SpiderTest): - spider_class = InitSpider class XMLFeedSpiderTest(SpiderTest): - spider_class = XMLFeedSpider def test_register_namespace(self): @@ -174,7 +171,6 @@ class XMLFeedSpiderTest(SpiderTest): class CSVFeedSpiderTest(SpiderTest): - spider_class = CSVFeedSpider def test_parse_rows(self): @@ -196,7 +192,6 @@ class CSVFeedSpiderTest(SpiderTest): class CrawlSpiderTest(SpiderTest): - test_body = b"""Page title<title> <body> <p><a href="item/12.html">Item 12</a></p> @@ -210,7 +205,6 @@ class CrawlSpiderTest(SpiderTest): spider_class = CrawlSpider def test_rule_without_link_extractor(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -234,7 +228,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_links(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -261,7 +254,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_links_filter(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -290,7 +282,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_links_generator(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -318,7 +309,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -347,7 +337,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request_with_response(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -383,7 +372,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request_instance_method(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -410,7 +398,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request_instance_method_with_response(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -467,7 +454,6 @@ class CrawlSpiderTest(SpiderTest): class SitemapSpiderTest(SpiderTest): - spider_class = SitemapSpider BODY = b"SITEMAP" @@ -689,7 +675,6 @@ class DeprecationTest(unittest.TestCase): class NoParseMethodSpiderTest(unittest.TestCase): - spider_class = Spider def test_undefined_parse_method(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index da656303d..04025d30d 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -114,13 +114,11 @@ class SpiderLoaderTest(unittest.TestCase): self.assertEqual(crawler.spidercls.name, "spider1") def test_bad_spider_modules_exception(self): - module = "tests.test_spiderloader.test_spiders.doesnotexist" settings = Settings({"SPIDER_MODULES": [module]}) self.assertRaises(ImportError, SpiderLoader.from_settings, settings) def test_bad_spider_modules_warning(self): - with warnings.catch_warnings(record=True) as w: module = "tests.test_spiderloader.test_spiders.doesnotexist" settings = Settings( diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 1bc5ccb9a..2f0dda269 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -31,7 +31,6 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta = {} resp_headers = {} settings = {} @@ -51,7 +50,6 @@ class TestRefererMiddleware(TestCase): return Response(origin, headers=self.resp_headers) def test(self): - for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) @@ -770,7 +768,6 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): class TestRequestMetaSettingFallback(TestCase): - params = [ ( # When an unknown policy is referenced in Request.meta @@ -824,7 +821,6 @@ class TestRequestMetaSettingFallback(TestCase): ] def test(self): - origin = "http://www.scrapy.org" target = "http://www.example.com" @@ -923,7 +919,6 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): - settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} scenarii = [ ( @@ -966,7 +961,6 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.redirectmw = RedirectMiddleware(settings) def test(self): - for ( parent, target, diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 1586f90c5..4ce7cc9a4 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -73,7 +73,6 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): - chunksize = 100000 def queue(self): diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 18f63b540..dabd6054d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -20,7 +20,6 @@ except ImportError: class UtilsConsoleTestCase(unittest.TestCase): def test_get_shell_embed_func(self): - shell = get_shell_embed_func(["invalid"]) self.assertEqual(shell, None) @@ -30,14 +29,12 @@ class UtilsConsoleTestCase(unittest.TestCase): @unittest.skipIf(not bpy, "bpython not available in testenv") def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(["bpython"]) self.assertTrue(callable(shell)) self.assertEqual(shell.__name__, "_embed_bpython_shell") @unittest.skipIf(not ipy, "IPython not available in testenv") def test_get_shell_embed_func3(self): - # default shell should be 'ipython' shell = get_shell_embed_func() self.assertEqual(shell.__name__, "_embed_ipython_shell") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ed077440c..bab6e9b68 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,7 +7,6 @@ from tests import get_testdata class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) def test_xmliter(self): diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 4760bf92b..6ca272de1 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -462,7 +462,6 @@ class BackwardCompatibilityTestCase(unittest.TestCase): warning to be logged.""" class RequestFingerprinter: - cache = WeakKeyDictionary() def fingerprint(self, request): @@ -641,7 +640,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): def test_from_crawler_and_settings(self): class RequestFingerprinter: - # This method is ignored due to the presence of from_crawler @classmethod def from_settings(cls, settings): diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index c79a1fdce..cbe80e157 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -16,7 +16,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): - context = dict(project_name="proj", name="spi", classname="TheSpider") template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" diff --git a/tox.ini b/tox.ini index 453c28c4c..77385968e 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,black,typing,py +envlist = pre-commit,pylint,typing,py minversion = 1.7.0 [testenv] @@ -37,44 +37,34 @@ install_command = [testenv:typing] basepython = python3 deps = - lxml-stubs==0.2.0 + lxml-stubs==0.4.0 mypy==0.991 types-attrs==19.1.0 - types-pyOpenSSL==21.0.0 - types-setuptools==57.0.0 + types-pyOpenSSL==23.0.0.2 + types-setuptools==65.7.0.3 commands = mypy --show-error-codes {posargs: scrapy tests} -[testenv:security] +[testenv:pre-commit] basepython = python3 deps = - bandit==1.7.4 + pre-commit commands = - bandit -r -c .bandit.yml {posargs:scrapy} - -[testenv:flake8] -basepython = python3 -deps = - {[testenv]deps} - # Twisted[http2] is required to import some files - Twisted[http2]>=17.9.0 - flake8==6.0.0 -commands = - flake8 {posargs:docs scrapy tests} + pre-commit run {posargs:--all-files} [testenv:pylint] basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.15.6 + pylint==2.16.0 commands = pylint conftest.py docs extras scrapy setup.py tests [testenv:twinecheck] basepython = python3 deps = - twine==4.0.1 - build==0.9.0 + twine==4.0.2 + build==0.10.0 commands = python -m build --sdist twine check dist/* @@ -195,9 +185,3 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:black] -deps = - black==22.12.0 -commands = - black {posargs:--check .}