From f605defefc5cb8e7a969fba7da66c2087868c104 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 22 Jun 2026 17:12:16 +0200 Subject: [PATCH] Document scrapy-lint, remove start_url check (#7627) * Document scrapy-lint, remove start_url check * Remove the offsite URL check in favor of scrapy-lint --- .pre-commit-config.yaml | 2 +- docs/conf.py | 1 + docs/requirements.in | 2 +- docs/requirements.txt | 2 +- docs/topics/practices.rst | 8 ++++++++ scrapy/downloadermiddlewares/offsite.py | 18 +----------------- scrapy/spiders/__init__.py | 6 ------ tests/test_downloadermiddleware_offsite.py | 10 ++-------- tests/test_spider_crawl.py | 15 --------------- tox.ini | 2 +- 10 files changed, 16 insertions(+), 50 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b9ef3c04..754bd6f9c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.6 + rev: 0.8.8 hooks: - id: sphinx-scrapy diff --git a/docs/conf.py b/docs/conf.py index 99d5df7da..b950c4ee2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -158,6 +158,7 @@ scrapy_intersphinx_enable = [ "itemloaders", "parsel", "pytest", + "scrapy-lint", "sphinx", "tox", "twisted", diff --git a/docs/requirements.in b/docs/requirements.in index 140791641..a1f3a7468 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index 9c93dacd0..a5cbad302 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b3c58d6d3..8a04f7ada 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -440,6 +440,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites: If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. +.. _static-analysis: + +Static analysis +=============== + +Consider using :doc:`scrapy-lint `, a linter for Scrapy +projects that detects common mistakes and anti-patterns. + .. _Tor project: https://www.torproject.org/ .. _commercial support: https://www.scrapy.org/companies .. _ProxyMesh: https://proxymesh.com/ diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 10f19bacc..e03dc040e 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging import re -import warnings from typing import TYPE_CHECKING from scrapy import Request, Spider, signals @@ -75,25 +74,10 @@ class OffsiteMiddleware: allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: return re.compile("") # allow all by default - url_pattern = re.compile(r"^https?://.*$") - port_pattern = re.compile(r":\d+$") domains = [] for domain in allowed_domains: if domain is None: continue - if url_pattern.match(domain): - message = ( - "allowed_domains accepts only domains, not URLs. " - f"Ignoring URL entry {domain} in allowed_domains." - ) - warnings.warn(message, stacklevel=2) - elif port_pattern.search(domain): - message = ( - "allowed_domains accepts only domains without ports. " - f"Ignoring entry {domain} in allowed_domains." - ) - warnings.warn(message, stacklevel=2) - else: - domains.append(re.escape(domain)) + domains.append(re.escape(domain)) regex = rf"^(.*\.)?({'|'.join(domains)})$" return re.compile(regex) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 6e8c67adb..de527d04f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -125,12 +125,6 @@ class Spider(object_ref): .. seealso:: :ref:`start-requests` """ - if not self.start_urls and hasattr(self, "start_url"): - raise AttributeError( - "Crawling could not start: 'start_urls' not found " - "or empty (but found 'start_url' attribute instead, " - "did you miss an 's'?)" - ) for url in self.start_urls: yield Request(url, dont_filter=True) diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index dc0a31a76..edfb15d10 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,5 +1,3 @@ -import warnings - import pytest from scrapy import Request, Spider @@ -120,9 +118,7 @@ def test_process_request_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - mw.spider_opened(crawler.spider) + mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.process_request(request) is None for letter in ("b", "c"): @@ -210,9 +206,7 @@ def test_request_scheduled_invalid_domains(): allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) mw = OffsiteMiddleware.from_crawler(crawler) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - mw.spider_opened(crawler.spider) + mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.request_scheduled(request, crawler.spider) is None for letter in ("b", "c"): diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 2eeae110b..63010e195 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -2,10 +2,8 @@ from __future__ import annotations import re import warnings -from logging import ERROR import pytest -from testfixtures import LogCapture from w3lib.url import safe_url_string from scrapy.exceptions import ScrapyDeprecationWarning @@ -14,7 +12,6 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider from scrapy.utils.test import get_crawler from tests.test_spider import TestSpider -from tests.utils.decorators import inline_callbacks_test class TestCrawlSpider(TestSpider): @@ -247,18 +244,6 @@ class TestCrawlSpider(TestSpider): assert hasattr(spider, "_follow_links") assert not spider._follow_links - @inline_callbacks_test - def test_start_url(self): - class TestSpider(self.spider_class): - name = "test" - start_url = "https://www.example.com" - - crawler = get_crawler(TestSpider) - with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log: - yield crawler.crawl() - assert "Error while reading start items and requests" in str(log) - assert "did you miss an 's'?" in str(log) - def test_parse_response_use(self): class _CrawlSpider(CrawlSpider): name = "test" diff --git a/tox.ini b/tox.ini index 44e240e03..2fa7b455d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 envlist = pre-commit pylint