Document scrapy-lint, remove start_url check (#7627)

* Document scrapy-lint, remove start_url check

* Remove the offsite URL check in favor of scrapy-lint

.pre-commit-config.yaml

@@ -27,6 +27,6 @@ repos:
   hooks:
     - id: sphinx-lint
 - repo: https://github.com/scrapy/sphinx-scrapy
-  rev: 0.8.6
+  rev: 0.8.8
   hooks:
     - id: sphinx-scrapy

docs/conf.py

@@ -158,6 +158,7 @@ scrapy_intersphinx_enable = [
     "itemloaders",
     "parsel",
     "pytest",
+    "scrapy-lint",
     "sphinx",
     "tox",
     "twisted",

docs/requirements.in

@@ -5,4 +5,4 @@ sphinx
 sphinx-notfound-page
 sphinx-rtd-theme
 sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8

docs/requirements.txt

@@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
     # via
     #   -r docs/requirements.in
     #   sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
     # via -r docs/requirements.in
 sphinx-sitemap==2.9.0
     # via sphinx-scrapy

docs/topics/practices.rst

@@ -440,6 +440,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 If you are still unable to prevent your bot getting banned, consider contacting
 `commercial support`_.
 
+.. _static-analysis:
+
+Static analysis
+===============
+
+Consider using :doc:`scrapy-lint <scrapy-lint:index>`, a linter for Scrapy
+projects that detects common mistakes and anti-patterns.
+
 .. _Tor project: https://www.torproject.org/
 .. _commercial support: https://www.scrapy.org/companies
 .. _ProxyMesh: https://proxymesh.com/

scrapy/downloadermiddlewares/offsite.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 import re
-import warnings
 from typing import TYPE_CHECKING
 
 from scrapy import Request, Spider, signals
@@ -75,25 +74,10 @@ class OffsiteMiddleware:
         allowed_domains = getattr(spider, "allowed_domains", None)
         if not allowed_domains:
             return re.compile("")  # allow all by default
-        url_pattern = re.compile(r"^https?://.*$")
-        port_pattern = re.compile(r":\d+$")
         domains = []
         for domain in allowed_domains:
             if domain is None:
                 continue
-            if url_pattern.match(domain):
+            domains.append(re.escape(domain))
-                message = (
-                    "allowed_domains accepts only domains, not URLs. "
-                    f"Ignoring URL entry {domain} in allowed_domains."
-                )
-                warnings.warn(message, stacklevel=2)
-            elif port_pattern.search(domain):
-                message = (
-                    "allowed_domains accepts only domains without ports. "
-                    f"Ignoring entry {domain} in allowed_domains."
-                )
-                warnings.warn(message, stacklevel=2)
-            else:
-                domains.append(re.escape(domain))
         regex = rf"^(.*\.)?({'|'.join(domains)})$"
         return re.compile(regex)

scrapy/spiders/__init__.py

@@ -125,12 +125,6 @@ class Spider(object_ref):
 
         .. seealso:: :ref:`start-requests`
         """
-        if not self.start_urls and hasattr(self, "start_url"):
-            raise AttributeError(
-                "Crawling could not start: 'start_urls' not found "
-                "or empty (but found 'start_url' attribute instead, "
-                "did you miss an 's'?)"
-            )
         for url in self.start_urls:
             yield Request(url, dont_filter=True)
 

tests/test_downloadermiddleware_offsite.py

@@ -1,5 +1,3 @@
-import warnings
-
 import pytest
 
 from scrapy import Request, Spider
@@ -120,9 +118,7 @@ def test_process_request_invalid_domains():
     allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with warnings.catch_warnings():
+    mw.spider_opened(crawler.spider)
-        warnings.simplefilter("ignore", UserWarning)
-        mw.spider_opened(crawler.spider)
     request = Request("https://a.example")
     assert mw.process_request(request) is None
     for letter in ("b", "c"):
@@ -210,9 +206,7 @@ def test_request_scheduled_invalid_domains():
     allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with warnings.catch_warnings():
+    mw.spider_opened(crawler.spider)
-        warnings.simplefilter("ignore", UserWarning)
-        mw.spider_opened(crawler.spider)
     request = Request("https://a.example")
     assert mw.request_scheduled(request, crawler.spider) is None
     for letter in ("b", "c"):

tests/test_spider_crawl.py

@@ -2,10 +2,8 @@ from __future__ import annotations
 
 import re
 import warnings
-from logging import ERROR
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import safe_url_string
 
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -14,7 +12,6 @@ from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule, Spider
 from scrapy.utils.test import get_crawler
 from tests.test_spider import TestSpider
-from tests.utils.decorators import inline_callbacks_test
 
 
 class TestCrawlSpider(TestSpider):
@@ -247,18 +244,6 @@ class TestCrawlSpider(TestSpider):
         assert hasattr(spider, "_follow_links")
         assert not spider._follow_links
 
-    @inline_callbacks_test
-    def test_start_url(self):
-        class TestSpider(self.spider_class):
-            name = "test"
-            start_url = "https://www.example.com"
-
-        crawler = get_crawler(TestSpider)
-        with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log:
-            yield crawler.crawl()
-        assert "Error while reading start items and requests" in str(log)
-        assert "did you miss an 's'?" in str(log)
-
     def test_parse_response_use(self):
         class _CrawlSpider(CrawlSpider):
             name = "test"

tox.ini

@@ -5,7 +5,7 @@
 
 [tox]
 requires =
-    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6
+    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
 envlist =
     pre-commit
     pylint