mirror of https://github.com/scrapy/scrapy.git
Document scrapy-lint, remove start_url check (#7627)
* Document scrapy-lint, remove start_url check * Remove the offsite URL check in favor of scrapy-lint
This commit is contained in:
Adrian
GitHub
parent
7499d17e28
commit
f605defefc
|
|
@ -27,6 +27,6 @@ repos:
|
|||
hooks:
|
||||
- id: sphinx-lint
|
||||
- repo: https://github.com/scrapy/sphinx-scrapy
|
||||
rev: 0.8.6
|
||||
rev: 0.8.8
|
||||
hooks:
|
||||
- id: sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -158,6 +158,7 @@ scrapy_intersphinx_enable = [
|
|||
"itemloaders",
|
||||
"parsel",
|
||||
"pytest",
|
||||
"scrapy-lint",
|
||||
"sphinx",
|
||||
"tox",
|
||||
"twisted",
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ sphinx
|
|||
sphinx-notfound-page
|
||||
sphinx-rtd-theme
|
||||
sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
|
|||
# via
|
||||
# -r docs/requirements.in
|
||||
# sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
|
||||
# via -r docs/requirements.in
|
||||
sphinx-sitemap==2.9.0
|
||||
# via sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -440,6 +440,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
|||
If you are still unable to prevent your bot getting banned, consider contacting
|
||||
`commercial support`_.
|
||||
|
||||
.. _static-analysis:
|
||||
|
||||
Static analysis
|
||||
===============
|
||||
|
||||
Consider using :doc:`scrapy-lint <scrapy-lint:index>`, a linter for Scrapy
|
||||
projects that detects common mistakes and anti-patterns.
|
||||
|
||||
.. _Tor project: https://www.torproject.org/
|
||||
.. _commercial support: https://www.scrapy.org/companies
|
||||
.. _ProxyMesh: https://proxymesh.com/
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
|
|
@ -75,25 +74,10 @@ class OffsiteMiddleware:
|
|||
allowed_domains = getattr(spider, "allowed_domains", None)
|
||||
if not allowed_domains:
|
||||
return re.compile("") # allow all by default
|
||||
url_pattern = re.compile(r"^https?://.*$")
|
||||
port_pattern = re.compile(r":\d+$")
|
||||
domains = []
|
||||
for domain in allowed_domains:
|
||||
if domain is None:
|
||||
continue
|
||||
if url_pattern.match(domain):
|
||||
message = (
|
||||
"allowed_domains accepts only domains, not URLs. "
|
||||
f"Ignoring URL entry {domain} in allowed_domains."
|
||||
)
|
||||
warnings.warn(message, stacklevel=2)
|
||||
elif port_pattern.search(domain):
|
||||
message = (
|
||||
"allowed_domains accepts only domains without ports. "
|
||||
f"Ignoring entry {domain} in allowed_domains."
|
||||
)
|
||||
warnings.warn(message, stacklevel=2)
|
||||
else:
|
||||
domains.append(re.escape(domain))
|
||||
domains.append(re.escape(domain))
|
||||
regex = rf"^(.*\.)?({'|'.join(domains)})$"
|
||||
return re.compile(regex)
|
||||
|
|
|
|||
|
|
@ -125,12 +125,6 @@ class Spider(object_ref):
|
|||
|
||||
.. seealso:: :ref:`start-requests`
|
||||
"""
|
||||
if not self.start_urls and hasattr(self, "start_url"):
|
||||
raise AttributeError(
|
||||
"Crawling could not start: 'start_urls' not found "
|
||||
"or empty (but found 'start_url' attribute instead, "
|
||||
"did you miss an 's'?)"
|
||||
)
|
||||
for url in self.start_urls:
|
||||
yield Request(url, dont_filter=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
import warnings
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy import Request, Spider
|
||||
|
|
@ -120,9 +118,7 @@ def test_process_request_invalid_domains():
|
|||
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
|
||||
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
|
||||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", UserWarning)
|
||||
mw.spider_opened(crawler.spider)
|
||||
mw.spider_opened(crawler.spider)
|
||||
request = Request("https://a.example")
|
||||
assert mw.process_request(request) is None
|
||||
for letter in ("b", "c"):
|
||||
|
|
@ -210,9 +206,7 @@ def test_request_scheduled_invalid_domains():
|
|||
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
|
||||
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
|
||||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", UserWarning)
|
||||
mw.spider_opened(crawler.spider)
|
||||
mw.spider_opened(crawler.spider)
|
||||
request = Request("https://a.example")
|
||||
assert mw.request_scheduled(request, crawler.spider) is None
|
||||
for letter in ("b", "c"):
|
||||
|
|
|
|||
|
|
@ -2,10 +2,8 @@ from __future__ import annotations
|
|||
|
||||
import re
|
||||
import warnings
|
||||
from logging import ERROR
|
||||
|
||||
import pytest
|
||||
from testfixtures import LogCapture
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
|
@ -14,7 +12,6 @@ from scrapy.linkextractors import LinkExtractor
|
|||
from scrapy.spiders import CrawlSpider, Rule, Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.test_spider import TestSpider
|
||||
from tests.utils.decorators import inline_callbacks_test
|
||||
|
||||
|
||||
class TestCrawlSpider(TestSpider):
|
||||
|
|
@ -247,18 +244,6 @@ class TestCrawlSpider(TestSpider):
|
|||
assert hasattr(spider, "_follow_links")
|
||||
assert not spider._follow_links
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_start_url(self):
|
||||
class TestSpider(self.spider_class):
|
||||
name = "test"
|
||||
start_url = "https://www.example.com"
|
||||
|
||||
crawler = get_crawler(TestSpider)
|
||||
with LogCapture("scrapy.core.engine", propagate=False, level=ERROR) as log:
|
||||
yield crawler.crawl()
|
||||
assert "Error while reading start items and requests" in str(log)
|
||||
assert "did you miss an 's'?" in str(log)
|
||||
|
||||
def test_parse_response_use(self):
|
||||
class _CrawlSpider(CrawlSpider):
|
||||
name = "test"
|
||||
|
|
|
|||
Loading…
Reference in New Issue