diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5fc1285f..6840bfe23 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,14 +5,14 @@ repos: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 5.0.4 # 6.0.0 drops Python 3.7 support hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.11.5 # 5.12 drops Python 3.7 support hooks: - id: isort diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index efb4c202e..1f270837c 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -18,7 +18,6 @@ from pathlib import Path def main(): - # Used for remembering the file (and its contents) # so we don't have to open the same file again. _filename = None @@ -50,7 +49,6 @@ def main(): else: # If this is a new file if newfilename != _filename: - # Update the previous file if _filename: Path(_filename).write_text(_contents, encoding="utf-8") diff --git a/extras/qpsclient.py b/extras/qpsclient.py index acad71e07..119dfdabb 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -13,7 +13,6 @@ from scrapy.spiders import Spider class QPSSpider(Spider): - name = "qps" benchurl = "http://localhost:8880/" diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index de68c43a5..9baee3a48 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -14,7 +14,6 @@ from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: - requires_project = False crawler_process: Optional[CrawlerProcess] = None diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 911e5afe6..e1ccdc451 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -9,7 +9,6 @@ from scrapy.linkextractors import LinkExtractor class Command(ScrapyCommand): - default_settings = { "LOG_LEVEL": "INFO", "LOGSTATS_INTERVAL": 1, diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index df8006f36..2f0f1c7b9 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -3,7 +3,6 @@ from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommand): - requires_project = True def syntax(self): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index ca591011c..03a8ed5c7 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -6,7 +6,6 @@ from scrapy.exceptions import UsageError class Command(ScrapyCommand): - requires_project = True default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index a9076c5b1..1359e445f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -10,7 +10,6 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): - requires_project = False def syntax(self): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 90dd0874e..c1565a138 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -32,7 +32,6 @@ def extract_domain(url): class Command(ScrapyCommand): - requires_project = False default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2a81bd182..2f5032360 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -2,7 +2,6 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): - requires_project = True default_settings = {"LOG_ENABLED": False} diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 8a75f9270..58ed89a81 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -24,7 +24,6 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType: class Command(BaseRunSpiderCommand): - requires_project = False default_settings = {"SPIDER_LOADER_WARN_ONLY": True} diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index c43298ac7..318187204 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -5,7 +5,6 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): - requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 05c76d1eb..63c23d04c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -13,7 +13,6 @@ from scrapy.utils.url import guess_scheme class Command(ScrapyCommand): - requires_project = False default_settings = { "KEEP_ALIVE": True, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 88bd5bb33..fde609c6f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -28,7 +28,6 @@ def _make_writable(path): class Command(ScrapyCommand): - requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index faa0e242c..47582866b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -4,7 +4,6 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): - default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} def syntax(self): diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7e0b62bb0..7d27a5193 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -69,7 +69,6 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader: - DOWNLOAD_SLOT = "download_slot" def __init__(self, crawler): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8de5459e9..c3704de3d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -292,7 +292,6 @@ class ScrapyProxyAgent(Agent): class ScrapyAgent: - _Agent = Agent _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 5a94e66a6..56df48b2e 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -17,7 +17,6 @@ from scrapy.utils.defer import deferred_from_coro, mustbe_deferred class DownloaderMiddlewareManager(MiddlewareManager): - component_name = "downloader middleware" @classmethod diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 4558402b2..3d103652b 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -40,7 +40,6 @@ def _parse(url): class ScrapyHTTPPageGetter(HTTPClient): - delimiter = b"\n" def connectionMade(self): @@ -103,7 +102,6 @@ class ScrapyHTTPPageGetter(HTTPClient): # Twisted (https://github.com/twisted/twisted/pull/643), we merged its # non-overridden code into this class. class ScrapyHTTPClientFactory(ClientFactory): - protocol = ScrapyHTTPPageGetter waiting = 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index ba9c37e38..971cf52e4 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -46,7 +46,6 @@ def _isiterable(o) -> bool: class SpiderMiddlewareManager(MiddlewareManager): - component_name = "spider middleware" def __init__(self, *middlewares): diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 137ed5b18..04ae719de 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -30,7 +30,6 @@ class AjaxCrawlMiddleware: return cls(crawler.settings) def process_response(self, request, response, spider): - if not isinstance(response, HtmlResponse) or response.status != 200: return response diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 74c55f6e2..b9316c43a 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -27,7 +27,6 @@ HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddlew class HttpCacheMiddleware: - DOWNLOAD_EXCEPTIONS = ( defer.TimeoutError, TimeoutError, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 7b5f4824a..ead426951 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -53,7 +53,6 @@ class HttpCompressionMiddleware: request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): - if request.method == "HEAD": return response if isinstance(response, Response): diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index f442a3012..65f1d2224 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -26,7 +26,6 @@ def _build_redirect_request(source_request, *, url, **kwargs): class BaseRedirectMiddleware: - enabled_setting = "REDIRECT_ENABLED" def __init__(self, settings): @@ -115,7 +114,6 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): - enabled_setting = "METAREFRESH_ENABLED" def __init__(self, settings): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 11a30911c..081642a4b 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -122,7 +122,6 @@ def get_retry_request( class RetryMiddleware: - # IOError is raised by the HttpCompression middleware when trying to # decompress an empty response EXCEPTIONS_TO_RETRY = ( diff --git a/scrapy/extension.py b/scrapy/extension.py index 89f6a694d..4e365cfa1 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -8,7 +8,6 @@ from scrapy.utils.conf import build_component_list class ExtensionManager(MiddlewareManager): - component_name = "extension" @classmethod diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 2d120a6ed..dfe843974 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -41,7 +41,6 @@ class DummyPolicy: class RFC2616Policy: - MAXAGE = 3600 * 24 * 365 # one year def __init__(self, settings): diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 35e2808c2..510c903db 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -15,7 +15,6 @@ from scrapy.utils.deprecate import create_deprecated_class class JsonRequest(Request): - attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index e45d95602..73bb811de 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -29,7 +29,6 @@ _NONE = object() class TextResponse(Response): - _DEFAULT_ENCODING = "ascii" _cached_decoded_json = _NONE diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 66cf58d89..dd8dcdf7c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -194,7 +194,6 @@ class LxmlLinkExtractor: return True def matches(self, url): - if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): return False if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 536341fc6..df574a0a1 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -10,7 +10,6 @@ from scrapy.utils.defer import deferred_f_from_coro_f class ItemPipelineManager(MiddlewareManager): - component_name = "item pipeline" @classmethod diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 3430603ab..0be28933e 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -187,7 +187,6 @@ class S3FilesStore: class GCSFilesStore: - GCS_PROJECT_ID = None CACHE_CONTROL = "max-age=172800" @@ -253,7 +252,6 @@ class GCSFilesStore: class FTPFilesStore: - FTP_USERNAME = None FTP_PASSWORD = None USE_ACTIVE_MODE = None diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 31f51c004..401029439 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -23,7 +23,6 @@ def _DUMMY_CALLBACK(response): class MediaPipeline: - LOG_FAILED_RESULTS = True class SpiderInfo: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 6af8915c2..f01e9096c 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -12,7 +12,6 @@ from scrapy.utils.python import binary_is_text, to_bytes, to_unicode class ResponseTypes: - CLASSES = { "text/html": "scrapy.http.HtmlResponse", "application/atom+xml": "scrapy.http.XmlResponse", diff --git a/scrapy/shell.py b/scrapy/shell.py index ae6e641fd..bb3b1461c 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -25,7 +25,6 @@ from scrapy.utils.response import open_in_browser class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) def __init__(self, crawler, update_vars=None, code=None): diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d86f55a40..fd91e658b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -34,7 +34,6 @@ POLICY_SCRAPY_DEFAULT = "scrapy-default" class ReferrerPolicy: - NOREFERRER_SCHEMES: Tuple[str, ...] = LOCAL_SCHEMES name: str diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 05c425948..31e845716 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -60,7 +60,6 @@ class Rule: class CrawlSpider(Spider): - rules: Sequence[Rule] = () def __init__(self, *a, **kw): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 5ec0504a8..6afadc577 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -89,7 +89,7 @@ class XMLFeedSpider(Spider): yield node def _register_namespaces(self, selector): - for (prefix, uri) in self.namespaces: + for prefix, uri in self.namespaces: selector.register_namespace(prefix, uri) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c3cca9699..aaf75a519 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -10,7 +10,6 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): - sitemap_urls = () sitemap_rules = [("", "parse")] sitemap_follow = [""] diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 1089ba7b8..38884a9f0 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -6,7 +6,6 @@ from twisted.web.server import Site class Root(Resource): - isLeaf = True def getChild(self, name, request): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 10497e9ed..fa57a4f26 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -11,7 +11,6 @@ from collections.abc import Mapping class CaselessDict(dict): - __slots__ = () def __init__(self, seq=None): diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 61a4347ea..f4d6e0451 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -54,7 +54,6 @@ def create_deprecated_class( """ class DeprecatedClass(new_class.__class__): - deprecated_class = None warned_on_subclass = False diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 358f41679..414658944 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -9,7 +9,6 @@ from scrapy.http import Request, Response class ScrapyJSONEncoder(json.JSONEncoder): - DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index ecb2e31bf..5f9bdef37 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -5,7 +5,6 @@ from twisted.internet import defer, protocol class ProcessTest: - command = None prefix = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 0a27ccd6d..833aa3e20 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -144,7 +144,6 @@ def strip_url( origin_only=False, strip_fragment=True, ): - """Strip URL string from some of its components: - ``strip_credentials`` removes "user:password@" diff --git a/tests/mockserver.py b/tests/mockserver.py index 7991da9dc..d541f3f7c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -95,7 +95,6 @@ class BrokenDownloadResource(resource.Resource): class LeafResource(resource.Resource): - isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): diff --git a/tests/requirements.txt b/tests/requirements.txt index d9373dfa8..618949795 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,7 +2,7 @@ attrs pyftpdlib pytest -pytest-cov==3.0.0 +pytest-cov==4.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures diff --git a/tests/spiders.py b/tests/spiders.py index 2036e8cf3..6ff48f471 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -25,7 +25,6 @@ class MockServerSpider(Spider): class MetaSpider(MockServerSpider): - name = "meta" def __init__(self, *args, **kwargs): @@ -37,7 +36,6 @@ class MetaSpider(MockServerSpider): class FollowAllSpider(MetaSpider): - name = "follow" link_extractor = LinkExtractor() @@ -59,7 +57,6 @@ class FollowAllSpider(MetaSpider): class DelaySpider(MetaSpider): - name = "delay" def __init__(self, n=1, b=0, *args, **kwargs): @@ -81,7 +78,6 @@ class DelaySpider(MetaSpider): class SimpleSpider(MetaSpider): - name = "simple" def __init__(self, url="http://localhost:8998", *args, **kwargs): @@ -93,7 +89,6 @@ class SimpleSpider(MetaSpider): class AsyncDefSpider(SimpleSpider): - name = "asyncdef" async def parse(self, response): @@ -102,7 +97,6 @@ class AsyncDefSpider(SimpleSpider): class AsyncDefAsyncioSpider(SimpleSpider): - name = "asyncdef_asyncio" async def parse(self, response): @@ -112,7 +106,6 @@ class AsyncDefAsyncioSpider(SimpleSpider): class AsyncDefAsyncioReturnSpider(SimpleSpider): - name = "asyncdef_asyncio_return" async def parse(self, response): @@ -123,7 +116,6 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): - name = "asyncdef_asyncio_return_single_element" async def parse(self, response): @@ -134,7 +126,6 @@ class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): - name = "asyncdef_asyncio_reqs_return" async def parse(self, response): @@ -191,7 +182,6 @@ class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): class AsyncDefAsyncioGenSpider(SimpleSpider): - name = "asyncdef_asyncio_gen" async def parse(self, response): @@ -201,7 +191,6 @@ class AsyncDefAsyncioGenSpider(SimpleSpider): class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - name = "asyncdef_asyncio_gen_loop" async def parse(self, response): @@ -212,7 +201,6 @@ class AsyncDefAsyncioGenLoopSpider(SimpleSpider): class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - name = "asyncdef_asyncio_gen_complex" initial_reqs = 4 following_reqs = 3 @@ -246,7 +234,6 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): class ItemSpider(FollowAllSpider): - name = "item" def parse(self, response): @@ -261,7 +248,6 @@ class DefaultError(Exception): class ErrorSpider(FollowAllSpider): - name = "error" exception_cls = DefaultError @@ -275,7 +261,6 @@ class ErrorSpider(FollowAllSpider): class BrokenStartRequestsSpider(FollowAllSpider): - fail_before_yield = False fail_yielding = False @@ -305,7 +290,6 @@ class BrokenStartRequestsSpider(FollowAllSpider): class SingleRequestSpider(MetaSpider): - seed = None callback_func = None errback_func = None @@ -451,7 +435,6 @@ class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParse class BytesReceivedCallbackSpider(MetaSpider): - full_response_length = 2**18 @classmethod diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 1f299587f..129ef0121 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -2,7 +2,6 @@ from tests.test_commands import CommandTest class CheckCommandTest(CommandTest): - command = "check" def setUp(self): diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 124c968c2..d2027d1c2 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -6,7 +6,6 @@ from scrapy.utils.testsite import SiteTest class FetchTest(ProcessTest, SiteTest, unittest.TestCase): - command = "fetch" @defer.inlineCallbacks diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 8ce82db86..6589381f3 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -9,7 +9,6 @@ from tests import NON_EXISTING_RESOLVABLE, tests_datadir class ShellTest(ProcessTest, SiteTest, unittest.TestCase): - command = "shell" @defer.inlineCallbacks diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 3bf6019b5..a52d0d13c 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -8,7 +8,6 @@ from scrapy.utils.testproc import ProcessTest class VersionTest(ProcessTest, unittest.TestCase): - command = "version" @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index d7ee75559..00ddcdd3e 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -223,7 +223,6 @@ def get_permissions_dict( class StartprojectTemplatesTest(ProjectTest): - maxDiff = None def setUp(self): @@ -604,7 +603,6 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): - spider_filename = "myspider.py" debug_log_spider = """ @@ -873,7 +871,6 @@ class MySpider(scrapy.Spider): @skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): - spider_filename = "myspider.pyw" def setUp(self): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 8090d50b9..31fa1430d 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -25,7 +25,6 @@ from tests.test_downloader_handlers import ( @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): - scheme = "https" HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError" diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 2be32e37b..060cfe08b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -16,7 +16,6 @@ from scrapy.utils.test import get_crawler, get_from_asyncio_queue class ManagerTestCase(TestCase): - settings_dict = None def setUp(self): diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index 412c20a78..95739414e 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -17,7 +17,6 @@ def _test_data(formats): class DecompressionMiddlewareTest(TestCase): - test_formats = ["tar", "xml.bz2", "xml.gz", "zip"] uncompressed_body, test_responses = _test_data(test_formats) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a355a9b5b..f80eff3e6 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -14,7 +14,6 @@ from scrapy.utils.test import get_crawler class _BaseTest(unittest.TestCase): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" @@ -146,12 +145,10 @@ class DefaultStorageTest(_BaseTest): class DbmStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): - dbm_module = "tests.mocks.dummydbm" def _get_settings(self, **new_settings): @@ -165,7 +162,6 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): class FilesystemStorageTest(DefaultStorageTest): - storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" @@ -176,7 +172,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): class DummyPolicyTest(_BaseTest): - policy_class = "scrapy.extensions.httpcache.DummyPolicy" def test_middleware(self): @@ -270,7 +265,6 @@ class DummyPolicyTest(_BaseTest): class RFC2616PolicyTest(DefaultStorageTest): - policy_class = "scrapy.extensions.httpcache.RFC2616Policy" def _process_requestresponse(self, mw, request, response): diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index ca125ba36..9ec51f75b 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,6 @@ spider = Spider("foo") class TestHttpProxyMiddleware(TestCase): - failureException = AssertionError def setUp(self): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 02854c2a7..63bd61848 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -127,7 +127,6 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): - invalid_url = "http://www.scrapytest.org/invalid_url" def get_spider_and_middleware(self, settings=None): diff --git a/tests/test_engine.py b/tests/test_engine.py index 02b59f448..410eba921 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -243,7 +243,6 @@ class CrawlerRun: class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_crawler(self): - for spider in ( TestSpider, DictItemsSpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8e0999348..1491e788e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -55,7 +55,6 @@ class CustomFieldDataclass: class BaseItemExporterTest(unittest.TestCase): - item_class = TestItem custom_field_item_class = CustomFieldItem @@ -513,13 +512,11 @@ class XmlItemExporterTest(BaseItemExporterTest): class XmlItemExporterDataclassTest(XmlItemExporterTest): - item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = { "name": "Jesus", "age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}}, @@ -559,13 +556,11 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): - item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class JsonItemExporterTest(JsonLinesItemExporterTest): - _expected_nested = [JsonLinesItemExporterTest._expected_nested] def _get_exporter(self, **kwargs): @@ -627,13 +622,11 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): class JsonItemExporterDataclassTest(JsonItemExporterTest): - item_class = TestDataClass custom_field_item_class = CustomFieldDataclass class CustomExporterItemTest(unittest.TestCase): - item_class = TestItem def setUp(self): @@ -664,7 +657,6 @@ class CustomExporterItemTest(unittest.TestCase): class CustomExporterDataclassTest(CustomExporterItemTest): - item_class = TestDataClass diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 96f97ca99..eafe1b334 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1068,7 +1068,6 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), self.MyItem2({"hello": "world2", "foo": "bar2"}), @@ -1711,7 +1710,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin(self): - filename = self._named_tempfile("gzip_file") settings = { @@ -1731,7 +1729,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_gzip_plugin_compresslevel(self): - filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( self.expected, compresslevel=0 @@ -1839,7 +1836,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin(self): - filename = self._named_tempfile("lzma_file") settings = { @@ -1859,7 +1855,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_format(self): - filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( self.expected, format=lzma.FORMAT_XZ @@ -1893,7 +1888,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_check(self): - filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( self.expected, check=lzma.CHECK_NONE @@ -1927,7 +1921,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_preset(self): - filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( self.expected, preset=0 @@ -1986,7 +1979,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_bz2_plugin(self): - filename = self._named_tempfile("bz2_file") settings = { @@ -2006,7 +1998,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_bz2_plugin_compresslevel(self): - filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( self.expected, compresslevel=1 @@ -2056,7 +2047,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin_with_parameter(self): - expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2075,7 +2065,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_custom_plugin_with_compression(self): - expected = b"foo\r\n\nbar\r\n\n" filename_to_decompressor = { @@ -2555,7 +2544,6 @@ class BatchDeliveriesTest(FeedExportTestBase): ] class CustomS3FeedStorage(S3FeedStorage): - stubs = [] def open(self, *args, **kwargs): @@ -2828,7 +2816,6 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): class URIParamsTest: - spider_name = "uri_params_spider" deprecated_options = False diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5a7413b32..2bd68f846 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -19,7 +19,6 @@ from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): - request_class = Request default_method = "GET" default_headers = {} @@ -424,7 +423,6 @@ class RequestTest(unittest.TestCase): class FormRequestTest(RequestTest): - request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): @@ -1447,7 +1445,6 @@ def _qs(req, encoding="utf-8", to_unicode=False): class XmlRpcRequestTest(RequestTest): - request_class = XmlRpcRequest default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} diff --git a/tests/test_http_response.py b/tests/test_http_response.py index beb0b712b..dbc9f1fef 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -23,7 +23,6 @@ from tests import get_testdata class BaseResponseTest(unittest.TestCase): - response_class = Response def test_init(self): @@ -349,7 +348,6 @@ class BaseResponseTest(unittest.TestCase): class TextResponseTest(BaseResponseTest): - response_class = TextResponse def test_replace(self): @@ -835,11 +833,9 @@ class TextResponseTest(BaseResponseTest): class HtmlResponseTest(TextResponseTest): - response_class = HtmlResponse def test_html_encoding(self): - body = b"""Some page Price: \xa3100' @@ -878,7 +874,6 @@ class HtmlResponseTest(TextResponseTest): class XmlResponseTest(TextResponseTest): - response_class = XmlResponse def test_xml_encoding(self): diff --git a/tests/test_loader.py b/tests/test_loader.py index 5f4750ff3..0dc2de172 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -87,7 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationTestMixin: - item_class = None def test_keep_single_value(self): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 638af825b..3eae87599 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -469,7 +469,6 @@ class BasicItemLoaderTest(unittest.TestCase): class InitializationFromDictTest(unittest.TestCase): - item_class = dict def test_keep_single_value(self): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 0971a5a38..56810f2ff 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -16,7 +16,6 @@ from tests.spiders import ItemSpider class CustomItem(Item): - name = Field() def __str__(self): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 8f5d87ebf..cb9464949 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -116,7 +116,6 @@ class FileDownloadCrawlTestCase(TestCase): self.assertTrue((self.tmpmediastore / i["path"]).exists()) def _assert_files_download_failure(self, crawler, items, code, logs): - # check that the item does NOT have the "images/files" field populated self.assertEqual(len(items), 1) self.assertIn(self.media_key, items[0]) @@ -205,7 +204,6 @@ else: class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): - skip = skip_pillow pipeline_class = "scrapy.pipelines.images.ImagesPipeline" diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index cfdac1ab8..6fd8e6308 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -33,7 +33,6 @@ else: class ImagesPipelineTestCase(unittest.TestCase): - skip = skip_pillow def setUp(self): @@ -325,7 +324,6 @@ class DeprecatedImagesPipeline(ImagesPipeline): class ImagesPipelineTestCaseFieldsMixin: - skip = skip_pillow def test_item_fields_default(self): @@ -420,7 +418,6 @@ class ImagesPipelineTestCaseFieldsAttrsItem( class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): - skip = skip_pillow img_cls_attribute_names = [ diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 66a492217..d655eb128 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -37,7 +37,6 @@ def _mocked_download_func(request, info): class BaseMediaPipelineTestCase(unittest.TestCase): - pipeline_class = MediaPipeline settings = None @@ -213,7 +212,6 @@ class MockedMediaPipeline(MediaPipeline): class MediaPipelineTestCase(BaseMediaPipelineTestCase): - pipeline_class = MockedMediaPipeline def _callback(self, result): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 577522c6c..adce6cda7 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -154,7 +154,6 @@ class KeywordArgumentsSpider(MockServerSpider): class CallbackKeywordArgumentsTestCase(TestCase): - maxDiff = None def setUp(self): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 5acc412e5..bfb370373 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -43,7 +43,6 @@ class MockDownloader: class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict( SCHEDULER_DEBUG=False, SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", @@ -325,7 +324,6 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): @defer.inlineCallbacks def test_integration_downloader_aware_priority_queue(self): with MockServer() as mockserver: - url = mockserver.url("/status?n=200", is_secure=False) start_urls = [url] * 6 yield self.crawler.crawl(start_urls) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 2a3b2d529..3e8187096 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -93,7 +93,6 @@ class BaseSettingsTest(unittest.TestCase): with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object( attr, "set" ) as mock_set: - self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): diff --git a/tests/test_spider.py b/tests/test_spider.py index eb8a1f9f0..57c2b79cb 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -26,7 +26,6 @@ from tests import get_testdata class SpiderTest(unittest.TestCase): - spider_class = Spider def setUp(self): @@ -115,12 +114,10 @@ class SpiderTest(unittest.TestCase): class InitSpiderTest(SpiderTest): - spider_class = InitSpider class XMLFeedSpiderTest(SpiderTest): - spider_class = XMLFeedSpider def test_register_namespace(self): @@ -174,7 +171,6 @@ class XMLFeedSpiderTest(SpiderTest): class CSVFeedSpiderTest(SpiderTest): - spider_class = CSVFeedSpider def test_parse_rows(self): @@ -196,7 +192,6 @@ class CSVFeedSpiderTest(SpiderTest): class CrawlSpiderTest(SpiderTest): - test_body = b"""Page title<title> <body> <p><a href="item/12.html">Item 12</a></p> @@ -210,7 +205,6 @@ class CrawlSpiderTest(SpiderTest): spider_class = CrawlSpider def test_rule_without_link_extractor(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -234,7 +228,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_links(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -261,7 +254,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_links_filter(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -290,7 +282,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_links_generator(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -318,7 +309,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -347,7 +337,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request_with_response(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -383,7 +372,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request_instance_method(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -410,7 +398,6 @@ class CrawlSpiderTest(SpiderTest): ) def test_process_request_instance_method_with_response(self): - response = HtmlResponse( "http://example.org/somepage/index.html", body=self.test_body ) @@ -467,7 +454,6 @@ class CrawlSpiderTest(SpiderTest): class SitemapSpiderTest(SpiderTest): - spider_class = SitemapSpider BODY = b"SITEMAP" @@ -689,7 +675,6 @@ class DeprecationTest(unittest.TestCase): class NoParseMethodSpiderTest(unittest.TestCase): - spider_class = Spider def test_undefined_parse_method(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index da656303d..04025d30d 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -114,13 +114,11 @@ class SpiderLoaderTest(unittest.TestCase): self.assertEqual(crawler.spidercls.name, "spider1") def test_bad_spider_modules_exception(self): - module = "tests.test_spiderloader.test_spiders.doesnotexist" settings = Settings({"SPIDER_MODULES": [module]}) self.assertRaises(ImportError, SpiderLoader.from_settings, settings) def test_bad_spider_modules_warning(self): - with warnings.catch_warnings(record=True) as w: module = "tests.test_spiderloader.test_spiders.doesnotexist" settings = Settings( diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 1bc5ccb9a..2f0dda269 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -31,7 +31,6 @@ from scrapy.spiders import Spider class TestRefererMiddleware(TestCase): - req_meta = {} resp_headers = {} settings = {} @@ -51,7 +50,6 @@ class TestRefererMiddleware(TestCase): return Response(origin, headers=self.resp_headers) def test(self): - for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) @@ -770,7 +768,6 @@ class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware): class TestRequestMetaSettingFallback(TestCase): - params = [ ( # When an unknown policy is referenced in Request.meta @@ -824,7 +821,6 @@ class TestRequestMetaSettingFallback(TestCase): ] def test(self): - origin = "http://www.scrapy.org" target = "http://www.example.com" @@ -923,7 +919,6 @@ class TestPolicyHeaderPrecedence004( class TestReferrerOnRedirect(TestRefererMiddleware): - settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"} scenarii = [ ( @@ -966,7 +961,6 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.redirectmw = RedirectMiddleware(settings) def test(self): - for ( parent, target, diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 1586f90c5..4ce7cc9a4 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -73,7 +73,6 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): - chunksize = 100000 def queue(self): diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 18f63b540..dabd6054d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -20,7 +20,6 @@ except ImportError: class UtilsConsoleTestCase(unittest.TestCase): def test_get_shell_embed_func(self): - shell = get_shell_embed_func(["invalid"]) self.assertEqual(shell, None) @@ -30,14 +29,12 @@ class UtilsConsoleTestCase(unittest.TestCase): @unittest.skipIf(not bpy, "bpython not available in testenv") def test_get_shell_embed_func2(self): - shell = get_shell_embed_func(["bpython"]) self.assertTrue(callable(shell)) self.assertEqual(shell.__name__, "_embed_bpython_shell") @unittest.skipIf(not ipy, "IPython not available in testenv") def test_get_shell_embed_func3(self): - # default shell should be 'ipython' shell = get_shell_embed_func() self.assertEqual(shell.__name__, "_embed_ipython_shell") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ed077440c..bab6e9b68 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,7 +7,6 @@ from tests import get_testdata class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) def test_xmliter(self): diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 4760bf92b..6ca272de1 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -462,7 +462,6 @@ class BackwardCompatibilityTestCase(unittest.TestCase): warning to be logged.""" class RequestFingerprinter: - cache = WeakKeyDictionary() def fingerprint(self, request): @@ -641,7 +640,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): def test_from_crawler_and_settings(self): class RequestFingerprinter: - # This method is ignored due to the presence of from_crawler @classmethod def from_settings(cls, settings): diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index c79a1fdce..cbe80e157 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -16,7 +16,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): - context = dict(project_name="proj", name="spi", classname="TheSpider") template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" diff --git a/tox.ini b/tox.ini index 453c28c4c..7385e4468 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = security,flake8,black,typing,py +envlist = pre-commit,typing,py minversion = 1.7.0 [testenv] @@ -37,44 +37,34 @@ install_command = [testenv:typing] basepython = python3 deps = - lxml-stubs==0.2.0 + lxml-stubs==0.4.0 mypy==0.991 types-attrs==19.1.0 - types-pyOpenSSL==21.0.0 - types-setuptools==57.0.0 + types-pyOpenSSL==23.0.0.2 + types-setuptools==65.7.0.3 commands = mypy --show-error-codes {posargs: scrapy tests} -[testenv:security] +[testenv:pre-commit] basepython = python3 deps = - bandit==1.7.4 + pre-commit commands = - bandit -r -c .bandit.yml {posargs:scrapy} - -[testenv:flake8] -basepython = python3 -deps = - {[testenv]deps} - # Twisted[http2] is required to import some files - Twisted[http2]>=17.9.0 - flake8==6.0.0 -commands = - flake8 {posargs:docs scrapy tests} + pre-commit run {posargs:--all-files} [testenv:pylint] basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.15.6 + pylint==2.16.0 commands = pylint conftest.py docs extras scrapy setup.py tests [testenv:twinecheck] basepython = python3 deps = - twine==4.0.1 - build==0.9.0 + twine==4.0.2 + build==0.10.0 commands = python -m build --sdist twine check dist/* @@ -195,9 +185,3 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:black] -deps = - black==22.12.0 -commands = - black {posargs:--check .}