change crawler_process to optional

This commit is contained in:
Matthew Donoughe 2022-10-21 20:32:13 -04:00
parent 28396c3497
commit 726680c712
No known key found for this signature in database
GPG Key ID: 838812402CA8C19D
2 changed files with 7 additions and 2 deletions

View File

@ -4,9 +4,10 @@ Base class for Scrapy commands
import os
import argparse
from pathlib import Path
from typing import Any, Dict
from typing import Any, Dict, Optional
from twisted.python import failure
from scrapy.crawler import CrawlerProcess
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.exceptions import UsageError
@ -15,7 +16,7 @@ from scrapy.exceptions import UsageError
class ScrapyCommand:
requires_project = False
crawler_process: Any = None
crawler_process: Optional[CrawlerProcess] = None
# default settings to be used for this command instead of global defaults
default_settings: Dict[str, Any] = {}

View File

@ -133,6 +133,10 @@ class Command(ScrapyCommand):
return True
return False
assert (
self.crawler_process is not None
), "crawler_process must be set before calling run"
try:
spidercls = self.crawler_process.spider_loader.load(name)
except KeyError: