RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'scrapy/master' into issue_960_s3_feed_export

This commit is contained in:
Adrián Chaves 2023-03-15 14:07:54 +01:00
parent 45b9dbae40 d60b4edd11
commit dfbb63a2f1
40 changed files with 525 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.coveragerc
View File

@ -3,3 +3,4 @@ branch = true
include = scrapy/*
omit =
tests/*
disable_warnings = include-ignored

2
.github/workflows/checks.yml vendored
View File

@ -11,7 +11,7 @@ jobs:
- python-version: "3.11"
env:
TOXENV: pylint
- python-version: 3.7
- python-version: 3.8
env:
TOXENV: typing
- python-version: "3.11" # Keep in sync with .readthedocs.yml

3
docs/topics/commands.rst
View File

@ -238,9 +238,6 @@ genspider
Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
.. note:: Even if an HTTPS URL is specified, the protocol used in
``start_urls`` is always HTTP. This is a known issue: :issue:`3553`.
Usage example::
$ scrapy genspider -l

27
docs/topics/settings.rst
View File

@ -776,6 +776,31 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
.. setting:: DOWNLOAD_SLOTS
DOWNLOAD_SLOTS
----------------
Default: ``{}``
Allows to define concurrency/delay parameters on per slot(domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
"quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
.. note::
For other downloader slots default settings values will be used:
- :setting:`DOWNLOAD_DELAY`: ``delay``
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
@ -1090,7 +1115,7 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
:ref:`Python logging documentation <logrecord-attributes>` for the whole
list of available placeholders.
.. setting:: LOG_DATEFORMAT

16
scrapy/commands/genspider.py
View File

@ -31,6 +31,14 @@ def extract_domain(url):
return o.netloc
def verify_url_scheme(url):
"""Check url for scheme and insert https if none found."""
parsed = urlparse(url)
if parsed.scheme == "" and parsed.netloc == "":
parsed = urlparse("//" + url)._replace(scheme="https")
return parsed.geturl()
class Command(ScrapyCommand):
requires_project = False
default_settings = {"LOG_ENABLED": False}
@ -91,7 +99,7 @@ class Command(ScrapyCommand):
raise UsageError()
name, url = args[0:2]
domain = extract_domain(url)
url = verify_url_scheme(url)
module = sanitize_module_name(name)
if self.settings.get("BOT_NAME") == module:
@ -103,18 +111,20 @@ class Command(ScrapyCommand):
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, domain, opts.template, template_file)
self._genspider(module, name, url, opts.template, template_file)
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"')
def _genspider(self, module, name, domain, template_name, template_file):
def _genspider(self, module, name, url, template_name, template_file):
"""Generate the spider module, based on the given template"""
capitalized_module = "".join(s.capitalize() for s in module.split("_"))
domain = extract_domain(url)
tvars = {
"project_name": self.settings.get("BOT_NAME"),
"ProjectName": string_camelcase(self.settings.get("BOT_NAME")),
"module": module,
"name": name,
"url": url,
"domain": domain,
"classname": f"{capitalized_module}Spider",
}

28
scrapy/commands/parse.py
View File

@ -1,3 +1,4 @@
import inspect
import json
import logging
from typing import Dict
@ -10,7 +11,11 @@ from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import aiter_errback, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.spider import spidercls_for_request
logger = logging.getLogger(__name__)
@ -108,6 +113,25 @@ class Command(BaseRunSpiderCommand):
max_requests = max(self.requests)
return max(max_items, max_requests)
def handle_exception(self, _failure):
logger.error(
"An error is caught while iterating the async iterable",
exc_info=failure_to_exc_info(_failure),
)
def iterate_spider_output(self, result):
if inspect.isasyncgen(result):
d = deferred_from_coro(
collect_asyncgen(aiter_errback(result, self.handle_exception))
)
d.addCallback(self.iterate_spider_output)
return d
if inspect.iscoroutine(result):
d = deferred_from_coro(result)
d.addCallback(self.iterate_spider_output)
return d
return arg_to_iter(deferred_from_coro(result))
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
self.items[lvl] = old_items + new_items
@ -165,7 +189,7 @@ class Command(BaseRunSpiderCommand):
def run_callback(self, response, callback, cb_kwargs=None):
cb_kwargs = cb_kwargs or {}
d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
return d
def get_callback_from_rules(self, spider, response):

12
scrapy/core/downloader/__init__.py
View File

@ -84,6 +84,7 @@ class Downloader:
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
self._slot_gc_loop = task.LoopingCall(self._slot_gc)
self._slot_gc_loop.start(60)
self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})
def fetch(self, request, spider):
def _deactivate(response):
@ -94,17 +95,24 @@ class Downloader:
dfd = self.middleware.download(self._enqueue_request, request, spider)
return dfd.addBoth(_deactivate)
def needs_backout(self):
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
def _get_slot(self, request, spider):
key = self._get_slot_key(request, spider)
if key not in self.slots:
slot_settings = self.per_slot_settings.get(key, {})
conc = (
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
)
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
self.slots[key] = Slot(conc, delay, self.randomize_delay)
conc, delay = (
slot_settings.get("concurrency", conc),
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot
return key, self.slots[key]

31
scrapy/core/engine.py
View File

@ -7,13 +7,24 @@ For more information see docs/topics/architecture.rst
import logging
import warnings
from time import time
from typing import Callable, Iterable, Iterator, Optional, Set, Union
from typing import (
Any,
Callable,
Generator,
Iterable,
Iterator,
Optional,
Set,
Union,
cast,
)
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from twisted.internet.task import LoopingCall
from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import CloseSpider, DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Request, Response
@ -75,7 +86,7 @@ class ExecutionEngine:
self.paused = False
self.scheduler_cls = self._get_scheduler_class(crawler.settings)
downloader_cls = load_object(self.settings["DOWNLOADER"])
self.downloader = downloader_cls(crawler)
self.downloader: Downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
@ -91,20 +102,20 @@ class ExecutionEngine:
return scheduler_cls
@inlineCallbacks
def start(self) -> Deferred:
def start(self) -> Generator[Deferred, Any, None]:
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
self._closewait = Deferred()
self._closewait: Deferred = Deferred()
yield self._closewait
def stop(self) -> Deferred:
"""Gracefully stop the execution engine"""
@inlineCallbacks
def _finish_stopping_engine(_) -> Deferred:
def _finish_stopping_engine(_) -> Generator[Deferred, Any, None]:
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
self._closewait.callback(None)
@ -172,11 +183,13 @@ class ExecutionEngine:
self._spider_idle()
def _needs_backout(self) -> bool:
assert self.slot is not None # typing
assert self.scraper.slot is not None # typing
return (
not self.running
or self.slot.closing # type: ignore[union-attr]
or bool(self.slot.closing)
or self.downloader.needs_backout()
or self.scraper.slot.needs_backout() # type: ignore[union-attr]
or self.scraper.slot.needs_backout()
)
def _next_request_from_scheduler(self) -> Optional[Deferred]:
@ -196,7 +209,7 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
)
d.addBoth(lambda _: self.slot.remove_request(request))
d.addBoth(lambda _: cast(Slot, self.slot).remove_request(request))
d.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
@ -424,7 +437,7 @@ class ExecutionEngine:
dfd.addErrback(log_failure("Scraper close failure"))
if hasattr(self.slot.scheduler, "close"):
dfd.addBoth(lambda _: self.slot.scheduler.close(reason))
dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason))
dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(

16
scrapy/core/http2/agent.py
View File

@ -6,7 +6,12 @@ from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
from twisted.internet.endpoints import HostnameEndpoint
from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
from twisted.web.client import (
URI,
BrowserLikePolicyForHTTPS,
ResponseFailed,
_StandardEndpointFactory,
)
from twisted.web.error import SchemeNotSupported
from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
@ -35,7 +40,7 @@ class H2ConnectionPool:
# Received a request while connecting to remote
# Create a deferred which will fire with the H2ClientProtocol
# instance
d = Deferred()
d: Deferred = Deferred()
self._pending_requests[key].append(d)
return d
@ -53,14 +58,14 @@ class H2ConnectionPool:
) -> Deferred:
self._pending_requests[key] = deque()
conn_lost_deferred = Deferred()
conn_lost_deferred: Deferred = Deferred()
conn_lost_deferred.addCallback(self._remove_connection, key)
factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
conn_d = endpoint.connect(factory)
conn_d.addCallback(self.put_connection, key)
d = Deferred()
d: Deferred = Deferred()
self._pending_requests[key].append(d)
return d
@ -83,7 +88,7 @@ class H2ConnectionPool:
pending_requests = self._pending_requests.pop(key, None)
while pending_requests:
d = pending_requests.popleft()
d.errback(errors)
d.errback(ResponseFailed(errors))
def close_connections(self) -> None:
"""Close all the HTTP/2 connections and remove them from pool
@ -92,6 +97,7 @@ class H2ConnectionPool:
Deferred that fires when all connections have been closed
"""
for conn in self._connections.values():
assert conn.transport is not None # typing
conn.transport.abortConnection()

7
scrapy/core/http2/protocol.py
View File

@ -140,6 +140,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
This is used while initiating pending streams to make sure
that we initiate stream only during active HTTP/2 Connection
"""
assert self.transport is not None # typing
return bool(self.transport.connected) and self.metadata["settings_acknowledged"]
@property
@ -196,6 +197,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""Write data to the underlying transport connection
from the HTTP2 connection instance if any
"""
assert self.transport is not None # typing
# Reset the idle timeout as connection is still actively sending data
self.resetTimeout()
@ -226,6 +228,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# Initialize the timeout
self.setTimeout(self.IDLE_TIMEOUT)
assert self.transport is not None # typing
destination = self.transport.getPeer()
self.metadata["ip_address"] = ipaddress.ip_address(destination.host)
@ -237,12 +240,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
"""Helper function to lose the connection with the error sent as a
reason"""
self._conn_lost_errors += errors
assert self.transport is not None # typing
self.transport.loseConnection()
def handshakeCompleted(self) -> None:
"""
Close the connection if it's not made via the expected protocol
"""
assert self.transport is not None # typing
if (
self.transport.negotiatedProtocol is not None
and self.transport.negotiatedProtocol != PROTOCOL_NAME
@ -275,6 +280,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
# hyper-h2 does not drop the connection in this scenario, we
# need to abort the connection manually.
self._conn_lost_errors += [e]
assert self.transport is not None # typing
self.transport.abortConnection()
return
@ -388,6 +394,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self._send_pending_requests()
# Update certificate when our HTTP/2 connection is established
assert self.transport is not None # typing
self.metadata["certificate"] = Certificate(self.transport.getPeerCertificate())
def stream_ended(self, event: StreamEnded) -> None:

2
scrapy/core/http2/stream.py
View File

@ -151,7 +151,7 @@ class Stream:
else:
self.close(StreamCloseReason.CANCELLED)
self._deferred_response = Deferred(_cancel)
self._deferred_response: Deferred = Deferred(_cancel)
def __repr__(self) -> str:
return f"Stream(id={self.stream_id!r})"

8
scrapy/core/scraper.py
View File

@ -64,7 +64,7 @@ class Slot:
def add_response_request(
self, result: Union[Response, Failure], request: Request
) -> Deferred:
deferred = Deferred()
deferred: Deferred = Deferred()
self.queue.append((result, request, deferred))
if isinstance(result, Response):
self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
@ -203,10 +203,12 @@ class Scraper:
callback=callback, callbackKeywords=result.request.cb_kwargs
)
else: # result is a Failure
result.request = request
# TODO: properly type adding this attribute to a Failure
result.request = request # type: ignore[attr-defined]
warn_on_generator_with_return_value(spider, request.errback)
dfd = defer_fail(result)
dfd.addErrback(request.errback)
if request.errback:
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(

4
scrapy/core/spidermw.py
View File

@ -155,7 +155,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return dfd.result
return cast(MutableChain, dfd.result)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@ -180,7 +180,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
spider: Spider,
result: Union[Iterable, AsyncIterable],
start_index: int = 0,
) -> Deferred:
) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered: Union[MutableChain, MutableAsyncChain]

2
scrapy/downloadermiddlewares/decompression.py
View File

@ -51,7 +51,7 @@ class DecompressionMiddleware:
archive = BytesIO(response.body)
try:
zip_file = zipfile.ZipFile(archive)
except zipfile.BadZipfile:
except zipfile.BadZipFile:
return
namelist = zip_file.namelist()

56
scrapy/http/request/form.py
View File

@ -5,10 +5,17 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
from lxml.html import (
FormElement,
HTMLParser,
InputElement,
MultipleSelectOptions,
SelectElement,
TextareaElement,
)
from parsel.selector import create_root_node
from w3lib.html import strip_html5_whitespace
@ -19,7 +26,8 @@ from scrapy.utils.response import get_base_url
FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
FormdataType = Optional[Union[dict, List[FormdataKVType]]]
class FormRequest(Request):
@ -50,7 +58,7 @@ class FormRequest(Request):
response: TextResponse,
formname: Optional[str] = None,
formid: Optional[str] = None,
formnumber: Optional[int] = 0,
formnumber: int = 0,
formdata: FormdataType = None,
clickdata: Optional[dict] = None,
dont_click: bool = False,
@ -79,6 +87,7 @@ class FormRequest(Request):
def _get_form_url(form: FormElement, url: Optional[str]) -> str:
assert form.base_url is not None # typing
if url is None:
action = form.get("action")
if action is None:
@ -87,11 +96,11 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str:
return urljoin(form.base_url, url)
def _urlencode(seq: Iterable, enc: str) -> str:
def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
values = [
(to_bytes(k, enc), to_bytes(v, enc))
for k, vs in seq
for v in (vs if is_listlike(vs) else [vs])
for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
]
return urlencode(values, doseq=True)
@ -100,7 +109,7 @@ def _get_form(
response: TextResponse,
formname: Optional[str],
formid: Optional[str],
formnumber: Optional[int],
formnumber: int,
formxpath: Optional[str],
) -> FormElement:
"""Find the wanted form element within the given response."""
@ -133,13 +142,12 @@ def _get_form(
raise ValueError(f"No <form> element found with {formxpath}")
# If we get here, it means that either formname was None or invalid
if formnumber is not None:
try:
form = forms[formnumber]
except IndexError:
raise IndexError(f"Form number {formnumber} not found in {response}")
else:
return form
try:
form = forms[formnumber]
except IndexError:
raise IndexError(f"Form number {formnumber} not found in {response}")
else:
return form
def _get_inputs(
@ -147,7 +155,7 @@ def _get_inputs(
formdata: FormdataType,
dont_click: bool,
clickdata: Optional[dict],
) -> List[Tuple[str, str]]:
) -> List[FormdataKVType]:
"""Return a list of key-value pairs for the inputs found in the given form."""
try:
formdata_keys = dict(formdata or ()).keys()
@ -165,7 +173,7 @@ def _get_inputs(
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
namespaces={"re": "http://exslt.org/regular-expressions"},
)
values = [
values: List[FormdataKVType] = [
(k, "" if v is None else v)
for k, v in (_value(e) for e in inputs)
if k and k not in formdata_keys
@ -183,27 +191,25 @@ def _get_inputs(
return values
def _value(ele: HtmlElement):
def _value(
ele: Union[InputElement, SelectElement, TextareaElement]
) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]:
n = ele.name
v = ele.value
if ele.tag == "select":
return _select_value(ele, n, v)
return _select_value(cast(SelectElement, ele), n, v)
return n, v
def _select_value(ele: SelectElement, n: str, v: str):
def _select_value(
ele: SelectElement, n: Optional[str], v: Union[None, str, MultipleSelectOptions]
) -> Tuple[Optional[str], Union[None, str, MultipleSelectOptions]]:
multiple = ele.multiple
if v is None and not multiple:
# Match browser behaviour on simple select tag without options selected
# And for select tags without options
o = ele.value_options
return (n, o[0]) if o else (None, None)
if v is not None and multiple:
# This is a workaround to bug in lxml fixed 2.3.1
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
selected_options = ele.xpath(".//option[@selected]")
values = [(o.get("value") or o.text or "").strip() for o in selected_options]
return n, values
return n, v

22
scrapy/pipelines/files.py
View File

@ -13,8 +13,9 @@ from collections import defaultdict
from contextlib import suppress
from ftplib import FTP
from io import BytesIO
from os import PathLike
from pathlib import Path
from typing import DefaultDict, Optional, Set
from typing import DefaultDict, Optional, Set, Union
from urllib.parse import urlparse
from itemadapter import ItemAdapter
@ -36,24 +37,31 @@ from scrapy.utils.request import referer_str
logger = logging.getLogger(__name__)
def _to_string(path: Union[str, PathLike]) -> str:
return str(path) # convert a Path object to string
class FileException(Exception):
"""General media error exception"""
class FSFilesStore:
def __init__(self, basedir: str):
def __init__(self, basedir: Union[str, PathLike]):
basedir = _to_string(basedir)
if "://" in basedir:
basedir = basedir.split("://", 1)[1]
self.basedir = basedir
self._mkdir(Path(self.basedir))
self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
def persist_file(self, path: str, buf, info, meta=None, headers=None):
def persist_file(
self, path: Union[str, PathLike], buf, info, meta=None, headers=None
):
absolute_path = self._get_filesystem_path(path)
self._mkdir(absolute_path.parent, info)
absolute_path.write_bytes(buf.getvalue())
def stat_file(self, path: str, info):
def stat_file(self, path: Union[str, PathLike], info):
absolute_path = self._get_filesystem_path(path)
try:
last_modified = absolute_path.stat().st_mtime
@ -65,8 +73,8 @@ class FSFilesStore:
return {"last_modified": last_modified, "checksum": checksum}
def _get_filesystem_path(self, path: str) -> Path:
path_comps = path.split("/")
def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:
path_comps = _to_string(path).split("/")
return Path(self.basedir, *path_comps)
def _mkdir(self, dirname: Path, domain: Optional[str] = None):
@ -332,12 +340,12 @@ class FilesPipeline(MediaPipeline):
DEFAULT_FILES_RESULT_FIELD = "files"
def __init__(self, store_uri, download_func=None, settings=None):
store_uri = _to_string(store_uri)
if not store_uri:
raise NotConfigured
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
cls_name = "FilesPipeline"
self.store = self._get_store(store_uri)
resolve = functools.partial(

7
scrapy/settings/__init__.py
View File

@ -293,6 +293,13 @@ class BaseSettings(MutableMapping):
else:
self.attributes[name].set(value, priority)
def setdefault(self, name, default=None, priority="project"):
if name not in self:
self.set(name, default, priority)
return default
return self.attributes[name].value
def setdict(self, values, priority="project"):
self.update(values, priority)

2
scrapy/templates/spiders/basic.tmpl
View File

@ -4,7 +4,7 @@ import scrapy
class $classname(scrapy.Spider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = ["http://$domain/"]
start_urls = ["$url"]
def parse(self, response):
pass

2
scrapy/templates/spiders/crawl.tmpl
View File

@ -6,7 +6,7 @@ from scrapy.spiders import CrawlSpider, Rule
class $classname(CrawlSpider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = ["http://$domain/"]
start_urls = ["$url"]
rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)

2
scrapy/templates/spiders/csvfeed.tmpl
View File

@ -4,7 +4,7 @@ from scrapy.spiders import CSVFeedSpider
class $classname(CSVFeedSpider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = ["http://$domain/feed.csv"]
start_urls = ["$url"]
#headers = ["id", "name", "description", "image_link"]
#delimiter = "\t"

2
scrapy/templates/spiders/xmlfeed.tmpl
View File

@ -4,7 +4,7 @@ from scrapy.spiders import XMLFeedSpider
class $classname(XMLFeedSpider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = ["http://$domain/feed.xml"]
start_urls = ["$url"]
iterator = "iternodes" # you can change this; see the docs
itertag = "item" # change it accordingly

25
scrapy/utils/defer.py
View File

@ -17,6 +17,7 @@ from typing import (
List,
Optional,
Union,
cast,
)
from twisted.internet import defer
@ -38,7 +39,7 @@ def defer_fail(_failure: Failure) -> Deferred:
"""
from twisted.internet import reactor
d = Deferred()
d: Deferred = Deferred()
reactor.callLater(0.1, d.errback, _failure)
return d
@ -52,7 +53,7 @@ def defer_succeed(result) -> Deferred:
"""
from twisted.internet import reactor
d = Deferred()
d: Deferred = Deferred()
reactor.callLater(0.1, d.callback, result)
return d
@ -84,7 +85,7 @@ def mustbe_deferred(f: Callable, *args, **kw) -> Deferred:
def parallel(
iterable: Iterable, count: int, callable: Callable, *args, **named
) -> DeferredList:
) -> Deferred:
"""Execute a callable over the objects in the given iterable, in parallel,
using no more than ``count`` concurrent calls.
@ -182,7 +183,9 @@ class _AsyncCooperatorAdapter(Iterator):
def _call_anext(self) -> None:
# This starts waiting for the next result from aiterator.
# If aiterator is exhausted, _errback will be called.
self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
self.anext_deferred = cast(
Deferred, deferred_from_coro(self.aiterator.__anext__())
)
self.anext_deferred.addCallbacks(self._callback, self._errback)
def __next__(self) -> Deferred:
@ -190,7 +193,7 @@ class _AsyncCooperatorAdapter(Iterator):
# It also calls __anext__() if needed.
if self.finished:
raise StopIteration
d = Deferred()
d: Deferred = Deferred()
self.waiting_deferreds.append(d)
if not self.anext_deferred:
self._call_anext()
@ -199,17 +202,17 @@ class _AsyncCooperatorAdapter(Iterator):
def parallel_async(
async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named
) -> DeferredList:
) -> Deferred:
"""Like parallel but for async iterators"""
coop = Cooperator()
work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
dl = DeferredList([coop.coiterate(work) for _ in range(count)])
dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)])
return dl
def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
"""Return a Deferred built by chaining the given callbacks"""
d = Deferred()
d: Deferred = Deferred()
for x in callbacks:
d.addCallback(x, *a, **kw)
d.callback(input)
@ -220,7 +223,7 @@ def process_chain_both(
callbacks: Iterable[Callable], errbacks: Iterable[Callable], input, *a, **kw
) -> Deferred:
"""Return a Deferred built by chaining the given callbacks and errbacks"""
d = Deferred()
d: Deferred = Deferred()
for cb, eb in zip(callbacks, errbacks):
d.addCallbacks(
callback=cb,
@ -242,7 +245,7 @@ def process_parallel(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred
callbacks
"""
dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
d = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
return d
@ -285,7 +288,7 @@ def deferred_from_coro(o) -> Any:
if not is_asyncio_reactor_installed():
# wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
# that use asyncio, e.g. "await asyncio.sleep(1)"
return ensureDeferred(o)
return ensureDeferred(cast(Coroutine[Deferred, Any, Any], o))
# wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
event_loop = _get_asyncio_event_loop()
return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop))

4
scrapy/utils/python.py
View File

@ -8,7 +8,7 @@ import sys
import weakref
from functools import partial, wraps
from itertools import chain
from typing import AsyncGenerator, AsyncIterable, Iterable, Union
from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union
from scrapy.utils.asyncgen import as_async_generator
@ -45,7 +45,7 @@ def iflatten(x):
yield el
def is_listlike(x):
def is_listlike(x: Any) -> bool:
"""
>>> is_listlike("foo")
False

36
scrapy/utils/reactor.py
View File

@ -1,10 +1,11 @@
import asyncio
import sys
from contextlib import suppress
from warnings import catch_warnings, filterwarnings
from warnings import catch_warnings, filterwarnings, warn
from twisted.internet import asyncioreactor, error
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.misc import load_object
@ -54,7 +55,31 @@ class CallLaterOnce:
return self._func(*self._a, **self._kw)
def set_asyncio_event_loop_policy():
"""The policy functions from asyncio often behave unexpectedly,
so we restrict their use to the absolutely essential case.
This should only be used to install the reactor.
"""
_get_asyncio_event_loop_policy()
def get_asyncio_event_loop_policy():
warn(
"Call to deprecated function "
"scrapy.utils.reactor.get_asyncio_event_loop_policy().\n"
"\n"
"Please use get_event_loop, new_event_loop and set_event_loop"
" from asyncio instead, as the corresponding policy methods may lead"
" to unexpected behaviour.\n"
"This function is replaced by set_asyncio_event_loop_policy and"
" is meant to be used only when the reactor is being installed.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return _get_asyncio_event_loop_policy()
def _get_asyncio_event_loop_policy():
policy = asyncio.get_event_loop_policy()
if (
sys.version_info >= (3, 8)
@ -63,7 +88,6 @@ def get_asyncio_event_loop_policy():
):
policy = asyncio.WindowsSelectorEventLoopPolicy()
asyncio.set_event_loop_policy(policy)
return policy
@ -73,6 +97,7 @@ def install_reactor(reactor_path, event_loop_path=None):
path if the asyncio reactor is enabled"""
reactor_class = load_object(reactor_path)
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
set_asyncio_event_loop_policy()
with suppress(error.ReactorAlreadyInstalledError):
event_loop = set_asyncio_event_loop(event_loop_path)
asyncioreactor.install(eventloop=event_loop)
@ -90,7 +115,6 @@ def _get_asyncio_event_loop():
def set_asyncio_event_loop(event_loop_path):
"""Sets and returns the event loop with specified import path."""
policy = get_asyncio_event_loop_policy()
if event_loop_path is not None:
event_loop_class = load_object(event_loop_path)
event_loop = event_loop_class()
@ -109,15 +133,13 @@ def set_asyncio_event_loop(event_loop_path):
message="There is no current event loop",
category=DeprecationWarning,
)
event_loop = policy.get_event_loop()
event_loop = asyncio.get_event_loop()
except RuntimeError:
# `get_event_loop` raises RuntimeError when called with no asyncio
# event loop yet installed in the following scenarios:
# - From a thread other than the main thread. For example, when
# using ``scrapy shell``.
# - Previsibly on Python 3.14 and later.
# https://github.com/python/cpython/issues/100160#issuecomment-1345581902
event_loop = policy.new_event_loop()
event_loop = asyncio.new_event_loop()
asyncio.set_event_loop(event_loop)
return event_loop

7
scrapy/utils/response.py
View File

@ -6,7 +6,7 @@ import os
import re
import tempfile
import webbrowser
from typing import Any, Callable, Iterable, Optional, Tuple, Union
from typing import Any, Callable, Iterable, Tuple, Union
from weakref import WeakKeyDictionary
from twisted.web import http
@ -37,12 +37,13 @@ _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[floa
def get_meta_refresh(
response: "scrapy.http.response.text.TextResponse",
ignore_tags: Optional[Iterable[str]] = ("script", "noscript"),
ignore_tags: Iterable[str] = ("script", "noscript"),
) -> Union[Tuple[None, None], Tuple[float, str]]:
"""Parse the http-equiv refresh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
_metaref_cache[response] = html.get_meta_refresh(
# a w3lib typing bug here, fixed in https://github.com/scrapy/w3lib/pull/211
_metaref_cache[response] = html.get_meta_refresh( # type: ignore[assignment]
text, response.url, response.encoding, ignore_tags=ignore_tags
)
return _metaref_cache[response]

6
setup.cfg
View File

@ -6,6 +6,12 @@ universal=1
[mypy]
ignore_missing_imports = true
# Interface classes are hard to support
[mypy-twisted.internet.interfaces]
follow_imports = skip
[mypy-twisted.internet.reactor]
follow_imports = skip
# FIXME: remove the following sections once the issues are solved

3
tests/mockserver.py
View File

@ -10,6 +10,7 @@ from urllib.parse import urlencode
from OpenSSL import SSL
from twisted.internet import defer, reactor, ssl
from twisted.internet.protocol import ServerFactory
from twisted.internet.task import deferLater
from twisted.names import dns, error
from twisted.names.server import DNSServerFactory
@ -368,6 +369,8 @@ if __name__ == "__main__":
)
args = parser.parse_args()
factory: ServerFactory
if args.type == "http":
root = Root()
factory = Site(root)

110
tests/test_command_parse.py
View File

@ -30,14 +30,53 @@ import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.utils.test import get_from_asyncio_queue
import asyncio
class AsyncDefAsyncioSpider(scrapy.Spider):
name = 'asyncdef{self.spider_name}'
class AsyncDefAsyncioReturnSpider(scrapy.Spider):
name = "asyncdef_asyncio_return"
async def parse(self, response):
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
return [scrapy.Item(), dict(foo='bar')]
self.logger.info(f"Got response {{status}}")
return [{{'id': 1}}, {{'id': 2}}]
class AsyncDefAsyncioReturnSingleElementSpider(scrapy.Spider):
name = "asyncdef_asyncio_return_single_element"
async def parse(self, response):
await asyncio.sleep(0.1)
status = await get_from_asyncio_queue(response.status)
self.logger.info(f"Got response {{status}}")
return {{'foo': 42}}
class AsyncDefAsyncioGenLoopSpider(scrapy.Spider):
name = "asyncdef_asyncio_gen_loop"
async def parse(self, response):
for i in range(10):
await asyncio.sleep(0.1)
yield {{'foo': i}}
self.logger.info(f"Got response {{response.status}}")
class AsyncDefAsyncioSpider(scrapy.Spider):
name = "asyncdef_asyncio"
async def parse(self, response):
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.debug(f"Got response {{status}}")
class AsyncDefAsyncioGenExcSpider(scrapy.Spider):
name = "asyncdef_asyncio_gen_exc"
async def parse(self, response):
for i in range(10):
await asyncio.sleep(0.1)
yield {{'foo': i}}
if i > 5:
raise ValueError("Stopping the processing")
class MySpider(scrapy.Spider):
name = '{self.spider_name}'
@ -213,17 +252,76 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
self.assertIn("INFO: It Works!", _textmode(stderr))
@defer.inlineCallbacks
def test_asyncio_parse_items(self):
def test_async_def_asyncio_parse_items_list(self):
status, out, stderr = yield self.execute(
[
"--spider",
"asyncdef" + self.spider_name,
"asyncdef_asyncio_return",
"-c",
"parse",
self.url("/html"),
]
)
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
self.assertIn("INFO: Got response 200", _textmode(stderr))
self.assertIn("{'id': 1}", _textmode(out))
self.assertIn("{'id': 2}", _textmode(out))
@defer.inlineCallbacks
def test_async_def_asyncio_parse_items_single_element(self):
status, out, stderr = yield self.execute(
[
"--spider",
"asyncdef_asyncio_return_single_element",
"-c",
"parse",
self.url("/html"),
]
)
self.assertIn("INFO: Got response 200", _textmode(stderr))
self.assertIn("{'foo': 42}", _textmode(out))
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_loop(self):
status, out, stderr = yield self.execute(
[
"--spider",
"asyncdef_asyncio_gen_loop",
"-c",
"parse",
self.url("/html"),
]
)
self.assertIn("INFO: Got response 200", _textmode(stderr))
for i in range(10):
self.assertIn(f"{{'foo': {i}}}", _textmode(out))
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_exc(self):
status, out, stderr = yield self.execute(
[
"--spider",
"asyncdef_asyncio_gen_exc",
"-c",
"parse",
self.url("/html"),
]
)
self.assertIn("ValueError", _textmode(stderr))
for i in range(7):
self.assertIn(f"{{'foo': {i}}}", _textmode(out))
@defer.inlineCallbacks
def test_async_def_asyncio_parse(self):
_, _, stderr = yield self.execute(
[
"--spider",
"asyncdef_asyncio",
"-c",
"parse",
self.url("/html"),
]
)
self.assertIn("DEBUG: Got response 200", _textmode(stderr))
@defer.inlineCallbacks
def test_parse_items(self):

63
tests/test_commands.py
View File

@ -541,7 +541,7 @@ class GenspiderCommandTest(CommandTest):
).group(1),
)
self.assertEqual(
f"http://{domain}/",
f"https://{domain}",
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
@ -549,13 +549,64 @@ class GenspiderCommandTest(CommandTest):
)
def test_url_schema(self):
self.test_url("http://test.com", "test.com")
self.test_url("https://test.com", "test.com")
def test_url_path(self):
self.test_url("test.com/some/other/page", "test.com")
def test_template_start_urls(
self, url="test.com", expected="https://test.com", template="basic"
):
self.assertEqual(
0, self.call("genspider", "-t", template, "--force", "test_name", url)
)
self.assertEqual(
expected,
self.find_in_file(
Path(self.proj_mod_path, "spiders", "test_name.py"),
r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
).group(1),
)
def test_url_schema_path(self):
self.test_url("https://test.com/some/other/page", "test.com")
def test_genspider_basic_start_urls(self):
self.test_template_start_urls("https://test.com", "https://test.com", "basic")
self.test_template_start_urls("http://test.com", "http://test.com", "basic")
self.test_template_start_urls(
"http://test.com/other/path", "http://test.com/other/path", "basic"
)
self.test_template_start_urls(
"test.com/other/path", "https://test.com/other/path", "basic"
)
def test_genspider_crawl_start_urls(self):
self.test_template_start_urls("https://test.com", "https://test.com", "crawl")
self.test_template_start_urls("http://test.com", "http://test.com", "crawl")
self.test_template_start_urls(
"http://test.com/other/path", "http://test.com/other/path", "crawl"
)
self.test_template_start_urls(
"test.com/other/path", "https://test.com/other/path", "crawl"
)
self.test_template_start_urls("test.com", "https://test.com", "crawl")
def test_genspider_xmlfeed_start_urls(self):
self.test_template_start_urls(
"https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
)
self.test_template_start_urls(
"http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"
)
self.test_template_start_urls(
"test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
)
def test_genspider_csvfeed_start_urls(self):
self.test_template_start_urls(
"https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
)
self.test_template_start_urls(
"http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"
)
self.test_template_start_urls(
"test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
)
class GenspiderStandaloneCommandTest(ProjectTest):

4
tests/test_dependencies.py
View File

@ -37,7 +37,3 @@ class ScrapyUtilsTest(unittest.TestCase):
pinned_twisted_version_string = match[1]
self.assertEqual(twisted_version.short(), pinned_twisted_version_string)
if __name__ == "__main__":
unittest.main()

2
tests/test_downloadermiddleware_httpproxy.py
View File

@ -13,7 +13,7 @@ spider = Spider("foo")
class TestHttpProxyMiddleware(TestCase):
failureException = AssertionError
failureException = AssertionError # type: ignore[assignment]
def setUp(self):
self._oldenv = os.environ.copy()

72
tests/test_downloaderslotssettings.py Normal file
View File

@ -0,0 +1,72 @@
import time
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from tests.mockserver import MockServer
from tests.spiders import MetaSpider
class DownloaderSlotsSettingsTestSpider(MetaSpider):
name = "downloader_slots"
custom_settings = {
"DOWNLOAD_DELAY": 1,
"RANDOMIZE_DOWNLOAD_DELAY": False,
"DOWNLOAD_SLOTS": {
"quotes.toscrape.com": {
"concurrency": 1,
"delay": 2,
"randomize_delay": False,
},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
},
}
def start_requests(self):
self.times = {None: []}
slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]
for slot in slots:
url = self.mockserver.url(f"/?downloader_slot={slot}")
self.times[slot] = []
yield Request(url, callback=self.parse, meta={"download_slot": slot})
def parse(self, response):
slot = response.meta.get("download_slot", None)
self.times[slot].append(time.time())
url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
yield Request(url, callback=self.not_parse, meta={"download_slot": slot})
def not_parse(self, response):
slot = response.meta.get("download_slot", None)
self.times[slot].append(time.time())
class CrawlTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_delay(self):
crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider)
yield crawler.crawl(mockserver=self.mockserver)
slots = crawler.engine.downloader.slots
times = crawler.spider.times
tolerance = 0.3
delays_real = {k: v[1] - v[0] for k, v in times.items()}
error_delta = {
k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
for k, v in slots.items()
}
self.assertTrue(max(list(error_delta.values())) < tolerance)

5
tests/test_http2_client_protocol.py
View File

@ -5,6 +5,7 @@ import shutil
import string
from ipaddress import IPv4Address
from pathlib import Path
from typing import Dict
from unittest import mock, skipIf
from urllib.parse import urlencode
@ -87,6 +88,7 @@ class GetDataHtmlLarge(LeafResource):
class PostDataJsonMixin:
@staticmethod
def make_response(request: TxRequest, extra_data: str):
assert request.content is not None
response = {
"request-headers": {},
"request-body": json.loads(request.content.read()),
@ -145,7 +147,8 @@ class QueryParams(LeafResource):
request.setHeader("Content-Type", "application/json; charset=UTF-8")
request.setHeader("Content-Encoding", "UTF-8")
query_params = {}
query_params: Dict[str, str] = {}
assert request.args is not None
for k, v in request.args.items():
query_params[str(k, "utf-8")] = str(v[0], "utf-8")

16
tests/test_pipeline_files.py
View File

@ -480,6 +480,22 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
expected_value = settings.get(settings_attr)
self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value)
def test_file_pipeline_using_pathlike_objects(self):
class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
def file_path(self, request, response=None, info=None, *, item=None):
return Path("subdir") / Path(request.url).name
pipeline = CustomFilesPipelineWithPathLikeDir.from_settings(
Settings({"FILES_STORE": Path("./Temp")})
)
request = Request("http://example.com/image01.jpg")
self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg"))
def test_files_store_constructor_with_pathlike_object(self):
path = Path("./FileDir")
fs_store = FSFilesStore(path)
self.assertEqual(fs_store.basedir, str(path))
class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks

13
tests/test_settings/__init__.py
View File

@ -65,6 +65,19 @@ class BaseSettingsTest(unittest.TestCase):
def setUp(self):
self.settings = BaseSettings()
def test_setdefault_not_existing_value(self):
settings = BaseSettings()
value = settings.setdefault("TEST_OPTION", "value")
self.assertEqual(settings["TEST_OPTION"], "value")
self.assertEqual(value, "value")
self.assertIsNotNone(value)
def test_setdefault_existing_value(self):
settings = BaseSettings({"TEST_OPTION": "value"})
value = settings.setdefault("TEST_OPTION", None)
self.assertEqual(settings["TEST_OPTION"], "value")
self.assertEqual(value, "value")
def test_set_new_attribute(self):
self.settings.set("TEST_OPTION", "value", 0)
self.assertIn("TEST_OPTION", self.settings.attributes)

1
tests/test_spidermiddleware.py
View File

@ -112,6 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
Should work for process_spider_output and, when it's supported, process_start_requests.
"""
ITEM_TYPE: type
RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
@staticmethod

11
tests/test_utils_asyncio.py
View File

@ -1,9 +1,14 @@
import asyncio
import warnings
from unittest import TestCase
from pytest import mark
from scrapy.utils.reactor import install_reactor, is_asyncio_reactor_installed
from scrapy.utils.reactor import (
install_reactor,
is_asyncio_reactor_installed,
set_asyncio_event_loop,
)
@mark.usefixtures("reactor_pytest")
@ -23,3 +28,7 @@ class AsyncioTest(TestCase):
from twisted.internet import reactor
assert original_reactor == reactor
async def test_set_asyncio_event_loop(self):
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
assert set_asyncio_event_loop() is asyncio.get_running_loop()

4
tests/test_utils_iterators.py
View File

@ -527,7 +527,3 @@ class TestHelper(unittest.TestCase):
type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
)
self.assertEqual(a, b)
if __name__ == "__main__":
unittest.main()

4
tests/test_utils_python.py
View File

@ -257,7 +257,3 @@ class UtilsPythonTestCase(unittest.TestCase):
without_none_values({"one": 1, "none": None, "three": 3, "four": 4}),
{"one": 1, "three": 3, "four": 4},
)
if __name__ == "__main__":
unittest.main()

10
tox.ini
View File

@ -37,11 +37,13 @@ install_command =
[testenv:typing]
basepython = python3
deps =
lxml-stubs==0.4.0
mypy==0.991
mypy==1.0.1
types-attrs==19.1.0
types-pyOpenSSL==23.0.0.2
types-setuptools==65.7.0.3
types-lxml==2023.2.11
types-Pillow==9.4.0.16
types-Pygments==2.14.0.5
types-pyOpenSSL==23.0.0.4
types-setuptools==67.4.0.1
commands =
mypy --show-error-codes {posargs: scrapy tests}