RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use `dataclass` for the downloader `Slot` (#7405)

This commit is contained in:
Albert Eduardovich N. 2026-04-08 14:26:01 +03:00 committed by GitHub
parent 830eaeab5d
commit 8835a69f12
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
scrapy/core/downloader/__init__.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import random import random
from collections import deque from collections import deque
from dataclasses import dataclass, field
from datetime import datetime from datetime import datetime
from time import time from time import time
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
@ -40,24 +41,21 @@ if TYPE_CHECKING:
from scrapy.signalmanager import SignalManager from scrapy.signalmanager import SignalManager
@dataclass(slots=True, eq=False)
class Slot: class Slot:
"""Downloader slot""" """Downloader slot"""
def __init__( concurrency: int
self, delay: float
concurrency: int, randomize_delay: bool
delay: float,
randomize_delay: bool,
):
self.concurrency: int = concurrency
self.delay: float = delay
self.randomize_delay: bool = randomize_delay
self.active: set[Request] = set() active: set[Request] = field(default_factory=set, init=False, repr=False)
self.queue: deque[tuple[Request, Deferred[Response]]] = deque() queue: deque[tuple[Request, Deferred[Response]]] = field(
self.transferring: set[Request] = set() default_factory=deque, init=False, repr=False
self.lastseen: float = 0 )
self.latercall: CallLaterResult | None = None transferring: set[Request] = field(default_factory=set, init=False, repr=False)
lastseen: float = field(default=0, init=False, repr=False)
latercall: CallLaterResult | None = field(default=None, init=False, repr=False)
def free_transfer_slots(self) -> int: def free_transfer_slots(self) -> int:
return self.concurrency - len(self.transferring) return self.concurrency - len(self.transferring)
@ -72,14 +70,6 @@ class Slot:
self.latercall.cancel() self.latercall.cancel()
self.latercall = None self.latercall = None
def __repr__(self) -> str:
cls_name = self.__class__.__name__
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
f"randomize_delay={self.randomize_delay!r})"
)
def __str__(self) -> str: def __str__(self) -> str:
return ( return (
f"<downloader.Slot concurrency={self.concurrency!r} " f"<downloader.Slot concurrency={self.concurrency!r} "

2
tests/test_core_downloader.py
View File

@ -35,7 +35,7 @@ if TYPE_CHECKING:
class TestSlot: class TestSlot:
def test_repr(self): def test_repr(self):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)" assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)"
@pytest.mark.requires_reactor # this test is related to the Twisted HTTP code @pytest.mark.requires_reactor # this test is related to the Twisted HTTP code