mirror of https://github.com/scrapy/scrapy.git
153 lines
3.8 KiB
Python
153 lines
3.8 KiB
Python
"""
|
|
Queues that handle requests
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from scrapy.http import Request
|
|
from scrapy.spiders import Spider
|
|
from scrapy.squeues import (
|
|
FifoMemoryQueue,
|
|
LifoMemoryQueue,
|
|
MarshalFifoDiskQueue,
|
|
MarshalLifoDiskQueue,
|
|
PickleFifoDiskQueue,
|
|
PickleLifoDiskQueue,
|
|
)
|
|
from scrapy.utils.misc import build_from_crawler
|
|
from scrapy.utils.test import get_crawler
|
|
|
|
if TYPE_CHECKING:
|
|
import queuelib
|
|
|
|
from scrapy.crawler import Crawler
|
|
|
|
|
|
@pytest.fixture
|
|
def crawler() -> Crawler:
|
|
return get_crawler(Spider)
|
|
|
|
|
|
class TestRequestQueueBase(ABC):
|
|
@property
|
|
@abstractmethod
|
|
def is_fifo(self) -> bool:
|
|
raise NotImplementedError
|
|
|
|
def test_one_element(self, q: queuelib.queue.BaseQueue):
|
|
assert len(q) == 0
|
|
assert q.peek() is None
|
|
assert q.pop() is None
|
|
req = Request("http://www.example.com")
|
|
q.push(req)
|
|
assert len(q) == 1
|
|
result = q.peek()
|
|
assert result is not None
|
|
assert result.url == req.url
|
|
result = q.pop()
|
|
assert result is not None
|
|
assert result.url == req.url
|
|
assert len(q) == 0
|
|
assert q.peek() is None
|
|
assert q.pop() is None
|
|
|
|
def test_order(self, q: queuelib.queue.BaseQueue):
|
|
assert len(q) == 0
|
|
assert q.peek() is None
|
|
assert q.pop() is None
|
|
req1 = Request("http://www.example.com/1")
|
|
req2 = Request("http://www.example.com/2")
|
|
req3 = Request("http://www.example.com/3")
|
|
q.push(req1)
|
|
q.push(req2)
|
|
q.push(req3)
|
|
reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1]
|
|
for i, req in enumerate(reqs):
|
|
assert len(q) == 3 - i
|
|
result = q.peek()
|
|
assert result is not None
|
|
assert result.url == req.url
|
|
result = q.pop()
|
|
assert result is not None
|
|
assert result.url == req.url
|
|
assert len(q) == 0
|
|
assert q.peek() is None
|
|
assert q.pop() is None
|
|
|
|
|
|
class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):
|
|
is_fifo = True
|
|
|
|
@pytest.fixture
|
|
def q(self, crawler, tmp_path):
|
|
queue = build_from_crawler(
|
|
PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
|
|
)
|
|
try:
|
|
yield queue
|
|
finally:
|
|
queue.close()
|
|
|
|
|
|
class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):
|
|
is_fifo = False
|
|
|
|
@pytest.fixture
|
|
def q(self, crawler, tmp_path):
|
|
queue = build_from_crawler(
|
|
PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
|
|
)
|
|
try:
|
|
yield queue
|
|
finally:
|
|
queue.close()
|
|
|
|
|
|
class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):
|
|
is_fifo = True
|
|
|
|
@pytest.fixture
|
|
def q(self, crawler, tmp_path):
|
|
queue = build_from_crawler(
|
|
MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
|
|
)
|
|
try:
|
|
yield queue
|
|
finally:
|
|
queue.close()
|
|
|
|
|
|
class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):
|
|
is_fifo = False
|
|
|
|
@pytest.fixture
|
|
def q(self, crawler, tmp_path):
|
|
queue = build_from_crawler(
|
|
MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
|
|
)
|
|
try:
|
|
yield queue
|
|
finally:
|
|
queue.close()
|
|
|
|
|
|
class TestFifoMemoryQueueRequest(TestRequestQueueBase):
|
|
is_fifo = True
|
|
|
|
@pytest.fixture
|
|
def q(self, crawler):
|
|
return build_from_crawler(FifoMemoryQueue, crawler)
|
|
|
|
|
|
class TestLifoMemoryQueueRequest(TestRequestQueueBase):
|
|
is_fifo = False
|
|
|
|
@pytest.fixture
|
|
def q(self, crawler):
|
|
return build_from_crawler(LifoMemoryQueue, crawler)
|