""" Queues that handle requests """ from __future__ import annotations from abc import ABC, abstractmethod from typing import TYPE_CHECKING import pytest from scrapy.http import Request from scrapy.spiders import Spider from scrapy.squeues import ( FifoMemoryQueue, LifoMemoryQueue, MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue, ) from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: import queuelib from scrapy.crawler import Crawler @pytest.fixture def crawler() -> Crawler: return get_crawler(Spider) class TestRequestQueueBase(ABC): @property @abstractmethod def is_fifo(self) -> bool: raise NotImplementedError def test_one_element(self, q: queuelib.queue.BaseQueue): assert len(q) == 0 assert q.peek() is None assert q.pop() is None req = Request("http://www.example.com") q.push(req) assert len(q) == 1 result = q.peek() assert result is not None assert result.url == req.url result = q.pop() assert result is not None assert result.url == req.url assert len(q) == 0 assert q.peek() is None assert q.pop() is None def test_order(self, q: queuelib.queue.BaseQueue): assert len(q) == 0 assert q.peek() is None assert q.pop() is None req1 = Request("http://www.example.com/1") req2 = Request("http://www.example.com/2") req3 = Request("http://www.example.com/3") q.push(req1) q.push(req2) q.push(req3) reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1] for i, req in enumerate(reqs): assert len(q) == 3 - i result = q.peek() assert result is not None assert result.url == req.url result = q.pop() assert result is not None assert result.url == req.url assert len(q) == 0 assert q.peek() is None assert q.pop() is None class TestPickleFifoDiskQueueRequest(TestRequestQueueBase): is_fifo = True @pytest.fixture def q(self, crawler, tmp_path): queue = build_from_crawler( PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo") ) try: yield queue finally: queue.close() class TestPickleLifoDiskQueueRequest(TestRequestQueueBase): is_fifo = False @pytest.fixture def q(self, crawler, tmp_path): queue = build_from_crawler( PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo") ) try: yield queue finally: queue.close() class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase): is_fifo = True @pytest.fixture def q(self, crawler, tmp_path): queue = build_from_crawler( MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo") ) try: yield queue finally: queue.close() class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase): is_fifo = False @pytest.fixture def q(self, crawler, tmp_path): queue = build_from_crawler( MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo") ) try: yield queue finally: queue.close() class TestFifoMemoryQueueRequest(TestRequestQueueBase): is_fifo = True @pytest.fixture def q(self, crawler): return build_from_crawler(FifoMemoryQueue, crawler) class TestLifoMemoryQueueRequest(TestRequestQueueBase): is_fifo = False @pytest.fixture def q(self, crawler): return build_from_crawler(LifoMemoryQueue, crawler)