RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Filter test-time warnings. (#6501)

This commit is contained in:
Andrey Rakhmatullin 2024-10-21 15:30:49 +05:00 committed by GitHub
parent 6d65708cb7
commit 04d0411bf7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_crawl.py
View File

@ -428,7 +428,7 @@ with multiples lines
@defer.inlineCallbacks
def test_crawl_multiple(self):
runner = CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
runner = CrawlerRunner()
runner.crawl(
SimpleSpider,
self.mockserver.url("/status?n=200"),

12
tests/test_crawler.py
View File

@ -6,6 +6,7 @@ import subprocess
import sys
import warnings
from pathlib import Path
from typing import Any
import pytest
from packaging.version import parse as parse_version
@ -28,10 +29,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer, get_mockserver_env
# To prevent warnings.
BASE_SETTINGS = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
BASE_SETTINGS: dict[str, Any] = {}
def get_raw_crawler(spidercls=None, settings_dict=None):
@ -478,8 +476,6 @@ class CrawlerLoggingTestCase(unittest.TestCase):
custom_settings = {
"LOG_LEVEL": "INFO",
"LOG_FILE": str(log_file),
# settings to avoid extra warnings
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
configure_logging()
@ -582,7 +578,7 @@ class NoRequestsSpider(scrapy.Spider):
@mark.usefixtures("reactor_pytest")
class CrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
return CrawlerRunner({"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7"})
return CrawlerRunner()
@inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
@ -631,7 +627,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
)
else:
@ -640,7 +635,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
)
yield runner.crawl(NoRequestsSpider)

10
tests/test_downloadermiddleware_offsite.py
View File

@ -1,3 +1,5 @@
import warnings
import pytest
from scrapy import Request, Spider
@ -87,7 +89,9 @@ def test_process_request_invalid_domains():
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
mw.spider_opened(spider)
request = Request("https://a.example")
assert mw.process_request(request, spider) is None
for letter in ("b", "c"):
@ -175,7 +179,9 @@ def test_request_scheduled_invalid_domains():
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(spider)
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
mw.spider_opened(spider)
request = Request("https://a.example")
assert mw.request_scheduled(request, spider) is None
for letter in ("b", "c"):

6
tests/test_dupefilters.py
View File

@ -50,7 +50,6 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {
"DUPEFILTER_DEBUG": True,
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
@ -61,7 +60,6 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {
"DUPEFILTER_DEBUG": True,
"DUPEFILTER_CLASS": FromSettingsRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
@ -71,7 +69,6 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_direct_scheduler(self):
settings = {
"DUPEFILTER_CLASS": DirectDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
@ -176,7 +173,6 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {
"DUPEFILTER_DEBUG": False,
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
@ -205,7 +201,6 @@ class RFPDupeFilterTest(unittest.TestCase):
settings = {
"DUPEFILTER_DEBUG": True,
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
@ -243,7 +238,6 @@ class RFPDupeFilterTest(unittest.TestCase):
with LogCapture() as log:
settings = {
"DUPEFILTER_DEBUG": True,
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)

1
tests/test_pipeline_crawl.py
View File

@ -71,7 +71,6 @@ class FileDownloadCrawlTestCase(TestCase):
# prepare a directory for storing files
self.tmpmediastore = Path(mkdtemp())
self.settings = {
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
"ITEM_PIPELINES": {self.pipeline_class: 1},
self.store_setting_key: str(self.tmpmediastore),
}

1
tests/test_scheduler.py
View File

@ -53,7 +53,6 @@ class MockCrawler(Crawler):
"SCHEDULER_PRIORITY_QUEUE": priority_queue_cls,
"JOBDIR": jobdir,
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
super().__init__(Spider, settings)
self.engine = MockEngine(downloader=MockDownloader())

1
tests/test_spiderloader/__init__.py
View File

@ -103,7 +103,6 @@ class SpiderLoaderTest(unittest.TestCase):
runner = CrawlerRunner(
{
"SPIDER_MODULES": [module],
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
)

2
tests/test_utils_asyncio.py
View File

@ -1,8 +1,8 @@
import asyncio
import warnings
from unittest import TestCase
from pytest import mark
from twisted.trial.unittest import TestCase
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.reactor import (

6
tests/test_utils_datatypes.py
View File

@ -3,6 +3,8 @@ import unittest
import warnings
from collections.abc import Iterator, Mapping, MutableMapping
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.datatypes import (
@ -90,12 +92,14 @@ class CaseInsensitiveDictMixin:
self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
self.assertRaises(KeyError, d.__getitem__, "key_lower")
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_getdefault(self):
d = CaselessDict()
self.assertEqual(d.get("c", 5), 5)
d["c"] = 10
self.assertEqual(d.get("c", 5), 10)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_setdefault(self):
d = CaselessDict({"a": 1, "b": 2})
@ -212,11 +216,13 @@ class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
dict_class = CaselessDict
def test_deprecation_message(self):
with warnings.catch_warnings(record=True) as caught:
warnings.filterwarnings("always", category=ScrapyDeprecationWarning)
self.dict_class({"foo": "bar"})
self.assertEqual(len(caught), 1)

3
tests/test_utils_misc/__init__.py
View File

@ -4,6 +4,8 @@ import unittest
from pathlib import Path
from unittest import mock
import pytest
from scrapy.item import Field, Item
from scrapy.utils.misc import (
arg_to_iter,
@ -97,6 +99,7 @@ class UtilsMiscTestCase(unittest.TestCase):
list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]
)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_create_instance(self):
settings = mock.MagicMock()
crawler = mock.MagicMock(spec_set=["settings"])