RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce deps on unittest, unify inlineCallbacks imports in tests. (#6873)

This commit is contained in:
Andrey Rakhmatullin 2025-06-07 01:59:09 +05:00 committed by GitHub
parent 657e6cb2b5
commit d825133284
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
42 changed files with 380 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/test_closespider.py
View File

@ -1,4 +1,4 @@
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.utils.test import get_crawler
@ -22,7 +22,7 @@ class TestCloseSpider(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_itemcount(self):
close_on = 5
crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on})
@ -32,7 +32,7 @@ class TestCloseSpider(TestCase):
itemcount = crawler.stats.get_value("item_scraped_count")
assert itemcount >= close_on
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_pagecount(self):
close_on = 5
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on})
@ -42,7 +42,7 @@ class TestCloseSpider(TestCase):
pagecount = crawler.stats.get_value("response_received_count")
assert pagecount >= close_on
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_pagecount_no_item(self):
close_on = 5
max_items = 5
@ -62,7 +62,7 @@ class TestCloseSpider(TestCase):
itemcount = crawler.stats.get_value("item_scraped_count")
assert pagecount <= close_on + itemcount
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_pagecount_no_item_with_pagecount(self):
close_on_pagecount_no_item = 5
close_on_pagecount = 20
@ -79,7 +79,7 @@ class TestCloseSpider(TestCase):
pagecount = crawler.stats.get_value("response_received_count")
assert pagecount < close_on_pagecount
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_errorcount(self):
close_on = 5
crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on})
@ -91,7 +91,7 @@ class TestCloseSpider(TestCase):
assert crawler.stats.get_value("spider_exceptions/count") >= close_on
assert errorcount >= close_on
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_timeout(self):
close_on = 0.1
crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on})
@ -101,7 +101,7 @@ class TestCloseSpider(TestCase):
total_seconds = crawler.stats.get_value("elapsed_time_seconds")
assert total_seconds >= close_on
@defer.inlineCallbacks
@inlineCallbacks
def test_closespider_timeout_no_item(self):
timeout = 1
crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})

10
tests/test_command_fetch.py
View File

@ -1,4 +1,4 @@
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from tests.utils.testproc import ProcessTest
@ -8,17 +8,17 @@ from tests.utils.testsite import SiteTest
class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase):
command = "fetch"
@defer.inlineCallbacks
@inlineCallbacks
def test_output(self):
_, out, _ = yield self.execute([self.url("/text")])
assert out.strip() == b"Works"
@defer.inlineCallbacks
@inlineCallbacks
def test_redirect_default(self):
_, out, _ = yield self.execute([self.url("/redirect")])
assert out.strip() == b"Redirected here"
@defer.inlineCallbacks
@inlineCallbacks
def test_redirect_disabled(self):
_, out, err = yield self.execute(
["--no-redirect", self.url("/redirect-no-meta-refresh")]
@ -27,7 +27,7 @@ class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase):
assert b"downloader/response_status_count/302" in err, err
assert b"downloader/response_status_count/200" not in err, err
@defer.inlineCallbacks
@inlineCallbacks
def test_headers(self):
_, out, _ = yield self.execute([self.url("/text"), "--headers"])
out = out.replace(b"\r", b"") # required on win32

42
tests/test_command_parse.py
View File

@ -3,7 +3,7 @@ import os
import re
from pathlib import Path
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from scrapy.commands import parse
from scrapy.settings import Settings
@ -171,7 +171,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
"""
)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_arguments(self):
_, _, stderr = yield self.execute(
[
@ -187,7 +187,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "DEBUG: It Works!" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_request_with_meta(self):
raw_json_string = '{"foo" : "baz"}'
_, _, stderr = yield self.execute(
@ -218,7 +218,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "DEBUG: It Works!" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_request_with_cb_kwargs(self):
raw_json_string = '{"foo" : "bar", "key": "value"}'
_, _, stderr = yield self.execute(
@ -239,7 +239,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
"DEBUG: request.callback signature: (response, foo=None, key=None)" in log
)
@defer.inlineCallbacks
@inlineCallbacks
def test_request_without_meta(self):
_, _, stderr = yield self.execute(
[
@ -253,7 +253,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "DEBUG: It Works!" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_pipelines(self):
_, _, stderr = yield self.execute(
[
@ -268,7 +268,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "INFO: It Works!" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse_items_list(self):
status, out, stderr = yield self.execute(
[
@ -283,7 +283,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
assert "{'id': 1}" in _textmode(out)
assert "{'id': 2}" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse_items_single_element(self):
status, out, stderr = yield self.execute(
[
@ -297,7 +297,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
assert "INFO: Got response 200" in _textmode(stderr)
assert "{'foo': 42}" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncgen_parse_loop(self):
status, out, stderr = yield self.execute(
[
@ -312,7 +312,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
for i in range(10):
assert f"{{'foo': {i}}}" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncgen_parse_exc(self):
status, out, stderr = yield self.execute(
[
@ -327,7 +327,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
for i in range(7):
assert f"{{'foo': {i}}}" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse(self):
_, _, stderr = yield self.execute(
[
@ -340,21 +340,21 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "DEBUG: Got response 200" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_parse_items(self):
status, out, stderr = yield self.execute(
["--spider", self.spider_name, "-c", "parse", self.url("/html")]
)
assert "[{}, {'foo': 'bar'}]" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_parse_items_no_callback_passed(self):
status, out, stderr = yield self.execute(
["--spider", self.spider_name, self.url("/html")]
)
assert "[{}, {'foo': 'bar'}]" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_wrong_callback_passed(self):
status, out, stderr = yield self.execute(
["--spider", self.spider_name, "-c", "dummy", self.url("/html")]
@ -362,7 +362,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out))
assert "Cannot find callback" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_matching_rule_callback_set(self):
"""If a rule matches the URL, use it's defined callback."""
status, out, stderr = yield self.execute(
@ -370,7 +370,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "[{}, {'foo': 'bar'}]" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_matching_rule_default_callback(self):
"""If a rule match but it has no callback set, use the 'parse' callback."""
status, out, stderr = yield self.execute(
@ -378,7 +378,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "[{}, {'nomatch': 'default'}]" in _textmode(out)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_with_no_rules_attribute(self):
"""Using -r with a spider with no rule should not produce items."""
status, out, stderr = yield self.execute(
@ -387,14 +387,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out))
assert "No CrawlSpider rules found" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_missing_callback(self):
status, out, stderr = yield self.execute(
["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")]
)
assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out))
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_no_matching_rule(self):
"""The requested URL has no matching rule, so no items should be scraped"""
status, out, stderr = yield self.execute(
@ -403,12 +403,12 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out))
assert "Cannot find a rule that matches" in _textmode(stderr)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_not_exists_with_not_matched_url(self):
status, out, stderr = yield self.execute([self.url("/invalid_url")])
assert status == 0
@defer.inlineCallbacks
@inlineCallbacks
def test_output_flag(self):
"""Checks if a file was created successfully having
correct format containing correct data in it.

5
tests/test_command_runspider.py
View File

@ -10,7 +10,6 @@ from typing import TYPE_CHECKING
from unittest import skipIf
import pytest
from twisted.trial import unittest
from tests.test_commands import TestCommandBase
from tests.test_crawler import ExceptionSpider, NoRequestsSpider
@ -376,7 +375,7 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand):
def setUp(self):
if platform.system() != "Windows":
raise unittest.SkipTest("Windows required for .pyw files")
pytest.skip("Windows required for .pyw files")
return super().setUp()
def test_start_errors(self):
@ -385,4 +384,4 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand):
assert "badspider.pyw" in log
def test_runspider_unable_to_load(self):
raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
pytest.skip("Already Tested in 'RunSpiderCommandTest' ")

39
tests/test_command_shell.py
View File

@ -3,8 +3,9 @@ import sys
from io import BytesIO
from pathlib import Path
import pytest
from pexpect.popen_spawn import PopenSpawn
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from scrapy.utils.reactor import _asyncio_reactor_path
@ -17,52 +18,52 @@ from tests.utils.testsite import SiteTest
class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase):
command = "shell"
@defer.inlineCallbacks
@inlineCallbacks
def test_empty(self):
_, out, _ = yield self.execute(["-c", "item"])
assert b"{}" in out
@defer.inlineCallbacks
@inlineCallbacks
def test_response_body(self):
_, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"])
assert b"Works" in out
@defer.inlineCallbacks
@inlineCallbacks
def test_response_type_text(self):
_, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"])
assert b"TextResponse" in out
@defer.inlineCallbacks
@inlineCallbacks
def test_response_type_html(self):
_, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"])
assert b"HtmlResponse" in out
@defer.inlineCallbacks
@inlineCallbacks
def test_response_selector_html(self):
xpath = "response.xpath(\"//p[@class='one']/text()\").get()"
_, out, _ = yield self.execute([self.url("/html"), "-c", xpath])
assert out.strip() == b"Works"
@defer.inlineCallbacks
@inlineCallbacks
def test_response_encoding_gb18030(self):
_, out, _ = yield self.execute(
[self.url("/enc-gb18030"), "-c", "response.encoding"]
)
assert out.strip() == b"gb18030"
@defer.inlineCallbacks
@inlineCallbacks
def test_redirect(self):
_, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"])
assert out.strip().endswith(b"/redirected")
@defer.inlineCallbacks
@inlineCallbacks
def test_redirect_follow_302(self):
_, out, _ = yield self.execute(
[self.url("/redirect-no-meta-refresh"), "-c", "response.status"]
)
assert out.strip().endswith(b"200")
@defer.inlineCallbacks
@inlineCallbacks
def test_redirect_not_follow_302(self):
_, out, _ = yield self.execute(
[
@ -74,7 +75,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase):
)
assert out.strip().endswith(b"302")
@defer.inlineCallbacks
@inlineCallbacks
def test_fetch_redirect_follow_302(self):
"""Test that calling ``fetch(url)`` follows HTTP redirects by default."""
url = self.url("/redirect-no-meta-refresh")
@ -84,7 +85,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase):
assert b"Redirecting (302)" in errout
assert b"Crawled (200)" in errout
@defer.inlineCallbacks
@inlineCallbacks
def test_fetch_redirect_not_follow_302(self):
"""Test that calling ``fetch(url, redirect=False)`` disables automatic redirects."""
url = self.url("/redirect-no-meta-refresh")
@ -93,27 +94,27 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase):
assert errcode == 0, out
assert b"Crawled (302)" in errout
@defer.inlineCallbacks
@inlineCallbacks
def test_request_replace(self):
url = self.url("/text")
code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))"
errcode, out, _ = yield self.execute(["-c", code])
assert errcode == 0, out
@defer.inlineCallbacks
@inlineCallbacks
def test_scrapy_import(self):
url = self.url("/text")
code = f"fetch(scrapy.Request('{url}'))"
errcode, out, _ = yield self.execute(["-c", code])
assert errcode == 0, out
@defer.inlineCallbacks
@inlineCallbacks
def test_local_file(self):
filepath = Path(tests_datadir, "test_site", "index.html")
_, out, _ = yield self.execute([str(filepath), "-c", "item"])
assert b"{}" in out
@defer.inlineCallbacks
@inlineCallbacks
def test_local_nofile(self):
filepath = "file:///tests/sample_data/test_site/nothinghere.html"
errcode, out, err = yield self.execute(
@ -122,16 +123,16 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase):
assert errcode == 1, out or err
assert b"No such file or directory" in err
@defer.inlineCallbacks
@inlineCallbacks
def test_dns_failures(self):
if NON_EXISTING_RESOLVABLE:
raise unittest.SkipTest("Non-existing hosts are resolvable")
pytest.skip("Non-existing hosts are resolvable")
url = "www.somedomainthatdoesntexi.st"
errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False)
assert errcode == 1, out or err
assert b"DNS lookup failed" in err
@defer.inlineCallbacks
@inlineCallbacks
def test_shell_fetch_async(self):
url = self.url("/html")
code = f"fetch('{url}')"

6
tests/test_command_version.py
View File

@ -1,6 +1,6 @@
import sys
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
import scrapy
@ -10,13 +10,13 @@ from tests.utils.testproc import ProcessTest
class TestVersionCommand(ProcessTest, unittest.TestCase):
command = "version"
@defer.inlineCallbacks
@inlineCallbacks
def test_output(self):
encoding = sys.stdout.encoding or "utf-8"
_, out, _ = yield self.execute([])
assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}"
@defer.inlineCallbacks
@inlineCallbacks
def test_verbose_output(self):
encoding = sys.stdout.encoding or "utf-8"
_, out, _ = yield self.execute(["-v"])

4
tests/test_contracts.py
View File

@ -1,7 +1,7 @@
from unittest import TextTestResult
import pytest
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.python import failure
from twisted.trial import unittest
@ -502,7 +502,7 @@ class TestContractsManager(unittest.TestCase):
assert not self.results.failures
assert self.results.errors
@defer.inlineCallbacks
@inlineCallbacks
def test_same_url(self):
class TestSameUrlSpider(Spider):
name = "test_same_url"

135
tests/test_crawl.py
View File

@ -2,7 +2,6 @@ from __future__ import annotations
import json
import logging
import unittest
from ipaddress import IPv4Address
from socket import gethostbyname
from typing import Any
@ -10,7 +9,7 @@ from urllib.parse import urlparse
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
@ -67,21 +66,21 @@ class TestCrawl(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def test_follow_all(self):
crawler = get_crawler(FollowAllSpider)
yield crawler.crawl(mockserver=self.mockserver)
assert len(crawler.spider.urls_visited) == 11 # 10 + start_url
@defer.inlineCallbacks
@inlineCallbacks
def test_fixed_delay(self):
yield self._test_delay(total=3, delay=0.2)
@defer.inlineCallbacks
@inlineCallbacks
def test_randomized_delay(self):
yield self._test_delay(total=3, delay=0.1, randomize=True)
@defer.inlineCallbacks
@inlineCallbacks
def _test_delay(self, total, delay, randomize=False):
crawl_kwargs = {
"maxlatency": delay * 2,
@ -110,7 +109,7 @@ class TestCrawl(TestCase):
average = total_time / (len(times) - 1)
assert average <= delay / tolerance, "test total or delay values are too small"
@defer.inlineCallbacks
@inlineCallbacks
def test_timeout_success(self):
crawler = get_crawler(DelaySpider)
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
@ -118,7 +117,7 @@ class TestCrawl(TestCase):
assert crawler.spider.t2 > 0
assert crawler.spider.t2 > crawler.spider.t1
@defer.inlineCallbacks
@inlineCallbacks
def test_timeout_failure(self):
crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
@ -135,7 +134,7 @@ class TestCrawl(TestCase):
assert crawler.spider.t2_err > 0
assert crawler.spider.t2_err > crawler.spider.t1
@defer.inlineCallbacks
@inlineCallbacks
def test_retry_503(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
@ -144,7 +143,7 @@ class TestCrawl(TestCase):
)
self._assert_retried(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_retry_conn_failed(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
@ -153,10 +152,10 @@ class TestCrawl(TestCase):
)
self._assert_retried(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_retry_dns_error(self):
if NON_EXISTING_RESOLVABLE:
raise unittest.SkipTest("Non-existing hosts are resolvable")
pytest.skip("Non-existing hosts are resolvable")
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
# try to fetch the homepage of a nonexistent domain
@ -165,7 +164,7 @@ class TestCrawl(TestCase):
)
self._assert_retried(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_start_bug_before_yield(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenStartSpider)
@ -176,7 +175,7 @@ class TestCrawl(TestCase):
assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError
@defer.inlineCallbacks
@inlineCallbacks
def test_start_bug_yielding(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenStartSpider)
@ -187,7 +186,7 @@ class TestCrawl(TestCase):
assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError
@defer.inlineCallbacks
@inlineCallbacks
def test_start_items(self):
items = []
@ -202,7 +201,7 @@ class TestCrawl(TestCase):
assert len(log.records) == 0
assert items == [{"name": "test item"}]
@defer.inlineCallbacks
@inlineCallbacks
def test_start_unsupported_output(self):
"""Anything that is not a request is assumed to be an item, avoiding a
potentially expensive call to itemadapter.is_item(), and letting
@ -223,7 +222,7 @@ class TestCrawl(TestCase):
assert len(items) == 3
assert not any(isinstance(item, Request) for item in items)
@defer.inlineCallbacks
@inlineCallbacks
def test_start_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = get_crawler(DuplicateStartSpider, settings)
@ -241,7 +240,7 @@ class TestCrawl(TestCase):
)
assert crawler.spider.visited == 3
@defer.inlineCallbacks
@inlineCallbacks
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
@ -275,7 +274,7 @@ with multiples lines
)
assert str(log).count("Got response 200") == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_retry_conn_lost(self):
# connection lost after receiving data
crawler = get_crawler(SimpleSpider)
@ -285,7 +284,7 @@ with multiples lines
)
self._assert_retried(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_retry_conn_aborted(self):
# connection lost before receiving data
crawler = get_crawler(SimpleSpider)
@ -299,7 +298,7 @@ with multiples lines
assert str(log).count("Retrying") == 2
assert str(log).count("Gave up retrying") == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_referer_header(self):
"""Referer header is set by RefererMiddleware unless it is already set"""
req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1)
@ -327,7 +326,7 @@ with multiples lines
echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body))
assert echo3["headers"].get("Referer") == ["http://example.com"]
@defer.inlineCallbacks
@inlineCallbacks
def test_engine_status(self):
from scrapy.utils.engine import get_engine_status
@ -345,7 +344,7 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_format_engine_status(self):
from scrapy.utils.engine import format_engine_status
@ -370,7 +369,7 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1"
@defer.inlineCallbacks
@inlineCallbacks
def test_open_spider_error_on_faulty_pipeline(self):
settings = {
"ITEM_PIPELINES": {
@ -378,15 +377,13 @@ with multiples lines
}
}
crawler = get_crawler(SimpleSpider, settings)
yield self.assertFailure(
crawler.crawl(
with pytest.raises(ZeroDivisionError):
yield crawler.crawl(
self.mockserver.url("/status?n=200"), mockserver=self.mockserver
),
ZeroDivisionError,
)
)
assert not crawler.crawling
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlerrunner_accepts_crawler(self):
crawler = get_crawler(SimpleSpider)
runner = CrawlerRunner()
@ -398,7 +395,7 @@ with multiples lines
)
assert "Got response 200" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawl_multiple(self):
runner = CrawlerRunner(get_reactor_settings())
runner.crawl(
@ -431,7 +428,7 @@ class TestCrawlSpider(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def _run_spider(self, spider_cls):
items = []
@ -446,7 +443,7 @@ class TestCrawlSpider(TestCase):
)
return log, items, crawler.stats
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_with_parse(self):
crawler = get_crawler(CrawlSpiderWithParseMethod)
with LogCapture() as log:
@ -456,7 +453,7 @@ class TestCrawlSpider(TestCase):
assert "[parse] status 201 (foo: None)" in str(log)
assert "[parse] status 202 (foo: bar)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_with_async_callback(self):
crawler = get_crawler(CrawlSpiderWithAsyncCallback)
with LogCapture() as log:
@ -466,7 +463,7 @@ class TestCrawlSpider(TestCase):
assert "[parse_async] status 201 (foo: None)" in str(log)
assert "[parse_async] status 202 (foo: bar)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_with_async_generator_callback(self):
crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
with LogCapture() as log:
@ -476,7 +473,7 @@ class TestCrawlSpider(TestCase):
assert "[parse_async_gen] status 201 (foo: None)" in str(log)
assert "[parse_async_gen] status 202 (foo: bar)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_with_errback(self):
crawler = get_crawler(CrawlSpiderWithErrback)
with LogCapture() as log:
@ -489,7 +486,7 @@ class TestCrawlSpider(TestCase):
assert "[errback] status 500" in str(log)
assert "[errback] status 501" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawlspider_process_request_cb_kwargs(self):
crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
with LogCapture() as log:
@ -499,7 +496,7 @@ class TestCrawlSpider(TestCase):
assert "[parse] status 201 (foo: process_request)" in str(log)
assert "[parse] status 202 (foo: bar)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_parse(self):
crawler = get_crawler(AsyncDefSpider)
with LogCapture() as log:
@ -509,7 +506,7 @@ class TestCrawlSpider(TestCase):
assert "Got response 200" in str(log)
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse(self):
crawler = get_crawler(
AsyncDefAsyncioSpider,
@ -524,7 +521,7 @@ class TestCrawlSpider(TestCase):
assert "Got response 200" in str(log)
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse_items_list(self):
log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
assert "Got response 200" in str(log)
@ -532,7 +529,7 @@ class TestCrawlSpider(TestCase):
assert {"id": 2} in items
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse_items_single_element(self):
items = []
@ -549,7 +546,7 @@ class TestCrawlSpider(TestCase):
assert {"foo": 42} in items
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncgen_parse(self):
log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
assert "Got response 200" in str(log)
@ -557,7 +554,7 @@ class TestCrawlSpider(TestCase):
assert itemcount == 1
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncgen_parse_loop(self):
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
assert "Got response 200" in str(log)
@ -567,7 +564,7 @@ class TestCrawlSpider(TestCase):
assert {"foo": i} in items
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncgen_parse_exc(self):
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
log = str(log)
@ -579,7 +576,7 @@ class TestCrawlSpider(TestCase):
assert {"foo": i} in items
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncgen_parse_complex(self):
_, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
itemcount = stats.get_value("item_scraped_count")
@ -591,37 +588,37 @@ class TestCrawlSpider(TestCase):
assert {"index2": i} in items
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_asyncio_parse_reqs_list(self):
log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)
for req_id in range(3):
assert f"Got response 200, req_id {req_id}" in str(log)
@pytest.mark.only_not_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_deferred_direct(self):
_, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
assert items == [{"code": 200}]
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_deferred_wrapped(self):
log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
assert items == [{"code": 200}]
@defer.inlineCallbacks
@inlineCallbacks
def test_async_def_deferred_maybe_wrapped(self):
_, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
assert items == [{"code": 200}]
@defer.inlineCallbacks
@inlineCallbacks
def test_response_ssl_certificate_none(self):
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=False)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert crawler.spider.meta["responses"][0].certificate is None
@defer.inlineCallbacks
@inlineCallbacks
def test_response_ssl_certificate(self):
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=True)
@ -634,7 +631,7 @@ class TestCrawlSpider(TestCase):
@pytest.mark.xfail(
reason="Responses with no body return early and contain no certificate"
)
@defer.inlineCallbacks
@inlineCallbacks
def test_response_ssl_certificate_empty_response(self):
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/status?n=200", is_secure=True)
@ -644,7 +641,7 @@ class TestCrawlSpider(TestCase):
assert cert.getSubject().commonName == b"localhost"
assert cert.getIssuer().commonName == b"localhost"
@defer.inlineCallbacks
@inlineCallbacks
def test_dns_server_ip_address_none(self):
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/status?n=200")
@ -652,7 +649,7 @@ class TestCrawlSpider(TestCase):
ip_address = crawler.spider.meta["responses"][0].ip_address
assert ip_address is None
@defer.inlineCallbacks
@inlineCallbacks
def test_dns_server_ip_address(self):
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test")
@ -662,7 +659,7 @@ class TestCrawlSpider(TestCase):
assert isinstance(ip_address, IPv4Address)
assert str(ip_address) == gethostbyname(expected_netloc)
@defer.inlineCallbacks
@inlineCallbacks
def test_bytes_received_stop_download_callback(self):
crawler = get_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -676,7 +673,7 @@ class TestCrawlSpider(TestCase):
< crawler.spider.full_response_length
)
@defer.inlineCallbacks
@inlineCallbacks
def test_bytes_received_stop_download_errback(self):
crawler = get_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -692,7 +689,7 @@ class TestCrawlSpider(TestCase):
< crawler.spider.full_response_length
)
@defer.inlineCallbacks
@inlineCallbacks
def test_headers_received_stop_download_callback(self):
crawler = get_crawler(HeadersReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -702,7 +699,7 @@ class TestCrawlSpider(TestCase):
"headers_received"
)
@defer.inlineCallbacks
@inlineCallbacks
def test_headers_received_stop_download_errback(self):
crawler = get_crawler(HeadersReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -714,7 +711,7 @@ class TestCrawlSpider(TestCase):
"failure"
].value.response.headers == crawler.spider.meta.get("headers_received")
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback(self):
failures = []
@ -731,7 +728,7 @@ class TestCrawlSpider(TestCase):
assert "HTTP status code is not handled or not allowed" in str(log)
assert "Spider error processing" not in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_silence(self):
failures = []
@ -747,7 +744,7 @@ class TestCrawlSpider(TestCase):
assert "HTTP status code is not handled or not allowed" not in str(log)
assert "Spider error processing" not in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_exception(self):
def eb(failure: Failure) -> None:
raise ValueError("foo")
@ -759,7 +756,7 @@ class TestCrawlSpider(TestCase):
)
assert "Spider error processing" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_item(self):
def eb(failure: Failure) -> Any:
return {"foo": "bar"}
@ -773,7 +770,7 @@ class TestCrawlSpider(TestCase):
assert "Spider error processing" not in str(log)
assert "'item_scraped_count': 1" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_request(self):
def eb(failure: Failure) -> Request:
return Request(self.mockserver.url("/"))
@ -787,7 +784,7 @@ class TestCrawlSpider(TestCase):
assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_downloader_error(self):
failures = []
@ -804,7 +801,7 @@ class TestCrawlSpider(TestCase):
assert "Error downloading" in str(log)
assert "Spider error processing" not in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_downloader_error_exception(self):
def eb(failure: Failure) -> None:
raise ValueError("foo")
@ -817,7 +814,7 @@ class TestCrawlSpider(TestCase):
assert "Error downloading" in str(log)
assert "Spider error processing" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_downloader_error_item(self):
def eb(failure: Failure) -> Any:
return {"foo": "bar"}
@ -831,7 +828,7 @@ class TestCrawlSpider(TestCase):
assert "Spider error processing" not in str(log)
assert "'item_scraped_count': 1" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_spider_errback_downloader_error_request(self):
def eb(failure: Failure) -> Request:
return Request(self.mockserver.url("/"))
@ -845,7 +842,7 @@ class TestCrawlSpider(TestCase):
assert "Spider error processing" not in str(log)
assert "Crawled (200)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_raise_closespider(self):
def cb(response):
raise CloseSpider
@ -856,7 +853,7 @@ class TestCrawlSpider(TestCase):
assert "Closing spider (cancelled)" in str(log)
assert "Spider error processing" not in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_raise_closespider_reason(self):
def cb(response):
raise CloseSpider("my_reason")

12
tests/test_dependencies.py
View File

@ -1,25 +1,13 @@
import os
import re
from configparser import ConfigParser
from importlib import import_module
from pathlib import Path
import pytest
from twisted import version as twisted_version
from twisted.trial import unittest
class TestScrapyUtils:
def test_required_openssl_version(self):
try:
module = import_module("OpenSSL")
except ImportError:
raise unittest.SkipTest("OpenSSL is not available")
if hasattr(module, "__version__"):
installed_version = [int(x) for x in module.__version__.split(".")[:2]]
assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
def test_pinned_twisted_version(self):
"""When running tests within a Tox environment with pinned
dependencies, make sure that the version of Twisted is the pinned

2
tests/test_downloader_handlers.py
View File

@ -440,7 +440,7 @@ class TestFTPBase(unittest.TestCase):
class TestFTP(TestFTPBase):
def test_invalid_credentials(self):
if self.reactor_pytest != "default" and sys.platform == "win32":
raise unittest.SkipTest(
pytest.skip(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
)
from twisted.protocols.ftp import ConnectionLost

12
tests/test_downloader_handlers_http_base.py
View File

@ -14,7 +14,7 @@ from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer, error
from twisted.internet.defer import maybeDeferred
from twisted.internet.defer import inlineCallbacks, maybeDeferred
from twisted.protocols.policies import WrappingFactory
from twisted.trial import unittest
from twisted.web import resource, server, static, util
@ -186,7 +186,7 @@ class TestHttpBase(unittest.TestCase, ABC):
self.download_handler_cls, get_crawler()
)
@defer.inlineCallbacks
@inlineCallbacks
def tearDown(self):
yield self.port.stopListening()
if hasattr(self.download_handler, "close"):
@ -229,7 +229,7 @@ class TestHttpBase(unittest.TestCase, ABC):
async def test_timeout_download_from_spider_nodata_rcvd(self):
if self.reactor_pytest != "default" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
raise unittest.SkipTest(
pytest.skip(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
)
@ -245,7 +245,7 @@ class TestHttpBase(unittest.TestCase, ABC):
async def test_timeout_download_from_spider_server_hangs(self):
if self.reactor_pytest != "default" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
raise unittest.SkipTest(
pytest.skip(
"This test produces DirtyReactorAggregateError on Windows with asyncio"
)
# client connects, server send headers and some body bytes but hangs
@ -531,7 +531,7 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC):
crawler = get_crawler(settings_dict=settings_dict)
self.download_handler = build_from_crawler(self.download_handler_cls, crawler)
@defer.inlineCallbacks
@inlineCallbacks
def tearDown(self):
yield self.port.stopListening()
if hasattr(self.download_handler, "close"):
@ -665,7 +665,7 @@ class TestHttpProxyBase(unittest.TestCase, ABC):
self.download_handler_cls, get_crawler()
)
@defer.inlineCallbacks
@inlineCallbacks
def tearDown(self):
yield self.port.stopListening()
if hasattr(self.download_handler, "close"):

35
tests/test_downloadermiddleware_httpcompression.py
View File

@ -2,7 +2,6 @@ from gzip import GzipFile
from io import BytesIO
from logging import WARNING
from pathlib import Path
from unittest import SkipTest
import pytest
from testfixtures import LogCapture
@ -130,7 +129,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
response = self._getresponse("br")
request = response.request
assert response.headers["Content-Encoding"] == b"br"
@ -146,11 +145,11 @@ class TestHttpCompression:
try:
import brotli # noqa: F401
raise SkipTest("Requires not having brotli support")
pytest.skip("Requires not having brotli support")
except ImportError:
import brotlicffi # noqa: F401
raise SkipTest("Requires not having brotli support")
pytest.skip("Requires not having brotli support")
except ImportError:
pass
response = self._getresponse("br")
@ -180,7 +179,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
raw_content = None
for check_key in FORMAT:
if not check_key.startswith("zstd-"):
@ -201,7 +200,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
raise SkipTest("Requires not having zstandard support")
pytest.skip("Requires not having zstandard support")
except ImportError:
pass
response = self._getresponse("zstd-static-content-size")
@ -520,7 +519,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
self._test_compression_bomb_setting("br")
def test_compression_bomb_setting_deflate(self):
@ -533,7 +532,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
self._test_compression_bomb_setting("zstd")
def _test_compression_bomb_spider_attr(self, compression_id):
@ -556,7 +555,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
self._test_compression_bomb_spider_attr("br")
def test_compression_bomb_spider_attr_deflate(self):
@ -569,7 +568,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
@ -590,7 +589,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
self._test_compression_bomb_request_meta("br")
def test_compression_bomb_request_meta_deflate(self):
@ -603,7 +602,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
self._test_compression_bomb_request_meta("zstd")
def _test_download_warnsize_setting(self, compression_id):
@ -639,7 +638,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
self._test_download_warnsize_setting("br")
def test_download_warnsize_setting_deflate(self):
@ -652,7 +651,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
self._test_download_warnsize_setting("zstd")
def _test_download_warnsize_spider_attr(self, compression_id):
@ -690,7 +689,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
self._test_download_warnsize_spider_attr("br")
def test_download_warnsize_spider_attr_deflate(self):
@ -703,7 +702,7 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
self._test_download_warnsize_spider_attr("zstd")
def _test_download_warnsize_request_meta(self, compression_id):
@ -739,7 +738,7 @@ class TestHttpCompression:
except ImportError:
import brotlicffi # noqa: F401
except ImportError:
raise SkipTest("no brotli")
pytest.skip("no brotli")
self._test_download_warnsize_request_meta("br")
def test_download_warnsize_request_meta_deflate(self):
@ -752,5 +751,5 @@ class TestHttpCompression:
try:
import zstandard # noqa: F401
except ImportError:
raise SkipTest("no zstd support (zstandard)")
pytest.skip("no zstd support (zstandard)")
self._test_download_warnsize_request_meta("zstd")

8
tests/test_downloadermiddleware_robotstxt.py
View File

@ -235,12 +235,10 @@ Disallow: /some/randome/page.html
self, request: Request, middleware: RobotsTxtMiddleware
) -> None:
spider = None # not actually used
await maybe_deferred_to_future(
self.assertFailure(
middleware.process_request(request, spider), # type: ignore[arg-type]
IgnoreRequest,
with pytest.raises(IgnoreRequest):
await maybe_deferred_to_future(
maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload]
)
)
def assertRobotsTxtRequested(self, base_url: str) -> None:
calls = self.crawler.engine.download.call_args_list

4
tests/test_downloaderslotssettings.py
View File

@ -1,6 +1,6 @@
import time
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy import Request
@ -62,7 +62,7 @@ class CrawlTestCase(TestCase):
def setUp(self):
self.runner = CrawlerRunner()
@defer.inlineCallbacks
@inlineCallbacks
def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
yield crawler.crawl(mockserver=self.mockserver)

24
tests/test_engine.py
View File

@ -26,6 +26,7 @@ import pytest
from itemadapter import ItemAdapter
from pydispatch import dispatcher
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from twisted.web import server, static, util
@ -390,7 +391,7 @@ class TestEngineBase(unittest.TestCase):
class TestEngine(TestEngineBase):
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler(self):
for spider in (
MySpider,
@ -407,20 +408,20 @@ class TestEngine(TestEngineBase):
self._assert_signals_caught(run)
self._assert_bytes_received(run)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_dupefilter(self):
run = CrawlerRun(DupeFilterSpider)
yield run.run()
self._assert_scheduled_requests(run, count=8)
self._assert_dropped_requests(run)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_itemerror(self):
run = CrawlerRun(ItemZeroDivisionErrorSpider)
yield run.run()
self._assert_items_error(run)
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler_change_close_reason_on_idle(self):
run = CrawlerRun(ChangeCloseReasonSpider)
yield run.run()
@ -429,7 +430,7 @@ class TestEngine(TestEngineBase):
"reason": "custom_reason",
} == run.signals_caught[signals.spider_closed]
@defer.inlineCallbacks
@inlineCallbacks
def test_close_downloader(self):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.close()
@ -447,19 +448,14 @@ class TestEngine(TestEngineBase):
get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None
)
@defer.inlineCallbacks
@inlineCallbacks
def test_start_already_running_exception(self):
e = ExecutionEngine(get_crawler(MySpider), lambda _: None)
yield e.open_spider(MySpider(), [])
e.start()
def cb(exc: BaseException) -> None:
assert str(exc), "Engine already running"
try:
yield self.assertFailure(e.start(), RuntimeError).addBoth(cb)
finally:
yield e.stop()
with pytest.raises(RuntimeError, match="Engine already running"):
yield e.start()
yield e.stop()
def test_short_timeout(self):
args = (

4
tests/test_engine_stop_download_bytes.py
View File

@ -1,5 +1,5 @@
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from scrapy.exceptions import StopDownload
from tests.test_engine import (
@ -19,7 +19,7 @@ class BytesReceivedCrawlerRun(CrawlerRun):
class TestBytesReceivedEngine(TestEngineBase):
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler(self):
for spider in (
MySpider,

4
tests/test_engine_stop_download_headers.py
View File

@ -1,5 +1,5 @@
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from scrapy.exceptions import StopDownload
from tests.test_engine import (
@ -19,7 +19,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun):
class TestHeadersReceivedEngine(TestEngineBase):
@defer.inlineCallbacks
@inlineCallbacks
def test_crawler(self):
for spider in (
MySpider,

3
tests/test_exporters.py
View File

@ -4,7 +4,6 @@ import marshal
import pickle
import re
import tempfile
import unittest
from datetime import datetime
from io import BytesIO
from typing import Any
@ -662,7 +661,7 @@ class TestCustomExporterItem:
def setup_method(self):
if self.item_class is None:
raise unittest.SkipTest("item class is None")
pytest.skip("item class is None")
def test_exporter_custom_serializer(self):
class CustomItemExporter(BaseItemExporter):

3
tests/test_extension_periodic_log.py
View File

@ -1,7 +1,6 @@
from __future__ import annotations
import datetime
import unittest
from typing import Any, Callable
from scrapy.extensions.periodic_log import PeriodicLog
@ -66,7 +65,7 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
return CustomPeriodicLog.from_crawler(crawler)
class TestPeriodicLog(unittest.TestCase):
class TestPeriodicLog:
def test_extension_enabled(self):
# Expected that settings for this extension loaded successfully
# And on certain conditions - extension raising NotConfigured

12
tests/test_extension_telnet.py
View File

@ -1,6 +1,7 @@
import pytest
from twisted.conch.telnet import ITelnetProtocol
from twisted.cred import credentials
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from scrapy.extensions.telnet import TelnetConsole
@ -21,15 +22,16 @@ class TelnetExtensionTest(unittest.TestCase):
return console, portal
@defer.inlineCallbacks
@inlineCallbacks
def test_bad_credentials(self):
console, portal = self._get_console_and_portal()
creds = credentials.UsernamePassword(b"username", b"password")
d = portal.login(creds, None, ITelnetProtocol)
yield self.assertFailure(d, ValueError)
with pytest.raises(ValueError, match="Invalid credentials"):
yield d
console.stop_listening()
@defer.inlineCallbacks
@inlineCallbacks
def test_good_credentials(self):
console, portal = self._get_console_and_portal()
creds = credentials.UsernamePassword(
@ -39,7 +41,7 @@ class TelnetExtensionTest(unittest.TestCase):
yield d
console.stop_listening()
@defer.inlineCallbacks
@inlineCallbacks
def test_custom_credentials(self):
settings = {
"TELNETCONSOLE_USERNAME": "user",

179
tests/test_feedexport.py
View File

@ -26,6 +26,7 @@ import lxml.etree
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from w3lib.url import file_uri_to_path, path_to_file_uri
from zope.interface import implementer
@ -131,7 +132,7 @@ class TestFileFeedStorage(unittest.TestCase):
FileFeedStorage(str(path), feed_options={"overwrite": True}), path
)
@defer.inlineCallbacks
@inlineCallbacks
def _assert_stores(self, storage, path: Path, expected_content=b"content"):
spider = scrapy.Spider("default")
file = storage.open(spider)
@ -172,7 +173,7 @@ class TestFTPFeedStorage(unittest.TestCase):
finally:
path.unlink()
@defer.inlineCallbacks
@inlineCallbacks
def test_append(self):
with MockFTPServer() as ftp_server:
filename = "file"
@ -182,7 +183,7 @@ class TestFTPFeedStorage(unittest.TestCase):
yield self._store(url, b"bar", feed_options=feed_options)
self._assert_stored(ftp_server.path / filename, b"foobar")
@defer.inlineCallbacks
@inlineCallbacks
def test_overwrite(self):
with MockFTPServer() as ftp_server:
filename = "file"
@ -191,7 +192,7 @@ class TestFTPFeedStorage(unittest.TestCase):
yield self._store(url, b"bar")
self._assert_stored(ftp_server.path / filename, b"bar")
@defer.inlineCallbacks
@inlineCallbacks
def test_append_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
@ -202,7 +203,7 @@ class TestFTPFeedStorage(unittest.TestCase):
yield self._store(url, b"bar", feed_options=feed_options, settings=settings)
self._assert_stored(ftp_server.path / filename, b"foobar")
@defer.inlineCallbacks
@inlineCallbacks
def test_overwrite_active_mode(self):
with MockFTPServer() as ftp_server:
settings = {"FEED_STORAGE_FTP_ACTIVE": True}
@ -290,7 +291,7 @@ class TestS3FeedStorage(unittest.TestCase):
assert storage.access_key == "uri_key"
assert storage.secret_key == "uri_secret"
@defer.inlineCallbacks
@inlineCallbacks
def test_store(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
@ -431,7 +432,7 @@ class TestS3FeedStorage(unittest.TestCase):
assert storage.region_name == region_name
assert storage.s3_client._client_config.region_name == region_name
@defer.inlineCallbacks
@inlineCallbacks
def test_store_without_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv",
@ -451,7 +452,7 @@ class TestS3FeedStorage(unittest.TestCase):
)
assert acl is None
@defer.inlineCallbacks
@inlineCallbacks
def test_store_with_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
@ -489,7 +490,7 @@ class TestGCSFeedStorage(unittest.TestCase):
try:
from google.cloud.storage import Client # noqa: F401
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
pytest.skip("GCSFeedStorage requires google-cloud-storage")
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
crawler = get_crawler(settings_dict=settings)
@ -503,7 +504,7 @@ class TestGCSFeedStorage(unittest.TestCase):
try:
from google.cloud.storage import Client # noqa: F401
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
pytest.skip("GCSFeedStorage requires google-cloud-storage")
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
crawler = get_crawler(settings_dict=settings)
@ -515,12 +516,12 @@ class TestGCSFeedStorage(unittest.TestCase):
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
assert storage.acl is None
@defer.inlineCallbacks
@inlineCallbacks
def test_store(self):
try:
from google.cloud.storage import Client # noqa: F401
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
pytest.skip("GCSFeedStorage requires google-cloud-storage")
uri = "gs://mybucket/export.csv"
project_id = "myproject-123"
@ -556,7 +557,7 @@ class TestGCSFeedStorage(unittest.TestCase):
class TestStdoutFeedStorage(unittest.TestCase):
@defer.inlineCallbacks
@inlineCallbacks
def test_store(self):
out = BytesIO()
storage = StdoutFeedStorage("stdout:", _stdout=out)
@ -669,7 +670,7 @@ class TestFeedExportBase(ABC, unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
@defer.inlineCallbacks
@inlineCallbacks
def exported_data(self, items, settings):
"""
Return exported data which a spider yielding ``items`` would return.
@ -684,7 +685,7 @@ class TestFeedExportBase(ABC, unittest.TestCase):
data = yield self.run_and_export(TestSpider, settings)
return data
@defer.inlineCallbacks
@inlineCallbacks
def exported_no_data(self, settings):
"""
Return exported data which a spider yielding no ``items`` would return.
@ -699,7 +700,7 @@ class TestFeedExportBase(ABC, unittest.TestCase):
data = yield self.run_and_export(TestSpider, settings)
return data
@defer.inlineCallbacks
@inlineCallbacks
def assertExported(self, items, header, rows, settings=None):
yield self.assertExportedCsv(items, header, rows, settings)
yield self.assertExportedJsonLines(items, rows, settings)
@ -770,7 +771,7 @@ class ExceptionJsonItemExporter(JsonItemExporter):
class TestFeedExport(TestFeedExportBase):
@defer.inlineCallbacks
@inlineCallbacks
def run_and_export(self, spider_cls, settings):
"""Run spider with specified settings; return exported data."""
@ -800,7 +801,7 @@ class TestFeedExport(TestFeedExportBase):
return content
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedCsv(self, items, header, rows, settings=None):
settings = settings or {}
settings.update(
@ -815,7 +816,7 @@ class TestFeedExport(TestFeedExportBase):
assert reader.fieldnames == list(header)
assert rows == list(reader)
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedJsonLines(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -830,7 +831,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{k: v for k, v in row.items() if v} for row in rows]
assert rows == parsed
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedXml(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -846,7 +847,7 @@ class TestFeedExport(TestFeedExportBase):
got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")]
assert rows == got_rows
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedMultiple(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -867,7 +868,7 @@ class TestFeedExport(TestFeedExportBase):
json_rows = json.loads(to_unicode(data["json"]))
assert rows == json_rows
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedPickle(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -884,7 +885,7 @@ class TestFeedExport(TestFeedExportBase):
result = self._load_until_eof(data["pickle"], load_func=pickle.load)
assert result == expected
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedMarshal(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -901,7 +902,7 @@ class TestFeedExport(TestFeedExportBase):
result = self._load_until_eof(data["marshal"], load_func=marshal.load)
assert result == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_stats_file_success(self):
settings = {
"FEEDS": {
@ -915,7 +916,7 @@ class TestFeedExport(TestFeedExportBase):
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_stats_file_failed(self):
settings = {
"FEEDS": {
@ -933,7 +934,7 @@ class TestFeedExport(TestFeedExportBase):
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_stats_multiple_file(self):
settings = {
"FEEDS": {
@ -955,7 +956,7 @@ class TestFeedExport(TestFeedExportBase):
crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1
)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items(self):
# feed exporters use field names from Item
items = [
@ -969,7 +970,7 @@ class TestFeedExport(TestFeedExportBase):
header = self.MyItem.fields.keys()
yield self.assertExported(items, header, rows)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"):
settings = {
@ -981,7 +982,7 @@ class TestFeedExport(TestFeedExportBase):
data = yield self.exported_no_data(settings)
assert data[fmt] is None
@defer.inlineCallbacks
@inlineCallbacks
def test_start_finish_exporting_items(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1001,7 +1002,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@defer.inlineCallbacks
@inlineCallbacks
def test_start_finish_exporting_no_items(self):
items = []
settings = {
@ -1019,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@defer.inlineCallbacks
@inlineCallbacks
def test_start_finish_exporting_items_exception(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1040,7 +1041,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@defer.inlineCallbacks
@inlineCallbacks
def test_start_finish_exporting_no_items_exception(self):
items = []
settings = {
@ -1059,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
@defer.inlineCallbacks
@inlineCallbacks
def test_export_no_items_store_empty(self):
formats = (
("json", b"[]"),
@ -1079,7 +1080,7 @@ class TestFeedExport(TestFeedExportBase):
data = yield self.exported_no_data(settings)
assert expctd == data[fmt]
@defer.inlineCallbacks
@inlineCallbacks
def test_export_no_items_multiple_feeds(self):
"""Make sure that `storage.store` is called for every feed."""
settings = {
@ -1097,7 +1098,7 @@ class TestFeedExport(TestFeedExportBase):
assert str(log).count("Storage.store is called") == 0
@defer.inlineCallbacks
@inlineCallbacks
def test_export_multiple_item_classes(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1119,7 +1120,7 @@ class TestFeedExport(TestFeedExportBase):
yield self.assertExportedCsv(items, header, rows_csv)
yield self.assertExportedJsonLines(items, rows_jl)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_empty_field_list(self):
# FEED_EXPORT_FIELDS==[] means the same as default None
items = [{"foo": "bar"}]
@ -1129,7 +1130,7 @@ class TestFeedExport(TestFeedExportBase):
yield self.assertExportedCsv(items, header, rows)
yield self.assertExportedJsonLines(items, rows, settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_field_list(self):
items = [{"foo": "bar"}]
header = ["foo", "baz"]
@ -1137,7 +1138,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header}
yield self.assertExported(items, header, rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_comma_separated_field_list(self):
items = [{"foo": "bar"}]
header = ["foo", "baz"]
@ -1145,7 +1146,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": ",".join(header)}
yield self.assertExported(items, header, rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_json_field_list(self):
items = [{"foo": "bar"}]
header = ["foo", "baz"]
@ -1153,7 +1154,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
yield self.assertExported(items, header, rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_field_names(self):
items = [{"foo": "bar"}]
header = {"foo": "Foo"}
@ -1161,7 +1162,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header}
yield self.assertExported(items, list(header.values()), rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_dict_field_names(self):
items = [{"foo": "bar"}]
header = {
@ -1172,7 +1173,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": header}
yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items_json_field_names(self):
items = [{"foo": "bar"}]
header = {"foo": "Foo"}
@ -1180,7 +1181,7 @@ class TestFeedExport(TestFeedExportBase):
settings = {"FEED_EXPORT_FIELDS": json.dumps(header)}
yield self.assertExported(items, list(header.values()), rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_based_on_item_classes(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1226,7 +1227,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items():
assert data[fmt] == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_export_based_on_custom_filters(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -1285,7 +1286,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items():
assert data[fmt] == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as
# a header for CSV, and all fields are used for JSON Lines.
@ -1298,7 +1299,7 @@ class TestFeedExport(TestFeedExportBase):
yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv)
yield self.assertExportedJsonLines(items, rows_jl)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_tuple(self):
items = [
{"foo": "bar1", "egg": "spam1"},
@ -1309,7 +1310,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}]
yield self.assertExported(items, ["foo", "baz"], rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_feed_export_fields(self):
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
@ -1335,7 +1336,7 @@ class TestFeedExport(TestFeedExportBase):
rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}]
yield self.assertExported(items, ["egg", "baz"], rows, settings=settings)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_encoding(self):
items = [{"foo": "Test\xd6"}]
@ -1380,7 +1381,7 @@ class TestFeedExport(TestFeedExportBase):
data = yield self.exported_data(items, settings)
assert data[fmt] == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_export_multiple_configs(self):
items = [{"foo": "FOO", "bar": "BAR"}]
@ -1420,7 +1421,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt, expected in formats.items():
assert data[fmt] == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_export_indentation(self):
items = [
{"foo": ["bar"]},
@ -1576,7 +1577,7 @@ class TestFeedExport(TestFeedExportBase):
data = yield self.exported_data(items, settings)
assert data[row["format"]] == row["expected"]
@defer.inlineCallbacks
@inlineCallbacks
def test_init_exporters_storages_with_crawler(self):
settings = {
"FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter},
@ -1589,7 +1590,7 @@ class TestFeedExport(TestFeedExportBase):
assert FromCrawlerCsvItemExporter.init_with_crawler
assert FromCrawlerFileFeedStorage.init_with_crawler
@defer.inlineCallbacks
@inlineCallbacks
def test_str_uri(self):
settings = {
"FEED_STORE_EMPTY": True,
@ -1598,7 +1599,7 @@ class TestFeedExport(TestFeedExportBase):
data = yield self.exported_no_data(settings)
assert data["csv"] == b""
@defer.inlineCallbacks
@inlineCallbacks
def test_multiple_feeds_success_logs_blocking_feed_storage(self):
settings = {
"FEEDS": {
@ -1619,7 +1620,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt in ["json", "xml", "csv"]:
assert f"Stored {fmt} feed (2 items)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
settings = {
"FEEDS": {
@ -1640,7 +1641,7 @@ class TestFeedExport(TestFeedExportBase):
for fmt in ["json", "xml", "csv"]:
assert f"Error storing {fmt} feed (2 items)" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_extend_kwargs(self):
items = [{"foo": "FOO", "bar": "BAR"}]
@ -1677,7 +1678,7 @@ class TestFeedExport(TestFeedExportBase):
data = yield self.exported_data(items, settings)
assert data[feed_options["format"]] == row["expected"]
@defer.inlineCallbacks
@inlineCallbacks
def test_storage_file_no_postprocessing(self):
@implementer(IFeedStorage)
class Storage:
@ -1699,7 +1700,7 @@ class TestFeedExport(TestFeedExportBase):
yield self.exported_no_data(settings)
assert Storage.open_file is Storage.store_file
@defer.inlineCallbacks
@inlineCallbacks
def test_storage_file_postprocessing(self):
@implementer(IFeedStorage)
class Storage:
@ -1752,7 +1753,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
def _named_tempfile(self, name) -> str:
return str(Path(self.temp_dir, name))
@defer.inlineCallbacks
@inlineCallbacks
def run_and_export(self, spider_cls, settings):
"""Run spider with specified settings; return exported data with filename."""
@ -1796,7 +1797,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data_stream.seek(0)
return data_stream.read()
@defer.inlineCallbacks
@inlineCallbacks
def test_gzip_plugin(self):
filename = self._named_tempfile("gzip_file")
@ -1815,7 +1816,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except OSError:
pytest.fail("Received invalid gzip data.")
@defer.inlineCallbacks
@inlineCallbacks
def test_gzip_plugin_compresslevel(self):
filename_to_compressed = {
self._named_tempfile("compresslevel_0"): self.get_gzip_compressed(
@ -1852,7 +1853,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_gzip_plugin_mtime(self):
filename_to_compressed = {
self._named_tempfile("mtime_123"): self.get_gzip_compressed(
@ -1887,7 +1888,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_gzip_plugin_filename(self):
filename_to_compressed = {
self._named_tempfile("filename_FILE1"): self.get_gzip_compressed(
@ -1922,7 +1923,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_lzma_plugin(self):
filename = self._named_tempfile("lzma_file")
@ -1941,7 +1942,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except lzma.LZMAError:
pytest.fail("Received invalid lzma data.")
@defer.inlineCallbacks
@inlineCallbacks
def test_lzma_plugin_format(self):
filename_to_compressed = {
self._named_tempfile("format_FORMAT_XZ"): lzma.compress(
@ -1974,7 +1975,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_lzma_plugin_check(self):
filename_to_compressed = {
self._named_tempfile("check_CHECK_NONE"): lzma.compress(
@ -2007,7 +2008,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_lzma_plugin_preset(self):
filename_to_compressed = {
self._named_tempfile("preset_PRESET_0"): lzma.compress(
@ -2040,11 +2041,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_lzma_plugin_filters(self):
if "PyPy" in sys.version:
# https://foss.heptapod.net/pypy/pypy/-/issues/3527
raise unittest.SkipTest("lzma filters doesn't work in PyPy")
pytest.skip("lzma filters doesn't work in PyPy")
filters = [{"id": lzma.FILTER_LZMA2}]
compressed = lzma.compress(self.expected, filters=filters)
@ -2065,7 +2066,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
result = lzma.decompress(data[filename])
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_bz2_plugin(self):
filename = self._named_tempfile("bz2_file")
@ -2084,7 +2085,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
except OSError:
pytest.fail("Received invalid bz2 data.")
@defer.inlineCallbacks
@inlineCallbacks
def test_bz2_plugin_compresslevel(self):
filename_to_compressed = {
self._named_tempfile("compresslevel_1"): bz2.compress(
@ -2117,7 +2118,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
assert compressed == data[filename]
assert result == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_custom_plugin(self):
filename = self._named_tempfile("csv_file")
@ -2133,7 +2134,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data = yield self.exported_data(self.items, settings)
assert data[filename] == self.expected
@defer.inlineCallbacks
@inlineCallbacks
def test_custom_plugin_with_parameter(self):
expected = b"foo\r\n\nbar\r\n\n"
filename = self._named_tempfile("newline")
@ -2151,7 +2152,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
data = yield self.exported_data(self.items, settings)
assert data[filename] == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_custom_plugin_with_compression(self):
expected = b"foo\r\n\nbar\r\n\n"
@ -2196,7 +2197,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
result = decompressor(data[filename])
assert result == expected
@defer.inlineCallbacks
@inlineCallbacks
def test_exports_compatibility_with_postproc(self):
import marshal
import pickle
@ -2254,7 +2255,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
class TestBatchDeliveries(TestFeedExportBase):
_file_mark = "_%(batch_time)s_#%(batch_id)02d_"
@defer.inlineCallbacks
@inlineCallbacks
def run_and_export(self, spider_cls, settings):
"""Run spider with specified settings; return exported data."""
@ -2276,7 +2277,7 @@ class TestBatchDeliveries(TestFeedExportBase):
content[feed["format"]].append(file.read_bytes())
return content
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedJsonLines(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -2298,7 +2299,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedCsv(self, items, header, rows, settings=None):
settings = settings or {}
settings.update(
@ -2318,7 +2319,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert list(got_batch) == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedXml(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -2339,7 +2340,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedMultiple(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -2371,7 +2372,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:]
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedPickle(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -2393,7 +2394,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def assertExportedMarshal(self, items, rows, settings=None):
settings = settings or {}
settings.update(
@ -2415,7 +2416,7 @@ class TestBatchDeliveries(TestFeedExportBase):
expected_batch, rows = rows[:batch_size], rows[batch_size:]
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def test_export_items(self):
"""Test partial deliveries in all supported formats"""
items = [
@ -2444,7 +2445,7 @@ class TestBatchDeliveries(TestFeedExportBase):
with pytest.raises(NotConfigured):
FeedExporter(crawler)
@defer.inlineCallbacks
@inlineCallbacks
def test_export_no_items_not_store_empty(self):
for fmt in ("json", "jsonlines", "xml", "csv"):
settings = {
@ -2460,7 +2461,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = dict(data)
assert len(data[fmt]) == 0
@defer.inlineCallbacks
@inlineCallbacks
def test_export_no_items_store_empty(self):
formats = (
("json", b"[]"),
@ -2484,7 +2485,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = dict(data)
assert data[fmt][0] == expctd
@defer.inlineCallbacks
@inlineCallbacks
def test_export_multiple_configs(self):
items = [
{"foo": "FOO", "bar": "BAR"},
@ -2540,7 +2541,7 @@ class TestBatchDeliveries(TestFeedExportBase):
for expected_batch, got_batch in zip(expected, data[fmt]):
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def test_batch_item_count_feeds_setting(self):
items = [{"foo": "FOO"}, {"foo": "FOO1"}]
formats = {
@ -2564,7 +2565,7 @@ class TestBatchDeliveries(TestFeedExportBase):
for expected_batch, got_batch in zip(expected, data[fmt]):
assert got_batch == expected_batch
@defer.inlineCallbacks
@inlineCallbacks
def test_batch_path_differ(self):
"""
Test that the name of all batch files differ from each other.
@ -2586,7 +2587,7 @@ class TestBatchDeliveries(TestFeedExportBase):
data = yield self.exported_data(items, settings)
assert len(items) == len(data["json"])
@defer.inlineCallbacks
@inlineCallbacks
def test_stats_batch_file_success(self):
settings = {
"FEEDS": {
@ -2604,7 +2605,7 @@ class TestBatchDeliveries(TestFeedExportBase):
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
@pytest.mark.requires_boto3
@defer.inlineCallbacks
@inlineCallbacks
def test_s3_export(self):
bucket = "mybucket"
items = [

4
tests/test_http2_client_protocol.py
View File

@ -9,7 +9,7 @@ from ipaddress import IPv4Address
from pathlib import Path
from tempfile import mkdtemp
from typing import TYPE_CHECKING
from unittest import mock, skipIf
from unittest import mock
from urllib.parse import urlencode
import pytest
@ -183,7 +183,7 @@ def get_client_certificate(
return PrivateCertificate.loadPEM(pem)
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
@pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled")
class TestHttps2ClientProtocol(TestCase):
scheme = "https"
key_file = Path(__file__).parent / "keys" / "localhost.key"

6
tests/test_logformatter.py
View File

@ -2,7 +2,7 @@ import logging
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase
@ -272,7 +272,7 @@ class TestShowOrSkipMessages(TestCase):
},
}
@defer.inlineCallbacks
@inlineCallbacks
def test_show_messages(self):
crawler = get_crawler(ItemSpider, self.base_settings)
with LogCapture() as lc:
@ -281,7 +281,7 @@ class TestShowOrSkipMessages(TestCase):
assert "Crawled (200) <GET http://127.0.0.1:" in str(lc)
assert "Dropped: Ignoring item" in str(lc)
@defer.inlineCallbacks
@inlineCallbacks
def test_skip_messages(self):
settings = self.base_settings.copy()
settings["LOG_FORMATTER"] = SkipMessagesLogFormatter

12
tests/test_pipeline_crawl.py
View File

@ -6,7 +6,7 @@ from tempfile import mkdtemp
from typing import TYPE_CHECKING, Any
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from w3lib.url import add_or_replace_parameter
@ -144,7 +144,7 @@ class TestFileDownloadCrawl(TestCase):
# check that no files were written to the media store
assert not list(self.tmpmediastore.iterdir())
@defer.inlineCallbacks
@inlineCallbacks
def test_download_media(self):
crawler = self._create_crawler(MediaDownloadSpider)
with LogCapture() as log:
@ -155,7 +155,7 @@ class TestFileDownloadCrawl(TestCase):
)
self._assert_files_downloaded(self.items, str(log))
@defer.inlineCallbacks
@inlineCallbacks
def test_download_media_wrong_urls(self):
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
with LogCapture() as log:
@ -166,7 +166,7 @@ class TestFileDownloadCrawl(TestCase):
)
self._assert_files_download_failure(crawler, self.items, 404, str(log))
@defer.inlineCallbacks
@inlineCallbacks
def test_download_media_redirected_default_failure(self):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
@ -178,7 +178,7 @@ class TestFileDownloadCrawl(TestCase):
)
self._assert_files_download_failure(crawler, self.items, 302, str(log))
@defer.inlineCallbacks
@inlineCallbacks
def test_download_media_redirected_allowed(self):
settings = {
**self.settings,
@ -195,7 +195,7 @@ class TestFileDownloadCrawl(TestCase):
self._assert_files_downloaded(self.items, str(log))
assert crawler.stats.get_value("downloader/response_status_count/302") == 3
@defer.inlineCallbacks
@inlineCallbacks
def test_download_media_file_path_error(self):
cls = load_object(self.pipeline_class)

22
tests/test_pipeline_files.py
View File

@ -16,7 +16,7 @@ from urllib.parse import urlparse
import attr
import pytest
from itemadapter import ItemAdapter
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
from scrapy.http import Request, Response
@ -159,7 +159,7 @@ class TestFilesPipeline(unittest.TestCase):
fullpath = Path(self.tempdir, "some", "image", "key.jpg")
assert self.pipeline.store._get_filesystem_path(path) == fullpath
@defer.inlineCallbacks
@inlineCallbacks
def test_file_not_expired(self):
item_url = "http://example.com/file.pdf"
item = _create_item_with_files(item_url)
@ -186,7 +186,7 @@ class TestFilesPipeline(unittest.TestCase):
for p in patchers:
p.stop()
@defer.inlineCallbacks
@inlineCallbacks
def test_file_expired(self):
item_url = "http://example.com/file2.pdf"
item = _create_item_with_files(item_url)
@ -217,7 +217,7 @@ class TestFilesPipeline(unittest.TestCase):
for p in patchers:
p.stop()
@defer.inlineCallbacks
@inlineCallbacks
def test_file_cached(self):
item_url = "http://example.com/file3.pdf"
item = _create_item_with_files(item_url)
@ -537,7 +537,7 @@ class TestFilesPipelineCustomSettings:
@pytest.mark.requires_botocore
class TestS3FilesStore(unittest.TestCase):
@defer.inlineCallbacks
@inlineCallbacks
def test_persist(self):
bucket = "mybucket"
key = "export.csv"
@ -577,7 +577,7 @@ class TestS3FilesStore(unittest.TestCase):
# The call to read does not happen with Stubber
assert buffer.method_calls == [mock.call.seek(0)]
@defer.inlineCallbacks
@inlineCallbacks
def test_stat(self):
bucket = "mybucket"
key = "export.csv"
@ -614,11 +614,11 @@ class TestS3FilesStore(unittest.TestCase):
"GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"
)
class TestGCSFilesStore(unittest.TestCase):
@defer.inlineCallbacks
@inlineCallbacks
def test_persist(self):
uri = os.environ.get("GCS_TEST_FILE_URI")
if not uri:
raise unittest.SkipTest("No GCS URI available for testing")
pytest.skip("No GCS URI available for testing")
data = b"TestGCSFilesStore: \xe2\x98\x83"
buf = BytesIO(data)
meta = {"foo": "bar"}
@ -639,7 +639,7 @@ class TestGCSFilesStore(unittest.TestCase):
assert blob.content_type == "application/octet-stream"
assert expected_policy in acl
@defer.inlineCallbacks
@inlineCallbacks
def test_blob_path_consistency(self):
"""Test to make sure that paths used to store files is the same as the one used to get
already uploaded files.
@ -647,7 +647,7 @@ class TestGCSFilesStore(unittest.TestCase):
try:
import google.cloud.storage # noqa: F401
except ModuleNotFoundError:
raise unittest.SkipTest("google-cloud-storage is not installed")
pytest.skip("google-cloud-storage is not installed")
with (
mock.patch("google.cloud.storage"),
mock.patch("scrapy.pipelines.files.time"),
@ -666,7 +666,7 @@ class TestGCSFilesStore(unittest.TestCase):
class TestFTPFileStore(unittest.TestCase):
@defer.inlineCallbacks
@inlineCallbacks
def test_persist(self):
data = b"TestFTPFilesStore: \xe2\x98\x83"
buf = BytesIO(data)

13
tests/test_pipelines.py
View File

@ -1,8 +1,7 @@
import asyncio
import pytest
from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.trial import unittest
from scrapy import Request, Spider, signals
@ -100,33 +99,33 @@ class TestPipeline(unittest.TestCase):
self.items = []
return crawler
@defer.inlineCallbacks
@inlineCallbacks
def test_simple_pipeline(self):
crawler = self._create_crawler(SimplePipeline)
yield crawler.crawl(mockserver=self.mockserver)
assert len(self.items) == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_deferred_pipeline(self):
crawler = self._create_crawler(DeferredPipeline)
yield crawler.crawl(mockserver=self.mockserver)
assert len(self.items) == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_asyncdef_pipeline(self):
crawler = self._create_crawler(AsyncDefPipeline)
yield crawler.crawl(mockserver=self.mockserver)
assert len(self.items) == 1
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_asyncdef_asyncio_pipeline(self):
crawler = self._create_crawler(AsyncDefAsyncioPipeline)
yield crawler.crawl(mockserver=self.mockserver)
assert len(self.items) == 1
@pytest.mark.only_not_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_asyncdef_not_asyncio_pipeline(self):
crawler = self._create_crawler(AsyncDefNotAsyncioPipeline)
yield crawler.crawl(mockserver=self.mockserver)

8
tests/test_proxy_connect.py
View File

@ -8,7 +8,7 @@ from urllib.parse import urlsplit, urlunsplit
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.http import Request
@ -89,14 +89,14 @@ class TestProxyConnect(TestCase):
self._proxy.stop()
os.environ = self._oldenv
@defer.inlineCallbacks
@inlineCallbacks
def test_https_connect_tunnel(self):
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, log)
@defer.inlineCallbacks
@inlineCallbacks
def test_https_tunnel_auth_error(self):
os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
crawler = get_crawler(SimpleSpider)
@ -106,7 +106,7 @@ class TestProxyConnect(TestCase):
# he just sees a TunnelError.
self._assert_got_tunnel_error(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_https_tunnel_without_leak_proxy_authorization_header(self):
request = Request(self.mockserver.url("/echo", is_secure=True))
crawler = get_crawler(SingleRequestSpider)

16
tests/test_request_attribute_binding.py
View File

@ -1,5 +1,5 @@
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy import Request, signals
@ -66,7 +66,7 @@ class TestCrawl(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def test_response_200(self):
url = self.mockserver.url("/status?n=200")
crawler = get_crawler(SingleRequestSpider)
@ -74,7 +74,7 @@ class TestCrawl(TestCase):
response = crawler.spider.meta["responses"][0]
assert response.request.url == url
@defer.inlineCallbacks
@inlineCallbacks
def test_response_error(self):
for status in ("404", "500"):
url = self.mockserver.url(f"/status?n={status}")
@ -85,7 +85,7 @@ class TestCrawl(TestCase):
assert failure.request.url == url
assert response.request.url == url
@defer.inlineCallbacks
@inlineCallbacks
def test_downloader_middleware_raise_exception(self):
url = self.mockserver.url("/status?n=200")
crawler = get_crawler(
@ -101,7 +101,7 @@ class TestCrawl(TestCase):
assert failure.request.url == url
assert isinstance(failure.value, ZeroDivisionError)
@defer.inlineCallbacks
@inlineCallbacks
def test_downloader_middleware_override_request_in_process_response(self):
"""
Downloader middleware which returns a response with an specific 'request' attribute.
@ -144,7 +144,7 @@ class TestCrawl(TestCase):
),
)
@defer.inlineCallbacks
@inlineCallbacks
def test_downloader_middleware_override_in_process_exception(self):
"""
An exception is raised but caught by the next middleware, which
@ -167,7 +167,7 @@ class TestCrawl(TestCase):
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == OVERRIDDEN_URL
@defer.inlineCallbacks
@inlineCallbacks
def test_downloader_middleware_do_not_override_in_process_exception(self):
"""
An exception is raised but caught by the next middleware, which
@ -190,7 +190,7 @@ class TestCrawl(TestCase):
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == url
@defer.inlineCallbacks
@inlineCallbacks
def test_downloader_middleware_alternative_callback(self):
"""
Downloader middleware which returns a response with a

4
tests/test_request_cb_kwargs.py
View File

@ -1,5 +1,5 @@
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.http import Request
@ -161,7 +161,7 @@ class TestCallbackKeywordArguments(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def test_callback_kwargs(self):
crawler = get_crawler(KeywordArgumentsSpider)
with LogCapture() as log:

10
tests/test_request_left.py
View File

@ -1,4 +1,4 @@
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.signals import request_left_downloader
@ -34,25 +34,25 @@ class TestCatching(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def test_success(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/status?n=200"))
assert crawler.spider.caught_times == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_timeout(self):
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
assert crawler.spider.caught_times == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_disconnect(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/drop"))
assert crawler.spider.caught_times == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_noconnect(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")

6
tests/test_scheduler.py
View File

@ -7,7 +7,7 @@ from collections import deque
from typing import Any, NamedTuple
import pytest
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.core.downloader import Downloader
@ -362,11 +362,11 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase):
},
)
@defer.inlineCallbacks
@inlineCallbacks
def tearDown(self):
yield self.crawler.stop()
@defer.inlineCallbacks
@inlineCallbacks
def test_integration_downloader_aware_priority_queue(self):
with MockServer() as mockserver:
url = mockserver.url("/status?n=200", is_secure=False)

5
tests/test_scheduler_base.py
View File

@ -5,6 +5,7 @@ from urllib.parse import urljoin
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.core.scheduler import BaseScheduler
@ -118,7 +119,7 @@ class SimpleSchedulerTest(TestCase, InterfaceCheckMixin):
def setUp(self):
self.scheduler = SimpleScheduler()
@defer.inlineCallbacks
@inlineCallbacks
def test_enqueue_dequeue(self):
open_result = yield self.scheduler.open(Spider("foo"))
assert open_result == "open"
@ -147,7 +148,7 @@ class SimpleSchedulerTest(TestCase, InterfaceCheckMixin):
class MinimalSchedulerCrawlTest(TestCase):
scheduler_cls = MinimalScheduler
@defer.inlineCallbacks
@inlineCallbacks
def test_crawl(self):
with MockServer() as mockserver:
settings = {

4
tests/test_signals.py
View File

@ -1,5 +1,5 @@
import pytest
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy import Request, Spider, signals
@ -53,7 +53,7 @@ class MockServerTestCase(TestCase):
self.items.append(item)
@pytest.mark.only_asyncio
@defer.inlineCallbacks
@inlineCallbacks
def test_simple_pipeline(self):
crawler = get_crawler(ItemSpider)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)

15
tests/test_spidermiddleware.py
View File

@ -8,6 +8,7 @@ from unittest import mock
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.core.spidermw import SpiderMiddlewareManager
@ -129,7 +130,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
yield {"foo": 2}
yield {"foo": 3}
@defer.inlineCallbacks
@inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
self.crawler = get_crawler(
@ -142,7 +143,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
)
return result
@defer.inlineCallbacks
@inlineCallbacks
def _test_simple_base(
self, *mw_classes, downgrade: bool = False, start_index: int | None = None
):
@ -159,7 +160,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
ProcessSpiderOutputSimpleMiddleware in mw_classes
)
@defer.inlineCallbacks
@inlineCallbacks
def _test_asyncgen_base(
self, *mw_classes, downgrade: bool = False, start_index: int | None = None
):
@ -299,7 +300,7 @@ class ProcessSpiderOutputCoroutineMiddleware:
class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
@defer.inlineCallbacks
@inlineCallbacks
def test_non_iterable(self):
with pytest.raises(
_InvalidOutput,
@ -309,7 +310,7 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
ProcessSpiderOutputNonIterableMiddleware,
)
@defer.inlineCallbacks
@inlineCallbacks
def test_coroutine(self):
with pytest.raises(
_InvalidOutput,
@ -444,7 +445,7 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
@defer.inlineCallbacks
@inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: int | None = None):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
@ -519,7 +520,7 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
def _scrape_func(self, *args, **kwargs):
1 / 0
@defer.inlineCallbacks
@inlineCallbacks
def _test_asyncgen_nodowngrade(self, *mw_classes):
with pytest.raises(
_InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"

8
tests/test_spidermiddleware_httperror.py
View File

@ -2,7 +2,7 @@ import logging
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy.http import Request, Response
@ -182,7 +182,7 @@ class TestHttpErrorMiddlewareIntegrational(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
@ -196,7 +196,7 @@ class TestHttpErrorMiddlewareIntegrational(TestCase):
assert get_value("httperror/response_ignored_status_count/402") == 1
assert get_value("httperror/response_ignored_status_count/500") == 1
@defer.inlineCallbacks
@inlineCallbacks
def test_logging(self):
crawler = get_crawler(_HttpErrorSpider)
with LogCapture() as log:
@ -210,7 +210,7 @@ class TestHttpErrorMiddlewareIntegrational(TestCase):
assert "Ignoring response <200" not in str(log)
assert "Ignoring response <402" not in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_logging_level(self):
# HttpError logs ignored responses with level INFO
crawler = get_crawler(_HttpErrorSpider)

26
tests/test_spidermiddleware_output_chain.py
View File

@ -1,5 +1,5 @@
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.trial.unittest import TestCase
from scrapy import Request, Spider
@ -308,14 +308,14 @@ class TestSpiderMiddleware(TestCase):
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
@inlineCallbacks
def crawl_log(self, spider):
crawler = get_crawler(spider)
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver)
return log
@defer.inlineCallbacks
@inlineCallbacks
def test_recovery(self):
"""
(0) Recover from an exception in a spider callback. The final item count should be 3
@ -328,7 +328,7 @@ class TestSpiderMiddleware(TestCase):
assert str(log).count("Middleware: TabError exception caught") == 1
assert "'item_scraped_count': 3" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_recovery_asyncgen(self):
"""
Same as test_recovery but with an async callback.
@ -338,7 +338,7 @@ class TestSpiderMiddleware(TestCase):
assert str(log).count("Middleware: TabError exception caught") == 1
assert "'item_scraped_count': 3" in str(log)
@defer.inlineCallbacks
@inlineCallbacks
def test_process_spider_input_without_errback(self):
"""
(1.1) An exception from the process_spider_input chain should be caught by the
@ -348,7 +348,7 @@ class TestSpiderMiddleware(TestCase):
assert "Middleware: will raise IndexError" in str(log1)
assert "Middleware: IndexError exception caught" in str(log1)
@defer.inlineCallbacks
@inlineCallbacks
def test_process_spider_input_with_errback(self):
"""
(1.2) An exception from the process_spider_input chain should not be caught by the
@ -362,7 +362,7 @@ class TestSpiderMiddleware(TestCase):
assert "{'from': 'callback'}" not in str(log1)
assert "'item_scraped_count': 1" in str(log1)
@defer.inlineCallbacks
@inlineCallbacks
def test_generator_callback(self):
"""
(2) An exception from a spider callback (returning a generator) should
@ -373,7 +373,7 @@ class TestSpiderMiddleware(TestCase):
assert "Middleware: ImportError exception caught" in str(log2)
assert "'item_scraped_count': 2" in str(log2)
@defer.inlineCallbacks
@inlineCallbacks
def test_async_generator_callback(self):
"""
Same as test_generator_callback but with an async callback.
@ -382,7 +382,7 @@ class TestSpiderMiddleware(TestCase):
assert "Middleware: ImportError exception caught" in str(log2)
assert "'item_scraped_count': 2" in str(log2)
@defer.inlineCallbacks
@inlineCallbacks
def test_generator_callback_right_after_callback(self):
"""
(2.1) Special case of (2): Exceptions should be caught
@ -392,7 +392,7 @@ class TestSpiderMiddleware(TestCase):
assert "Middleware: ImportError exception caught" in str(log21)
assert "'item_scraped_count': 2" in str(log21)
@defer.inlineCallbacks
@inlineCallbacks
def test_not_a_generator_callback(self):
"""
(3) An exception from a spider callback (returning a list) should
@ -402,7 +402,7 @@ class TestSpiderMiddleware(TestCase):
assert "Middleware: ZeroDivisionError exception caught" in str(log3)
assert "item_scraped_count" not in str(log3)
@defer.inlineCallbacks
@inlineCallbacks
def test_not_a_generator_callback_right_after_callback(self):
"""
(3.1) Special case of (3): Exceptions should be caught
@ -414,7 +414,7 @@ class TestSpiderMiddleware(TestCase):
assert "Middleware: ZeroDivisionError exception caught" in str(log31)
assert "item_scraped_count" not in str(log31)
@defer.inlineCallbacks
@inlineCallbacks
def test_generator_output_chain(self):
"""
(4) An exception from a middleware's process_spider_output method should be sent
@ -461,7 +461,7 @@ class TestSpiderMiddleware(TestCase):
assert str(item_recovered) in str(log4)
assert "parse-second-item" not in str(log4)
@defer.inlineCallbacks
@inlineCallbacks
def test_not_a_generator_output_chain(self):
"""
(5) An exception from a middleware's process_spider_output method should be sent

13
tests/test_squeues_request.py
View File

@ -4,7 +4,6 @@ Queues that handle requests
import shutil
import tempfile
import unittest
import pytest
import queuelib
@ -46,7 +45,7 @@ class RequestQueueTestMixin:
def test_one_element_with_peek(self):
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
pytest.skip("The queuelib queues do not define peek")
q = self.queue()
assert len(q) == 0
assert q.peek() is None
@ -63,7 +62,7 @@ class RequestQueueTestMixin:
def test_one_element_without_peek(self):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues define peek")
pytest.skip("The queuelib queues define peek")
q = self.queue()
assert len(q) == 0
assert q.pop() is None
@ -84,7 +83,7 @@ class RequestQueueTestMixin:
class FifoQueueMixin(RequestQueueTestMixin):
def test_fifo_with_peek(self):
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
pytest.skip("The queuelib queues do not define peek")
q = self.queue()
assert len(q) == 0
assert q.peek() is None
@ -111,7 +110,7 @@ class FifoQueueMixin(RequestQueueTestMixin):
def test_fifo_without_peek(self):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
pytest.skip("The queuelib queues do not define peek")
q = self.queue()
assert len(q) == 0
assert q.pop() is None
@ -140,7 +139,7 @@ class FifoQueueMixin(RequestQueueTestMixin):
class LifoQueueMixin(RequestQueueTestMixin):
def test_lifo_with_peek(self):
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
pytest.skip("The queuelib queues do not define peek")
q = self.queue()
assert len(q) == 0
assert q.peek() is None
@ -167,7 +166,7 @@ class LifoQueueMixin(RequestQueueTestMixin):
def test_lifo_without_peek(self):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("The queuelib queues do not define peek")
pytest.skip("The queuelib queues do not define peek")
q = self.queue()
assert len(q) == 0
assert q.pop() is None

13
tests/test_utils_log.py
View File

@ -4,7 +4,6 @@ import json
import logging
import re
import sys
import unittest
from io import StringIO
from typing import TYPE_CHECKING, Any
@ -100,21 +99,19 @@ class TestLogCounterHandler:
assert self.crawler.stats.get_value("log_count/INFO") is None
class StreamLoggerTest(unittest.TestCase):
def setUp(self):
self.stdout = sys.stdout
class TestStreamLogger:
def test_redirect(self):
logger = logging.getLogger("test")
logger.setLevel(logging.WARNING)
old_stdout = sys.stdout
sys.stdout = StreamLogger(logger, logging.ERROR)
def tearDown(self):
sys.stdout = self.stdout
def test_redirect(self):
with LogCapture() as log:
print("test log msg")
log.check(("test", "ERROR", "test log msg"))
sys.stdout = old_stdout
@pytest.mark.parametrize(
("base_extra", "log_extra", "expected_extra"),

3
tests/test_utils_signal.py
View File

@ -4,6 +4,7 @@ import pytest
from pydispatch import dispatcher
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
from twisted.trial import unittest
@ -17,7 +18,7 @@ from scrapy.utils.test import get_from_asyncio_queue
class TestSendCatchLog(unittest.TestCase):
@defer.inlineCallbacks
@inlineCallbacks
def test_send_catch_log(self):
test_signal = object()
handlers_called = set()

4
tests/test_utils_trackref.py
View File

@ -2,7 +2,7 @@ from io import StringIO
from time import sleep, time
from unittest import mock
from twisted.trial.unittest import SkipTest
import pytest
from scrapy.utils import trackref
@ -71,7 +71,7 @@ Foo 1 oldest: 0s ago\n\n"""
sleep(0.01)
o3_time = time()
if o3_time <= o1_time:
raise SkipTest("time.time is not precise enough")
pytest.skip("time.time is not precise enough")
o3 = Foo() # noqa: F841
assert trackref.get_oldest("Foo") is o1

3
tests/test_utils_url.py
View File

@ -1,4 +1,3 @@
import unittest
import warnings
import pytest
@ -267,7 +266,7 @@ def create_guess_scheme_t(args):
def create_skipped_scheme_t(args):
def do_expected(self):
raise unittest.SkipTest(args[2])
pytest.skip(args[2])
return do_expected

35
tests/test_webclient.py
View File

@ -290,28 +290,22 @@ class TestWebClient(unittest.TestCase):
d.addCallback(self.assertEqual, to_bytes(f"127.0.0.1:{self.portno}"))
return d
@inlineCallbacks
def test_timeoutTriggering(self):
"""
When a non-zero timeout is passed to L{getPage} and that many
seconds elapse before the server responds to the request. the
L{Deferred} is errbacked with a L{error.TimeoutError}.
"""
finished = self.assertFailure(
getPage(self.getURL("wait"), timeout=0.000001), defer.TimeoutError
)
def cleanup(passthrough):
# Clean up the server which is hanging around not doing
# anything.
connected = list(self.wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected:
connected[0].transport.loseConnection()
return passthrough
finished.addBoth(cleanup)
return finished
with pytest.raises(defer.TimeoutError):
yield getPage(self.getURL("wait"), timeout=0.000001)
# Clean up the server which is hanging around not doing
# anything.
connected = list(self.wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected:
connected[0].transport.loseConnection()
def testNotFound(self):
return getPage(self.getURL("notsuchfile")).addCallback(self._cbNoSuchFile)
@ -384,6 +378,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
self.getURL("payload"), body=s, contextFactory=client_context_factory
).addCallback(self.assertEqual, to_bytes(s))
@inlineCallbacks
def testPayloadDisabledCipher(self):
s = "0123456789" * 10
crawler = get_crawler(
@ -392,7 +387,7 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
}
)
client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler)
d = getPage(
self.getURL("payload"), body=s, contextFactory=client_context_factory
)
return self.assertFailure(d, OpenSSL.SSL.Error)
with pytest.raises(OpenSSL.SSL.Error):
yield getPage(
self.getURL("payload"), body=s, contextFactory=client_context_factory
)