RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Address 2/3 of warnings from tests (#5561)

This commit is contained in:
Mikhail Korobov 2022-07-19 20:39:26 +05:00 committed by GitHub
parent 44580851ff
commit b103664bf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 424 additions and 468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pytest.ini
View File

@ -21,3 +21,6 @@ addopts =
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
filterwarnings =
ignore:scrapy.downloadermiddlewares.decompression is deprecated
ignore:Module scrapy.utils.reqser is deprecated

4
tests/test_contracts.py
View File

@ -5,11 +5,11 @@ from twisted.python import failure
from twisted.trial import unittest
from scrapy import FormRequest
from scrapy.crawler import CrawlerRunner
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.utils.test import get_crawler
from scrapy.contracts import ContractsManager, Contract
from scrapy.contracts.default import (
UrlContract,
@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase):
TestSameUrlSpider.parse_first.__doc__ = contract_doc
TestSameUrlSpider.parse_second.__doc__ = contract_doc
crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
crawler = get_crawler(TestSameUrlSpider)
yield crawler.crawl()
self.assertEqual(crawler.spider.visited, 2)

100
tests/test_crawl.py
View File

@ -18,6 +18,7 @@ from scrapy.exceptions import StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver import MockServer
from tests.spiders import (
@ -49,14 +50,13 @@ class CrawlTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_follow_all(self):
crawler = self.runner.create_crawler(FollowAllSpider)
crawler = get_crawler(FollowAllSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url
@ -79,7 +79,7 @@ class CrawlTestCase(TestCase):
settings = {"DOWNLOAD_DELAY": delay,
'RANDOMIZE_DOWNLOAD_DELAY': randomize}
crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
crawler = get_crawler(FollowAllSpider, settings)
yield crawler.crawl(**crawl_kwargs)
times = crawler.spider.times
total_time = times[-1] - times[0]
@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
# of ``total`` and ``delay`` values that are too small for the test
# code above to have any meaning.
settings["DOWNLOAD_DELAY"] = 0
crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
crawler = get_crawler(FollowAllSpider, settings)
yield crawler.crawl(**crawl_kwargs)
times = crawler.spider.times
total_time = times[-1] - times[0]
@ -102,7 +102,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_success(self):
crawler = self.runner.create_crawler(DelaySpider)
crawler = get_crawler(DelaySpider)
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 > 0)
@ -110,7 +110,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_timeout_failure(self):
crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
self.assertTrue(crawler.spider.t1 > 0)
self.assertTrue(crawler.spider.t2 == 0)
@ -125,14 +125,14 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_retry_503(self):
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
self._assert_retried(log)
@defer.inlineCallbacks
def test_retry_conn_failed(self):
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
self._assert_retried(log)
@ -141,7 +141,7 @@ class CrawlTestCase(TestCase):
def test_retry_dns_error(self):
if NON_EXISTING_RESOLVABLE:
raise unittest.SkipTest("Non-existing hosts are resolvable")
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
# try to fetch the homepage of a non-existent domain
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
@ -150,7 +150,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_bug_before_yield(self):
with LogCapture('scrapy', level=logging.ERROR) as log:
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
crawler = get_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
self.assertEqual(len(log.records), 1)
@ -161,7 +161,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_bug_yielding(self):
with LogCapture('scrapy', level=logging.ERROR) as log:
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
crawler = get_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
self.assertEqual(len(log.records), 1)
@ -172,7 +172,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_lazyness(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
crawler = get_crawler(BrokenStartRequestsSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(
crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@ -181,7 +181,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_start_requests_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
crawler = get_crawler(DuplicateStartRequestsSpider, settings)
yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
self.assertEqual(crawler.spider.visited, 6)
@ -210,7 +210,7 @@ Connection: close
foo body
with multiples lines
'''})
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)
self.assertEqual(str(log).count("Got response 200"), 1)
@ -218,7 +218,7 @@ with multiples lines
@defer.inlineCallbacks
def test_retry_conn_lost(self):
# connection lost after receiving data
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
self._assert_retried(log)
@ -226,7 +226,7 @@ with multiples lines
@defer.inlineCallbacks
def test_retry_conn_aborted(self):
# connection lost before receiving data
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
self._assert_retried(log)
@ -245,7 +245,7 @@ with multiples lines
req0.meta['next'] = req1
req1.meta['next'] = req2
req2.meta['next'] = req3
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=req0, mockserver=self.mockserver)
# basic asserts in case of weird communication errors
self.assertIn('responses', crawler.spider.meta)
@ -271,7 +271,7 @@ with multiples lines
def cb(response):
est.append(get_engine_status(crawler.engine))
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
self.assertEqual(len(est), 1, est)
s = dict(est[0])
@ -286,7 +286,7 @@ with multiples lines
def cb(response):
est.append(format_engine_status(crawler.engine))
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
self.assertEqual(len(est), 1, est)
est = est[0].split("\n")[2:-2] # remove header & footer
@ -317,7 +317,7 @@ with multiples lines
def start_requests(self):
raise TestError
crawler = self.runner.create_crawler(FaultySpider)
crawler = get_crawler(FaultySpider)
yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
self.assertFalse(crawler.crawling)
@ -328,26 +328,28 @@ with multiples lines
"tests.pipelines.ZeroDivisionErrorPipeline": 300,
}
}
crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider, settings)
yield self.assertFailure(
self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
ZeroDivisionError)
self.assertFalse(crawler.crawling)
@defer.inlineCallbacks
def test_crawlerrunner_accepts_crawler(self):
crawler = self.runner.create_crawler(SimpleSpider)
crawler = get_crawler(SimpleSpider)
runner = CrawlerRunner()
with LogCapture() as log:
yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@defer.inlineCallbacks
def test_crawl_multiple(self):
self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
with LogCapture() as log:
yield self.runner.join()
yield runner.join()
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
@ -358,7 +360,6 @@ class CrawlSpiderTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@ -370,7 +371,7 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(spider_cls)
crawler = get_crawler(spider_cls)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@ -378,10 +379,9 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_crawlspider_with_parse(self):
self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
crawler = get_crawler(CrawlSpiderWithParseMethod)
with LogCapture() as log:
yield self.runner.join()
yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("[parse] status 200 (foo: None)", str(log))
self.assertIn("[parse] status 201 (foo: None)", str(log))
@ -389,10 +389,9 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_crawlspider_with_errback(self):
self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
crawler = get_crawler(CrawlSpiderWithErrback)
with LogCapture() as log:
yield self.runner.join()
yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("[parse] status 200 (foo: None)", str(log))
self.assertIn("[parse] status 201 (foo: None)", str(log))
@ -403,18 +402,19 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_async_def_parse(self):
self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
crawler = get_crawler(AsyncDefSpider)
with LogCapture() as log:
yield self.runner.join()
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse(self):
runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})
runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
crawler = get_crawler(AsyncDefAsyncioSpider, {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
})
with LogCapture() as log:
yield runner.join()
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@ -433,7 +433,7 @@ class CrawlSpiderTestCase(TestCase):
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@ -479,14 +479,14 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_response_ssl_certificate_none(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=False)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
@defer.inlineCallbacks
def test_response_ssl_certificate(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/echo?body=test", is_secure=True)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
cert = crawler.spider.meta['responses'][0].certificate
@ -497,7 +497,7 @@ class CrawlSpiderTestCase(TestCase):
@mark.xfail(reason="Responses with no body return early and contain no certificate")
@defer.inlineCallbacks
def test_response_ssl_certificate_empty_response(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url("/status?n=200", is_secure=True)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
cert = crawler.spider.meta['responses'][0].certificate
@ -507,7 +507,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_dns_server_ip_address_none(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
@ -515,7 +515,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_dns_server_ip_address(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
url = self.mockserver.url('/echo?body=test')
expected_netloc, _ = urlparse(url).netloc.split(':')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
@ -525,7 +525,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_bytes_received_stop_download_callback(self):
crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
crawler = get_crawler(BytesReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
@ -534,7 +534,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_bytes_received_stop_download_errback(self):
crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
crawler = get_crawler(BytesReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
@ -549,7 +549,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_headers_received_stop_download_callback(self):
crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
crawler = get_crawler(HeadersReceivedCallbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
@ -557,7 +557,7 @@ class CrawlSpiderTestCase(TestCase):
@defer.inlineCallbacks
def test_headers_received_stop_download_errback(self):
crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
crawler = get_crawler(HeadersReceivedErrbackSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("response"))
self.assertIsInstance(crawler.spider.meta["failure"], Failure)

36
tests/test_crawler.py
View File

@ -13,11 +13,13 @@ from twisted.trial import unittest
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
from scrapy.utils.test import get_crawler
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
from scrapy.utils.test import get_testenv
@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase):
class CrawlerTestCase(BaseCrawlerTest):
def setUp(self):
self.crawler = Crawler(DefaultSpider, Settings())
def test_populate_spidercls_settings(self):
spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest):
settings = Settings()
settings.setdict(project_settings, priority='project')
crawler = Crawler(CustomSettingsSpider, settings)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
crawler = Crawler(CustomSettingsSpider, settings)
self.assertEqual(crawler.settings.get('TEST1'), 'spider')
self.assertEqual(crawler.settings.get('TEST2'), 'spider')
@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest):
self.assertTrue(crawler.settings.frozen)
def test_crawler_accepts_dict(self):
crawler = Crawler(DefaultSpider, {'foo': 'bar'})
crawler = get_crawler(DefaultSpider, {'foo': 'bar'})
self.assertEqual(crawler.settings['foo'], 'bar')
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
def test_crawler_accepts_None(self):
crawler = Crawler(DefaultSpider)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
crawler = Crawler(DefaultSpider)
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
def test_crawler_rejects_spider_objects(self):
@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase):
'AUTOTHROTTLE_ENABLED': True
}
crawler = Crawler(MySpider, {})
crawler = get_crawler(MySpider)
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
self.assertIn(AutoThrottle, enabled_exts)
@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
class MySpider(scrapy.Spider):
name = 'spider'
Crawler(MySpider, {})
get_crawler(MySpider)
assert get_scrapy_root_handler() is None
def test_spider_custom_settings_log_level(self):
@ -111,7 +114,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
configure_logging()
self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
crawler = Crawler(MySpider, {})
crawler = get_crawler(MySpider)
self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
info_count = crawler.stats.get_value('log_count/INFO')
logging.debug('debug message')
@ -148,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
}
configure_logging()
Crawler(MySpider, {})
get_crawler(MySpider)
logging.debug('debug message')
with open(log_file, 'rb') as fo:
@ -229,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider):
@mark.usefixtures('reactor_pytest')
class CrawlerRunnerHasSpider(unittest.TestCase):
def _runner(self):
return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'})
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful(self):
runner = CrawlerRunner()
runner = self._runner()
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, False)
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_successful_for_several(self):
runner = CrawlerRunner()
runner = self._runner()
yield runner.crawl(NoRequestsSpider)
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, False)
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_failed(self):
runner = CrawlerRunner()
runner = self._runner()
try:
yield runner.crawl(ExceptionSpider)
@ -257,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler_runner_bootstrap_failed_for_several(self):
runner = CrawlerRunner()
runner = self._runner()
try:
yield runner.crawl(ExceptionSpider)
@ -275,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
if self.reactor_pytest == 'asyncio':
CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
})
else:
msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
with self.assertRaisesRegex(Exception, msg):
runner = CrawlerRunner(settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION",
})
yield runner.crawl(NoRequestsSpider)

14
tests/test_downloadermiddleware_httpauth.py
View File

@ -1,7 +1,9 @@
import unittest
import pytest
from w3lib.http import basic_auth_header
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.spiders import Spider
@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.spider = TestSpiderLegacy('foo')
def test_auth(self):
mw = HttpAuthMiddleware()
mw.spider_opened(self.spider)
with pytest.warns(ScrapyDeprecationWarning,
match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
mw = HttpAuthMiddleware()
mw.spider_opened(self.spider)
# initial request, sets the domain and sends the header
req = Request('http://example.com/')
@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
self.assertNotIn('Authorization', req.headers)
def test_auth_already_set(self):
mw = HttpAuthMiddleware()
mw.spider_opened(self.spider)
with pytest.warns(ScrapyDeprecationWarning,
match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
mw = HttpAuthMiddleware()
mw.spider_opened(self.spider)
req = Request('http://example.com/',
headers=dict(Authorization='Digest 123'))
assert mw.process_request(req, self.spider) is None

12
tests/test_downloadermiddleware_httpproxy.py
View File

@ -1,13 +1,13 @@
import os
from functools import partial
import pytest
from twisted.trial.unittest import TestCase
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.crawler import Crawler
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler
spider = Spider('foo')
@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase):
os.environ = self._oldenv
def test_not_enabled(self):
settings = Settings({'HTTPPROXY_ENABLED': False})
crawler = Crawler(Spider, settings)
self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})
with pytest.raises(NotConfigured):
HttpProxyMiddleware.from_crawler(crawler)
def test_no_environment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}

7
tests/test_downloadermiddleware_stats.py
View File

@ -1,7 +1,9 @@
import warnings
from itertools import product
from unittest import TestCase
from scrapy.downloadermiddlewares.stats import DownloaderStats
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.response import response_httprepr
@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase):
for test_response in test_responses:
self.crawler.stats.set_value('downloader/response_bytes', 0)
self.mw.process_response(self.req, test_response, self.spider)
self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
resp_size = len(response_httprepr(test_response))
self.assertStatsEqual('downloader/response_bytes', resp_size)
def test_process_exception(self):
self.mw.process_exception(self.req, MyException(), self.spider)

27
tests/test_dupefilters.py
View File

@ -10,7 +10,6 @@ from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
from scrapy.core.scheduler import Scheduler
from scrapy.utils.python import to_bytes
from scrapy.utils.job import job_dir
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@ -29,8 +28,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@classmethod
def from_crawler(cls, crawler):
debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
df = cls(job_dir(crawler.settings), debug)
df = super().from_crawler(crawler)
df.method = 'from_crawler'
return df
@ -38,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
class FromSettingsRFPDupeFilter(RFPDupeFilter):
@classmethod
def from_settings(cls, settings):
debug = settings.getbool('DUPEFILTER_DEBUG')
df = cls(job_dir(settings), debug)
def from_settings(cls, settings, *, fingerprinter=None):
df = super().from_settings(settings, fingerprinter=fingerprinter)
df.method = 'from_settings'
return df
@ -53,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_crawler_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
@ -61,14 +59,16 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_df_from_settings_scheduler(self):
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertTrue(scheduler.df.debug)
self.assertEqual(scheduler.df.method, 'from_settings')
def test_df_direct_scheduler(self):
settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
self.assertEqual(scheduler.df.method, 'n/a')
@ -171,7 +171,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': False,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = _get_dupefilter(crawler=crawler)
@ -197,7 +198,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log_debug(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True,
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = _get_dupefilter(crawler=crawler)
@ -230,7 +232,8 @@ class RFPDupeFilterTest(unittest.TestCase):
def test_log_debug_default_dupefilter(self):
with LogCapture() as log:
settings = {'DUPEFILTER_DEBUG': True}
settings = {'DUPEFILTER_DEBUG': True,
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}
crawler = get_crawler(SimpleSpider, settings_dict=settings)
spider = SimpleSpider.from_crawler(crawler)
dupefilter = _get_dupefilter(crawler=crawler)

228
tests/test_engine.py
View File

@ -13,10 +13,11 @@ module with the ``runserver`` argument::
import os
import re
import sys
import warnings
from collections import defaultdict
from urllib.parse import urlparse
from dataclasses import dataclass
import pytest
import attr
from itemadapter import ItemAdapter
from pydispatch import dispatcher
@ -50,6 +51,13 @@ class AttrsItem:
price = attr.ib(default=0)
@dataclass
class DataClassItem:
name: str = ""
url: str = ""
price: int = 0
class TestSpider(Spider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider):
item_cls = AttrsItem
try:
from dataclasses import make_dataclass
except ImportError:
DataClassItemsSpider = None
else:
TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef]
def parse_item(self, response):
item = super().parse_item(response)
return TestDataClass(**item)
class DataClassItemsSpider(TestSpider):
item_cls = DataClassItem
class ItemZeroDivisionErrorSpider(TestSpider):
@ -188,7 +187,7 @@ class CrawlerRun:
return self.deferred
def stop(self):
self.port.stopListening()
self.port.stopListening() # FIXME: wait for this Deferred
for name, signal in vars(signals).items():
if not name.startswith('_'):
disconnect_all(signal)
@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase):
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
if spider is None:
continue
self.run = CrawlerRun(spider)
yield self.run.run()
self._assert_visited_urls()
self._assert_scheduled_requests(count=9)
self._assert_downloaded_responses(count=9)
self._assert_scraped_items()
self._assert_signals_caught()
self._assert_bytes_received()
run = CrawlerRun(spider)
yield run.run()
self._assert_visited_urls(run)
self._assert_scheduled_requests(run, count=9)
self._assert_downloaded_responses(run, count=9)
self._assert_scraped_items(run)
self._assert_signals_caught(run)
self._assert_bytes_received(run)
@defer.inlineCallbacks
def test_crawler_dupefilter(self):
self.run = CrawlerRun(TestDupeFilterSpider)
yield self.run.run()
self._assert_scheduled_requests(count=8)
self._assert_dropped_requests()
run = CrawlerRun(TestDupeFilterSpider)
yield run.run()
self._assert_scheduled_requests(run, count=8)
self._assert_dropped_requests(run)
@defer.inlineCallbacks
def test_crawler_itemerror(self):
self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
yield self.run.run()
self._assert_items_error()
run = CrawlerRun(ItemZeroDivisionErrorSpider)
yield run.run()
self._assert_items_error(run)
@defer.inlineCallbacks
def test_crawler_change_close_reason_on_idle(self):
self.run = CrawlerRun(ChangeCloseReasonSpider)
yield self.run.run()
self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
self.run.signals_caught[signals.spider_closed])
run = CrawlerRun(ChangeCloseReasonSpider)
yield run.run()
self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},
run.signals_caught[signals.spider_closed])
def _assert_visited_urls(self):
def _assert_visited_urls(self, run: CrawlerRun):
must_be_visited = ["/", "/redirect", "/redirected",
"/item1.html", "/item2.html", "/item999.html"]
urls_visited = {rp[0].url for rp in self.run.respplug}
urls_expected = {self.run.geturl(p) for p in must_be_visited}
urls_visited = {rp[0].url for rp in run.respplug}
urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
def _assert_scheduled_requests(self, count=None):
self.assertEqual(count, len(self.run.reqplug))
def _assert_scheduled_requests(self, run: CrawlerRun, count=None):
self.assertEqual(count, len(run.reqplug))
paths_expected = ['/item999.html', '/item2.html', '/item1.html']
urls_requested = {rq[0].url for rq in self.run.reqplug}
urls_expected = {self.run.geturl(p) for p in paths_expected}
urls_requested = {rq[0].url for rq in run.reqplug}
urls_expected = {run.geturl(p) for p in paths_expected}
assert urls_expected <= urls_requested
scheduled_requests_count = len(self.run.reqplug)
dropped_requests_count = len(self.run.reqdropped)
responses_count = len(self.run.respplug)
scheduled_requests_count = len(run.reqplug)
dropped_requests_count = len(run.reqdropped)
responses_count = len(run.respplug)
self.assertEqual(scheduled_requests_count,
dropped_requests_count + responses_count)
self.assertEqual(len(self.run.reqreached),
self.assertEqual(len(run.reqreached),
responses_count)
def _assert_dropped_requests(self):
self.assertEqual(len(self.run.reqdropped), 1)
def _assert_dropped_requests(self, run: CrawlerRun):
self.assertEqual(len(run.reqdropped), 1)
def _assert_downloaded_responses(self, count):
def _assert_downloaded_responses(self, run: CrawlerRun, count):
# response tests
self.assertEqual(count, len(self.run.respplug))
self.assertEqual(count, len(self.run.reqreached))
self.assertEqual(count, len(run.respplug))
self.assertEqual(count, len(run.reqreached))
for response, _ in self.run.respplug:
if self.run.getpath(response.url) == '/item999.html':
for response, _ in run.respplug:
if run.getpath(response.url) == '/item999.html':
self.assertEqual(404, response.status)
if self.run.getpath(response.url) == '/redirect':
if run.getpath(response.url) == '/redirect':
self.assertEqual(302, response.status)
def _assert_items_error(self):
self.assertEqual(2, len(self.run.itemerror))
for item, response, spider, failure in self.run.itemerror:
def _assert_items_error(self, run: CrawlerRun):
self.assertEqual(2, len(run.itemerror))
for item, response, spider, failure in run.itemerror:
self.assertEqual(failure.value.__class__, ZeroDivisionError)
self.assertEqual(spider, self.run.spider)
self.assertEqual(spider, run.spider)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
def _assert_scraped_items(self):
self.assertEqual(2, len(self.run.itemresp))
for item, response in self.run.itemresp:
def _assert_scraped_items(self, run: CrawlerRun):
self.assertEqual(2, len(run.itemresp))
for item, response in run.itemresp:
item = ItemAdapter(item)
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
def _assert_headers_received(self):
for headers in self.run.headers.values():
def _assert_headers_received(self, run: CrawlerRun):
for headers in run.headers.values():
self.assertIn(b"Server", headers)
self.assertIn(b"TwistedWeb", headers[b"Server"])
self.assertIn(b"Date", headers)
self.assertIn(b"Content-Type", headers)
def _assert_bytes_received(self):
self.assertEqual(9, len(self.run.bytes))
for request, data in self.run.bytes.items():
def _assert_bytes_received(self, run: CrawlerRun):
self.assertEqual(9, len(run.bytes))
for request, data in run.bytes.items():
joined_data = b"".join(data)
if self.run.getpath(request.url) == "/":
if run.getpath(request.url) == "/":
self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
elif self.run.getpath(request.url) == "/item1.html":
elif run.getpath(request.url) == "/item1.html":
self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
elif self.run.getpath(request.url) == "/item2.html":
elif run.getpath(request.url) == "/item2.html":
self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
elif self.run.getpath(request.url) == "/redirected":
elif run.getpath(request.url) == "/redirected":
self.assertEqual(joined_data, b"Redirected here")
elif self.run.getpath(request.url) == '/redirect':
elif run.getpath(request.url) == '/redirect':
self.assertEqual(
joined_data,
b"\n<html>\n"
@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase):
b" </body>\n"
b"</html>\n"
)
elif self.run.getpath(request.url) == "/tem999.html":
elif run.getpath(request.url) == "/tem999.html":
self.assertEqual(
joined_data,
b"\n<html>\n"
@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase):
b" </body>\n"
b"</html>\n"
)
elif self.run.getpath(request.url) == "/numbers":
elif run.getpath(request.url) == "/numbers":
# signal was fired multiple times
self.assertTrue(len(data) > 1)
# bytes were received in order
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.assertEqual(joined_data, b"".join(numbers))
def _assert_signals_caught(self):
assert signals.engine_started in self.run.signals_caught
assert signals.engine_stopped in self.run.signals_caught
assert signals.spider_opened in self.run.signals_caught
assert signals.spider_idle in self.run.signals_caught
assert signals.spider_closed in self.run.signals_caught
assert signals.headers_received in self.run.signals_caught
def _assert_signals_caught(self, run: CrawlerRun):
assert signals.engine_started in run.signals_caught
assert signals.engine_stopped in run.signals_caught
assert signals.spider_opened in run.signals_caught
assert signals.spider_idle in run.signals_caught
assert signals.spider_closed in run.signals_caught
assert signals.headers_received in run.signals_caught
self.assertEqual({'spider': self.run.spider},
self.run.signals_caught[signals.spider_opened])
self.assertEqual({'spider': self.run.spider},
self.run.signals_caught[signals.spider_idle])
self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
self.run.signals_caught[signals.spider_closed])
self.assertEqual({'spider': run.spider},
run.signals_caught[signals.spider_opened])
self.assertEqual({'spider': run.spider},
run.signals_caught[signals.spider_idle])
self.assertEqual({'spider': run.spider, 'reason': 'finished'},
run.signals_caught[signals.spider_closed])
@defer.inlineCallbacks
def test_close_downloader(self):
@ -407,28 +404,29 @@ class EngineTest(unittest.TestCase):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
yield self.assertFailure(e.start(), RuntimeError).addBoth(
lambda exc: self.assertEqual(str(exc), "Engine already running")
)
yield e.stop()
try:
yield self.assertFailure(e.start(), RuntimeError).addBoth(
lambda exc: self.assertEqual(str(exc), "Engine already running")
)
finally:
yield e.stop()
@defer.inlineCallbacks
def test_close_spiders_downloader(self):
with warnings.catch_warnings(record=True) as warning_list:
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.open_spiders is deprecated, "
"please use ExecutionEngine.spider instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
self.assertEqual(len(e.open_spiders), 1)
yield e.close()
self.assertEqual(len(e.open_spiders), 0)
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
)
@defer.inlineCallbacks
def test_close_engine_spiders_downloader(self):
with warnings.catch_warnings(record=True) as warning_list:
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.open_spiders is deprecated, "
"please use ExecutionEngine.spider instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
yield e.open_spider(TestSpider(), [])
e.start()
@ -436,61 +434,47 @@ class EngineTest(unittest.TestCase):
yield e.close()
self.assertFalse(e.running)
self.assertEqual(len(e.open_spiders), 0)
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
)
@defer.inlineCallbacks
def test_crawl_deprecated_spider_arg(self):
with warnings.catch_warnings(record=True) as warning_list:
with pytest.warns(ScrapyDeprecationWarning,
match="Passing a 'spider' argument to "
"ExecutionEngine.crawl is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.crawl(Request("data:,"), spider)
yield e.close()
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
)
@defer.inlineCallbacks
def test_download_deprecated_spider_arg(self):
with warnings.catch_warnings(record=True) as warning_list:
with pytest.warns(ScrapyDeprecationWarning,
match="Passing a 'spider' argument to "
"ExecutionEngine.download is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.download(Request("data:,"), spider)
yield e.close()
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"Passing a 'spider' argument to ExecutionEngine.download is deprecated",
)
@defer.inlineCallbacks
def test_deprecated_schedule(self):
with warnings.catch_warnings(record=True) as warning_list:
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
spider = TestSpider()
yield e.open_spider(spider, [])
e.start()
e.schedule(Request("data:,"), spider)
yield e.close()
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(
str(warning_list[0].message),
"ExecutionEngine.schedule is deprecated, please use "
"ExecutionEngine.crawl or ExecutionEngine.download instead",
)
@defer.inlineCallbacks
def test_deprecated_has_capacity(self):
with warnings.catch_warnings(record=True) as warning_list:
with pytest.warns(ScrapyDeprecationWarning,
match="ExecutionEngine.has_capacity is deprecated"):
e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
self.assertTrue(e.has_capacity())
spider = TestSpider()
@ -499,8 +483,6 @@ class EngineTest(unittest.TestCase):
e.start()
yield e.close()
self.assertTrue(e.has_capacity())
self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
if __name__ == "__main__":

32
tests/test_engine_stop_download_bytes.py
View File

@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
if spider is None:
continue
self.run = BytesReceivedCrawlerRun(spider)
run = BytesReceivedCrawlerRun(spider)
with LogCapture() as log:
yield self.run.run()
yield run.run()
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
f"Download stopped for <GET http://localhost:{run.portno}/redirected> "
"from signal handler BytesReceivedCrawlerRun.bytes_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
f"Download stopped for <GET http://localhost:{self.run.portno}/> "
f"Download stopped for <GET http://localhost:{run.portno}/> "
"from signal handler BytesReceivedCrawlerRun.bytes_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
f"Download stopped for <GET http://localhost:{run.portno}/numbers> "
"from signal handler BytesReceivedCrawlerRun.bytes_received"))
self._assert_visited_urls()
self._assert_scheduled_requests(count=9)
self._assert_downloaded_responses(count=9)
self._assert_signals_caught()
self._assert_headers_received()
self._assert_bytes_received()
self._assert_visited_urls(run)
self._assert_scheduled_requests(run, count=9)
self._assert_downloaded_responses(run, count=9)
self._assert_signals_caught(run)
self._assert_headers_received(run)
self._assert_bytes_received(run)
def _assert_bytes_received(self):
self.assertEqual(9, len(self.run.bytes))
for request, data in self.run.bytes.items():
def _assert_bytes_received(self, run: CrawlerRun):
self.assertEqual(9, len(run.bytes))
for request, data in run.bytes.items():
joined_data = b"".join(data)
self.assertTrue(len(data) == 1) # signal was fired only once
if self.run.getpath(request.url) == "/numbers":
if run.getpath(request.url) == "/numbers":
# Received bytes are not the complete response. The exact amount depends
# on the buffer size, which can vary, so we only check that the amount
# of received bytes is strictly less than the full response.

32
tests/test_engine_stop_download_headers.py
View File

@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest):
@defer.inlineCallbacks
def test_crawler(self):
for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
if spider is None:
continue
self.run = HeadersReceivedCrawlerRun(spider)
run = HeadersReceivedCrawlerRun(spider)
with LogCapture() as log:
yield self.run.run()
yield run.run()
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"
f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"
" signal handler HeadersReceivedCrawlerRun.headers_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"
f"Download stopped for <GET http://localhost:{run.portno}/> from signal"
" handler HeadersReceivedCrawlerRun.headers_received"))
log.check_present(("scrapy.core.downloader.handlers.http11",
"DEBUG",
f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"
f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"
" signal handler HeadersReceivedCrawlerRun.headers_received"))
self._assert_visited_urls()
self._assert_downloaded_responses(count=6)
self._assert_signals_caught()
self._assert_bytes_received()
self._assert_headers_received()
self._assert_visited_urls(run)
self._assert_downloaded_responses(run, count=6)
self._assert_signals_caught(run)
self._assert_bytes_received(run)
self._assert_headers_received(run)
def _assert_bytes_received(self):
self.assertEqual(0, len(self.run.bytes))
def _assert_bytes_received(self, run: CrawlerRun):
self.assertEqual(0, len(run.bytes))
def _assert_visited_urls(self):
def _assert_visited_urls(self, run: CrawlerRun):
must_be_visited = ["/", "/redirect", "/redirected"]
urls_visited = {rp[0].url for rp in self.run.respplug}
urls_expected = {self.run.geturl(p) for p in must_be_visited}
urls_visited = {rp[0].url for rp in run.respplug}
urls_expected = {run.geturl(p) for p in must_be_visited}
assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"

269
tests/test_feedexport.py
View File

@ -22,6 +22,7 @@ from urllib.parse import urljoin, quote
from urllib.request import pathname2url
import lxml.etree
import pytest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial import unittest
@ -30,7 +31,6 @@ from zope.interface import implementer
from zope.interface.verify import verifyObject
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter
from scrapy.extensions.feedexport import (
@ -697,9 +697,9 @@ class FeedExportTest(FeedExportTestBase):
content = {}
try:
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
crawler = get_crawler(spider_cls, settings)
yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
@ -1554,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
content = {}
try:
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
crawler = get_crawler(spider_cls, settings)
yield crawler.crawl()
for file_path, feed_options in FEEDS.items():
if not os.path.exists(str(file_path)):
@ -2026,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
content = defaultdict(list)
try:
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))
spider_cls.start_urls = [s.url('/')]
yield runner.crawl(spider_cls)
crawler = get_crawler(spider_cls, settings)
yield crawler.crawl()
for path, feed in FEEDS.items():
dir_name = os.path.dirname(path)
@ -2048,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['jl']:
@ -2064,7 +2064,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
data = yield self.exported_data(items, settings)
for batch in data['csv']:
got_batch = csv.DictReader(to_unicode(batch).splitlines())
@ -2080,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['xml']:
@ -2098,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
# XML
@ -2123,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import pickle
@ -2140,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
},
})
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import marshal
@ -2166,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'FEED_EXPORT_BATCH_ITEM_COUNT': 2
}
header = self.MyItem.fields.keys()
yield self.assertExported(items, header, rows, settings=Settings(settings))
yield self.assertExported(items, header, rows, settings=settings)
def test_wrong_path(self):
""" If path is without %(batch_time)s and %(batch_id) an exception must be raised """
@ -2382,9 +2382,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
yield item
with MockServer() as server:
runner = CrawlerRunner(Settings(settings))
TestSpider.start_urls = [server.url('/')]
yield runner.crawl(TestSpider)
crawler = get_crawler(TestSpider, settings)
yield crawler.crawl()
self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
for stub in CustomS3FeedStorage.stubs[:-1]:
@ -2434,25 +2434,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': StdoutFeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="StdoutFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
messages = tuple(str(item.message) for item in w
if item.category is ScrapyDeprecationWarning)
self.assertEqual(
messages,
(
(
"StdoutFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove "
"this warning. This parameter will become mandatory "
"in a future version of Scrapy."
),
)
)
class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
@ -2476,25 +2467,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': FileFeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="FileFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
messages = tuple(str(item.message) for item in w
if item.category is ScrapyDeprecationWarning)
self.assertEqual(
messages,
(
(
"FileFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove "
"this warning. This parameter will become mandatory "
"in a future version of Scrapy."
),
)
)
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
@ -2524,26 +2506,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': S3FeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="S3FeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
messages = tuple(str(item.message) for item in w
if item.category is ScrapyDeprecationWarning)
self.assertEqual(
messages,
(
(
"S3FeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove "
"this warning. This parameter will become mandatory "
"in a future version of Scrapy."
),
)
)
def test_from_crawler(self):
settings_dict = {
@ -2552,26 +2526,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
},
}
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
"the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
messages = tuple(str(item.message) for item in w
if item.category is ScrapyDeprecationWarning)
self.assertEqual(
messages,
(
(
"S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
"does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove "
"this warning. This parameter will become mandatory "
"in a future version of Scrapy."
),
)
)
class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
@ -2601,26 +2567,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': FTPFeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="FTPFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
messages = tuple(str(item.message) for item in w
if item.category is ScrapyDeprecationWarning)
self.assertEqual(
messages,
(
(
"FTPFeedStorageWithoutFeedOptions does not support "
"the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove "
"this warning. This parameter will become mandatory "
"in a future version of Scrapy."
),
)
)
def test_from_crawler(self):
settings_dict = {
@ -2629,50 +2587,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
},
}
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings_dict)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
"the 'feed_options' keyword argument."):
feed_exporter.open_spider(spider)
messages = tuple(str(item.message) for item in w
if item.category is ScrapyDeprecationWarning)
self.assertEqual(
messages,
(
(
"FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
"does not support the 'feed_options' keyword argument. Add a "
"'feed_options' parameter to its signature to remove "
"this warning. This parameter will become mandatory "
"in a future version of Scrapy."
),
)
)
class URIParamsTest:
spider_name = "uri_params_spider"
deprecated_options = False
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
raise NotImplementedError
def _crawler_feed_exporter(self, settings):
if self.deprecated_options:
with pytest.warns(ScrapyDeprecationWarning,
match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
else:
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
return crawler, feed_exporter
def test_default(self):
settings = self.build_settings(
uri='file:///tmp/%(name)s',
)
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
messages = tuple(
str(item.message) for item in w
if item.category is ScrapyDeprecationWarning
)
self.assertEqual(messages, tuple())
self.assertEqual(
feed_exporter.slots[0].uri,
@ -2687,28 +2645,13 @@ class URIParamsTest:
uri='file:///tmp/%(name)s',
uri_params=uri_params,
)
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with pytest.warns(ScrapyDeprecationWarning,
match="Modifying the params dictionary in-place"):
feed_exporter.open_spider(spider)
messages = tuple(
str(item.message) for item in w
if item.category is ScrapyDeprecationWarning
)
self.assertEqual(
messages,
(
(
'Modifying the params dictionary in-place in the '
'function defined in the FEED_URI_PARAMS setting or '
'in the uri_params key of the FEEDS setting is '
'deprecated. The function must return a new '
'dictionary instead.'
),
)
)
self.assertEqual(
feed_exporter.slots[0].uri,
@ -2723,18 +2666,14 @@ class URIParamsTest:
uri='file:///tmp/%(name)s',
uri_params=uri_params,
)
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
with self.assertRaises(KeyError):
feed_exporter.open_spider(spider)
messages = tuple(
str(item.message) for item in w
if item.category is ScrapyDeprecationWarning
)
self.assertEqual(messages, tuple())
def test_params_as_is(self):
def uri_params(params, spider):
@ -2744,17 +2683,12 @@ class URIParamsTest:
uri='file:///tmp/%(name)s',
uri_params=uri_params,
)
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
messages = tuple(
str(item.message) for item in w
if item.category is ScrapyDeprecationWarning
)
self.assertEqual(messages, tuple())
self.assertEqual(
feed_exporter.slots[0].uri,
@ -2769,17 +2703,12 @@ class URIParamsTest:
uri='file:///tmp/%(foo)s',
uri_params=uri_params,
)
crawler = get_crawler(settings_dict=settings)
feed_exporter = FeedExporter.from_crawler(crawler)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
messages = tuple(
str(item.message) for item in w
if item.category is ScrapyDeprecationWarning
)
self.assertEqual(messages, tuple())
self.assertEqual(
feed_exporter.slots[0].uri,
@ -2788,6 +2717,7 @@ class URIParamsTest:
class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
deprecated_options = True
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
extra_settings = {}
@ -2800,6 +2730,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
deprecated_options = False
def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
options = {

6
tests/test_logformatter.py
View File

@ -5,8 +5,8 @@ from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase as TwistedTestCase
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import DropItem
from scrapy.utils.test import get_crawler
from scrapy.http import Request, Response
from scrapy.item import Item, Field
from scrapy.logformatter import LogFormatter
@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
@defer.inlineCallbacks
def test_show_messages(self):
crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
crawler = get_crawler(ItemSpider, self.base_settings)
with LogCapture() as lc:
yield crawler.crawl(mockserver=self.mockserver)
self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
def test_skip_messages(self):
settings = self.base_settings.copy()
settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
crawler = get_crawler(ItemSpider, settings)
with LogCapture() as lc:
yield crawler.crawl(mockserver=self.mockserver)
self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))

12
tests/test_pipeline_crawl.py
View File

@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):
self.tmpmediastore = self.mktemp()
os.mkdir(self.tmpmediastore)
self.settings = {
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
'ITEM_PIPELINES': {self.pipeline_class: 1},
self.store_setting_key: self.tmpmediastore,
}
@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase):
def _on_item_scraped(self, item):
self.items.append(item)
def _create_crawler(self, spider_class, **kwargs):
crawler = self.runner.create_crawler(spider_class, **kwargs)
def _create_crawler(self, spider_class, runner=None, **kwargs):
if runner is None:
runner = self.runner
crawler = runner.create_crawler(spider_class, **kwargs)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
return crawler
@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_redirected_allowed(self):
settings = dict(self.settings)
settings.update({'MEDIA_ALLOW_REDIRECTS': True})
self.runner = CrawlerRunner(settings)
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
runner = CrawlerRunner(settings)
crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
with LogCapture() as log:
yield crawler.crawl(
self.mockserver.url("/files/images/"),

21
tests/test_request_attribute_binding.py
View File

@ -2,8 +2,8 @@ from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy import Request, signals
from scrapy.crawler import CrawlerRunner
from scrapy.http.response import Response
from scrapy.utils.test import get_crawler
from testfixtures import LogCapture
@ -71,7 +71,7 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_response_200(self):
url = self.mockserver.url("/status?n=200")
crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.request.url, url)
@ -80,7 +80,7 @@ class CrawlTestCase(TestCase):
def test_response_error(self):
for status in ("404", "500"):
url = self.mockserver.url(f"/status?n={status}")
crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
failure = crawler.spider.meta["failure"]
response = failure.value.response
@ -90,12 +90,11 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_downloader_middleware_raise_exception(self):
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
RaiseExceptionRequestMiddleware: 590,
},
})
crawler = runner.create_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
failure = crawler.spider.meta["failure"]
self.assertEqual(failure.request.url, url)
@ -117,12 +116,11 @@ class CrawlTestCase(TestCase):
signal_params["request"] = request
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
ProcessResponseMiddleware: 595,
}
})
crawler = runner.create_crawler(SingleRequestSpider)
crawler.signals.connect(signal_handler, signal=signals.response_received)
with LogCapture() as log:
@ -147,13 +145,12 @@ class CrawlTestCase(TestCase):
The spider callback should receive the overridden response.request
"""
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
RaiseExceptionRequestMiddleware: 590,
CatchExceptionOverrideRequestMiddleware: 595,
},
})
crawler = runner.create_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.body, b"Caught ZeroDivisionError")
@ -168,13 +165,12 @@ class CrawlTestCase(TestCase):
The spider callback should receive the original response.request
"""
url = self.mockserver.url("/status?n=200")
runner = CrawlerRunner(settings={
crawler = get_crawler(SingleRequestSpider, {
"DOWNLOADER_MIDDLEWARES": {
RaiseExceptionRequestMiddleware: 590,
CatchExceptionDoNotOverrideRequestMiddleware: 595,
},
})
crawler = runner.create_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.body, b"Caught ZeroDivisionError")
@ -186,12 +182,11 @@ class CrawlTestCase(TestCase):
Downloader middleware which returns a response with a
specific 'request' attribute, with an alternative callback
"""
runner = CrawlerRunner(settings={
crawler = get_crawler(AlternativeCallbacksSpider, {
"DOWNLOADER_MIDDLEWARES": {
AlternativeCallbacksMiddleware: 595,
}
})
crawler = runner.create_crawler(AlternativeCallbacksSpider)
with LogCapture() as log:
url = self.mockserver.url("/status?n=200")

5
tests/test_request_cb_kwargs.py
View File

@ -3,7 +3,7 @@ from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
from scrapy.utils.test import get_crawler
from tests.spiders import MockServerSpider
from tests.mockserver import MockServer
@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_callback_kwargs(self):
crawler = self.runner.create_crawler(KeywordArgumentsSpider)
crawler = get_crawler(KeywordArgumentsSpider)
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(all(crawler.spider.checks))

3
tests/test_scheduler.py
View File

@ -52,6 +52,7 @@ class MockCrawler(Crawler):
SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
JOBDIR=jobdir,
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION',
)
super().__init__(Spider, settings)
self.engine = MockEngine(downloader=MockDownloader())
@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase):
SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
CONCURRENT_REQUESTS_PER_IP=1,
)
crawler = Crawler(Spider, settings)
crawler = get_crawler(Spider, settings)
scheduler = Scheduler.from_crawler(crawler)
spider = Spider(name='spider')
scheduler.open(spider)

15
tests/test_scheduler_base.py
View File

@ -7,10 +7,10 @@ from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
from scrapy.core.scheduler import BaseScheduler
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.request import request_fingerprint
from scrapy.utils.request import fingerprint
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS]
class MinimalScheduler:
def __init__(self) -> None:
self.requests: Dict[str, Request] = {}
self.requests: Dict[bytes, Request] = {}
def has_pending_requests(self) -> bool:
return bool(self.requests)
def enqueue_request(self, request: Request) -> bool:
fp = request_fingerprint(request)
fp = fingerprint(request)
if fp not in self.requests:
self.requests[fp] = request
return True
@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):
@defer.inlineCallbacks
def test_crawl(self):
with MockServer() as mockserver:
settings = {"SCHEDULER": self.scheduler_cls}
settings = {
"SCHEDULER": self.scheduler_cls,
}
with LogCapture() as log:
yield CrawlerRunner(settings).crawl(TestSpider, mockserver)
crawler = get_crawler(TestSpider, settings)
yield crawler.crawl(mockserver)
for path in PATHS:
self.assertIn(f"{{'path': '{path}'}}", str(log))
self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log))

5
tests/test_spiderloader/__init__.py
View File

@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase):
def test_crawler_runner_loading(self):
module = 'tests.test_spiderloader.test_spiders.spider1'
runner = CrawlerRunner({'SPIDER_MODULES': [module]})
runner = CrawlerRunner({
'SPIDER_MODULES': [module],
'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION',
})
self.assertRaisesRegex(KeyError, 'Spider not found',
runner.create_crawler, 'spider2')

27
tests/test_utils_project.py
View File

@ -3,6 +3,7 @@ import os
import tempfile
import shutil
import contextlib
import warnings
from pytest import warns
@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase):
envvars = {
'SCRAPY_SETTINGS_MODULE': value,
}
with set_env(**envvars), warns(None) as warnings:
settings = get_project_settings()
assert not warnings
with warnings.catch_warnings():
warnings.simplefilter("error")
with set_env(**envvars):
settings = get_project_settings()
assert settings.get('SETTINGS_MODULE') == value
def test_invalid_envvar(self):
envvars = {
'SCRAPY_FOO': 'bar',
}
with set_env(**envvars), warns(None) as warnings:
get_project_settings()
assert len(warnings) == 1
assert warnings[0].category == ScrapyDeprecationWarning
assert str(warnings[0].message).endswith(': FOO')
with warns(ScrapyDeprecationWarning, match=': FOO') as record:
with set_env(**envvars):
get_project_settings()
assert len(record) == 1
def test_valid_and_invalid_envvars(self):
value = 'tests.test_cmdline.settings'
@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase):
'SCRAPY_FOO': 'bar',
'SCRAPY_SETTINGS_MODULE': value,
}
with set_env(**envvars), warns(None) as warnings:
settings = get_project_settings()
assert len(warnings) == 1
assert warnings[0].category == ScrapyDeprecationWarning
assert str(warnings[0].message).endswith(': FOO')
with warns(ScrapyDeprecationWarning, match=': FOO') as record:
with set_env(**envvars):
settings = get_project_settings()
assert len(record) == 1
assert settings.get('SETTINGS_MODULE') == value

13
tests/test_utils_request.py
View File

@ -308,13 +308,22 @@ class RequestFingerprintTest(FingerprintTest):
),
)
def setUp(self) -> None:
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
def tearDown(self) -> None:
warnings.simplefilter("default", ScrapyDeprecationWarning)
@pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
def test_part_separation(self):
super().test_part_separation()
class RequestFingerprintDeprecationTest(unittest.TestCase):
def test_deprecation_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
self.function(Request("http://www.example.com"))
request_fingerprint(Request("http://www.example.com"))
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any(
@ -326,7 +335,7 @@ class RequestFingerprintTest(FingerprintTest):
def test_deprecation_non_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
self.function(Request("http://www.example.com"), keep_fragments=True)
request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
any(

21
tests/test_utils_response.py
View File

@ -1,7 +1,9 @@
import os
import unittest
import warnings
from urllib.parse import urlparse
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase):
dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
def test_response_httprepr(self):
r1 = Response("http://www.example.com")
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
r1 = Response("http://www.example.com")
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
r1 = Response("http://www.example.com", status=404,
headers={"Content-type": "text/html"}, body=b"Some body")
self.assertEqual(response_httprepr(r1),
b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
r1 = Response("http://www.example.com", status=6666,
headers={"Content-type": "text/html"}, body=b"Some body")
self.assertEqual(response_httprepr(r1),
b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
def test_open_in_browser(self):
url = "http:///www.example.com/some/page.html"