You are being redirected
' - resp = Response(req.url, status=302, body=body, headers={ - 'Content-Length': str(len(body)), - 'Content-Type': 'text/html', - 'Content-Encoding': 'gzip', - 'Location': 'http://example.com/login', - }) + req = Request("http://example.com") + body = b"You are being redirected
" + resp = Response( + req.url, + status=302, + body=body, + headers={ + "Content-Length": str(len(body)), + "Content-Type": "text/html", + "Content-Encoding": "gzip", + "Location": "http://example.com/login", + }, + ) ret = self._download(request=req, response=resp) - self.assertTrue(isinstance(ret, Request), - f"Not redirected: {ret!r}") - self.assertEqual(to_bytes(ret.url), resp.headers['Location'], - "Not redirected to location header") + self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") + self.assertEqual( + to_bytes(ret.url), + resp.headers["Location"], + "Not redirected to location header", + ) def test_200_and_invalid_gzipped_body_must_fail(self): - req = Request('http://example.com') - body = b'You are being redirected
' - resp = Response(req.url, status=200, body=body, headers={ - 'Content-Length': str(len(body)), - 'Content-Type': 'text/html', - 'Content-Encoding': 'gzip', - 'Location': 'http://example.com/login', - }) - self.assertRaises(IOError, self._download, request=req, response=resp) + req = Request("http://example.com") + body = b"You are being redirected
" + resp = Response( + req.url, + status=200, + body=body, + headers={ + "Content-Length": str(len(body)), + "Content-Type": "text/html", + "Content-Encoding": "gzip", + "Location": "http://example.com/login", + }, + ) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): """Tests middleware returning a response from process_request.""" def test_download_func_not_called(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class ResponseMiddleware: def process_request(self, request, spider): @@ -112,7 +123,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase): self.mwman._add_middleware(ResponseMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] @@ -127,7 +138,7 @@ class ProcessRequestInvalidOutput(ManagerTestCase): """Invalid return value for process_request method should raise an exception""" def test_invalid_process_request(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: def process_request(self, request, spider): @@ -146,7 +157,7 @@ class ProcessResponseInvalidOutput(ManagerTestCase): """Invalid return value for process_response method should raise an exception""" def test_invalid_process_response(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: def process_response(self, request, response, spider): @@ -165,7 +176,7 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): """Invalid return value for process_exception method should raise an exception""" def test_invalid_process_exception(self): - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): @@ -187,7 +198,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): """Middlewares using Deferreds should work""" def test_deferred(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class DeferredMiddleware: def cb(self, result): @@ -200,7 +211,7 @@ class MiddlewareUsingDeferreds(ManagerTestCase): return d self.mwman._add_middleware(DeferredMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] @@ -211,12 +222,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.assertFalse(download_func.called) -@mark.usefixtures('reactor_pytest') +@mark.usefixtures("reactor_pytest") class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" def test_asyncdef(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): @@ -224,7 +235,7 @@ class MiddlewareUsingCoro(ManagerTestCase): return resp self.mwman._add_middleware(CoroMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] @@ -236,7 +247,7 @@ class MiddlewareUsingCoro(ManagerTestCase): @mark.only_asyncio() def test_asyncdef_asyncio(self): - resp = Response('http://example.com/index.html') + resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): @@ -245,7 +256,7 @@ class MiddlewareUsingCoro(ManagerTestCase): return result self.mwman._add_middleware(CoroMiddleware()) - req = Request('http://example.com/index.html') + req = Request("http://example.com/index.html") download_func = mock.MagicMock() dfd = self.mwman.download(download_func, req, self.spider) results = [] diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 5a56c9db2..043dc0a12 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,18 +1,17 @@ import unittest from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware +from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider -from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler - -__doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl'] +__doctests__ = ["scrapy.downloadermiddlewares.ajaxcrawl"] class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): - crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) - self.spider = crawler._create_spider('foo') + crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True}) + self.spider = crawler._create_spider("foo") self.mw = AjaxCrawlMiddleware.from_crawler(crawler) def _ajaxcrawlable_body(self): @@ -24,37 +23,41 @@ class AjaxCrawlMiddlewareTest(unittest.TestCase): return req, resp def test_non_get(self): - req, resp = self._req_resp('http://example.com/', {'method': 'HEAD'}) + req, resp = self._req_resp("http://example.com/", {"method": "HEAD"}) resp2 = self.mw.process_response(req, resp, self.spider) self.assertEqual(resp, resp2) def test_binary_response(self): - req = Request('http://example.com/') - resp = Response('http://example.com/', body=b'foobar\x00\x01\x02', request=req) + req = Request("http://example.com/") + resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req) resp2 = self.mw.process_response(req, resp, self.spider) self.assertIs(resp, resp2) def test_ajaxcrawl(self): req, resp = self._req_resp( - 'http://example.com/', - {'meta': {'foo': 'bar'}}, - {'body': self._ajaxcrawlable_body()} + "http://example.com/", + {"meta": {"foo": "bar"}}, + {"body": self._ajaxcrawlable_body()}, ) req2 = self.mw.process_response(req, resp, self.spider) - self.assertEqual(req2.url, 'http://example.com/?_escaped_fragment_=') - self.assertEqual(req2.meta['foo'], 'bar') + self.assertEqual(req2.url, "http://example.com/?_escaped_fragment_=") + self.assertEqual(req2.meta["foo"], "bar") def test_ajaxcrawl_loop(self): - req, resp = self._req_resp('http://example.com/', {}, {'body': self._ajaxcrawlable_body()}) + req, resp = self._req_resp( + "http://example.com/", {}, {"body": self._ajaxcrawlable_body()} + ) req2 = self.mw.process_response(req, resp, self.spider) resp2 = HtmlResponse(req2.url, body=resp.body, request=req2) resp3 = self.mw.process_response(req2, resp2, self.spider) assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) - self.assertEqual(resp3.request.url, 'http://example.com/?_escaped_fragment_=') + self.assertEqual(resp3.request.url, "http://example.com/?_escaped_fragment_=") assert resp3 is resp2 def test_noncrawlable_body(self): - req, resp = self._req_resp('http://example.com/', {}, {'body': b''}) + req, resp = self._req_resp( + "http://example.com/", {}, {"body": b""} + ) resp2 = self.mw.process_response(req, resp, self.spider) self.assertIs(resp, resp2) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index dd835b9c9..4a81a638e 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,14 +1,14 @@ import logging from unittest import TestCase -from testfixtures import LogCapture import pytest +from testfixtures import LogCapture from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request +from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.python import to_bytes @@ -48,24 +48,18 @@ def _cookies_to_set_cookie_list(cookies): return [] if isinstance(cookies, dict): cookies = ({"name": k, "value": v} for k, v in cookies.items()) - return filter( - None, - ( - _cookie_to_set_cookie_value(cookie) - for cookie in cookies - ) - ) + return filter(None, (_cookie_to_set_cookie_value(cookie) for cookie in cookies)) class CookiesMiddlewareTest(TestCase): - def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) + return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) def setUp(self): - self.spider = Spider('foo') + self.spider = Spider("foo") self.mw = CookiesMiddleware() self.redirect_middleware = RedirectMiddleware(settings=Settings()) @@ -74,346 +68,391 @@ class CookiesMiddlewareTest(TestCase): del self.redirect_middleware def test_basic(self): - req = Request('http://scrapytest.org/') + req = Request("http://scrapytest.org/") assert self.mw.process_request(req, self.spider) is None - assert 'Cookie' not in req.headers + assert "Cookie" not in req.headers - headers = {'Set-Cookie': 'C1=value1; path=/'} - res = Response('http://scrapytest.org/', headers=headers) + headers = {"Set-Cookie": "C1=value1; path=/"} + res = Response("http://scrapytest.org/", headers=headers) assert self.mw.process_response(req, res, self.spider) is res - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") assert self.mw.process_request(req2, self.spider) is None - self.assertEqual(req2.headers.get('Cookie'), b"C1=value1") + self.assertEqual(req2.headers.get("Cookie"), b"C1=value1") def test_setting_false_cookies_enabled(self): self.assertRaises( NotConfigured, CookiesMiddleware.from_crawler, - get_crawler(settings_dict={'COOKIES_ENABLED': False}) + get_crawler(settings_dict={"COOKIES_ENABLED": False}), ) def test_setting_default_cookies_enabled(self): self.assertIsInstance( - CookiesMiddleware.from_crawler(get_crawler()), - CookiesMiddleware + CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): self.assertIsInstance( CookiesMiddleware.from_crawler( - get_crawler(settings_dict={'COOKIES_ENABLED': True}) + get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), - CookiesMiddleware + CookiesMiddleware, ) def test_setting_enabled_cookies_debug(self): - crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True}) + crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) mw = CookiesMiddleware.from_crawler(crawler) with LogCapture( - 'scrapy.downloadermiddlewares.cookies', + "scrapy.downloadermiddlewares.cookies", propagate=False, level=logging.DEBUG, ) as log: - req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) + req = Request("http://scrapytest.org/") + res = Response( + "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} + ) mw.process_response(req, res, crawler.spider) - req2 = Request('http://scrapytest.org/sub1/') + req2 = Request("http://scrapytest.org/sub1/") mw.process_request(req2, crawler.spider) log.check( - ('scrapy.downloadermiddlewares.cookies', - 'DEBUG', - 'Received cookies from: <200 http://scrapytest.org/>\n' - 'Set-Cookie: C1=value1; path=/\n'), - ('scrapy.downloadermiddlewares.cookies', - 'DEBUG', - 'Sending cookies to:No Such Resource
\n" b"File not found.
\n" b" \n" - b"\n" + b"\n", ) elif run.getpath(request.url) == "/numbers": # signal was fired multiple times @@ -389,12 +407,16 @@ class EngineTest(unittest.TestCase): assert signals.spider_closed in run.signals_caught assert signals.headers_received in run.signals_caught - self.assertEqual({'spider': run.spider}, - run.signals_caught[signals.spider_opened]) - self.assertEqual({'spider': run.spider}, - run.signals_caught[signals.spider_idle]) - self.assertEqual({'spider': run.spider, 'reason': 'finished'}, - run.signals_caught[signals.spider_closed]) + self.assertEqual( + {"spider": run.spider}, run.signals_caught[signals.spider_opened] + ) + self.assertEqual( + {"spider": run.spider}, run.signals_caught[signals.spider_idle] + ) + self.assertEqual( + {"spider": run.spider, "reason": "finished"}, + run.signals_caught[signals.spider_closed], + ) @defer.inlineCallbacks def test_close_downloader(self): @@ -413,90 +435,17 @@ class EngineTest(unittest.TestCase): finally: yield e.stop() - @defer.inlineCallbacks - def test_close_spiders_downloader(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead"): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - self.assertEqual(len(e.open_spiders), 1) - yield e.close() - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_close_engine_spiders_downloader(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.open_spiders is deprecated, " - "please use ExecutionEngine.spider instead"): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - yield e.open_spider(TestSpider(), []) - e.start() - self.assertTrue(e.running) - yield e.close() - self.assertFalse(e.running) - self.assertEqual(len(e.open_spiders), 0) - - @defer.inlineCallbacks - def test_crawl_deprecated_spider_arg(self): - with pytest.warns(ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.crawl is deprecated"): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.crawl(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_download_deprecated_spider_arg(self): - with pytest.warns(ScrapyDeprecationWarning, - match="Passing a 'spider' argument to " - "ExecutionEngine.download is deprecated"): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.download(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_schedule(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead"): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - spider = TestSpider() - yield e.open_spider(spider, []) - e.start() - e.schedule(Request("data:,"), spider) - yield e.close() - - @defer.inlineCallbacks - def test_deprecated_has_capacity(self): - with pytest.warns(ScrapyDeprecationWarning, - match="ExecutionEngine.has_capacity is deprecated"): - e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) - self.assertTrue(e.has_capacity()) - spider = TestSpider() - yield e.open_spider(spider, []) - self.assertFalse(e.has_capacity()) - e.start() - yield e.close() - self.assertTrue(e.has_capacity()) - def test_short_timeout(self): args = ( sys.executable, - '-m', - 'scrapy.cmdline', - 'fetch', - '-s', - 'CLOSESPIDER_TIMEOUT=0.001', - '-s', - 'LOG_LEVEL=DEBUG', - 'http://toscrape.com', + "-m", + "scrapy.cmdline", + "fetch", + "-s", + "CLOSESPIDER_TIMEOUT=0.001", + "-s", + "LOG_LEVEL=DEBUG", + "http://toscrape.com", ) p = subprocess.Popen( args, @@ -506,7 +455,7 @@ class EngineTest(unittest.TestCase): def kill_proc(): p.kill() p.communicate() - assert False, 'Command took too much time to complete' + assert False, "Command took too much time to complete" timer = Timer(15, kill_proc) try: @@ -515,10 +464,10 @@ class EngineTest(unittest.TestCase): finally: timer.cancel() - self.assertNotIn(b'Traceback', stderr) + self.assertNotIn(b"Traceback", stderr) if __name__ == "__main__": - if len(sys.argv) > 1 and sys.argv[1] == 'runserver': + if len(sys.argv) > 1 and sys.argv[1] == "runserver": start_test_site(debug=True) reactor.run() diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 933e4067d..8dbb5b7ea 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -2,14 +2,13 @@ from testfixtures import LogCapture from twisted.internet import defer from scrapy.exceptions import StopDownload - from tests.test_engine import ( AttrsItemsSpider, + CrawlerRun, DataClassItemsSpider, DictItemsSpider, - TestSpider, - CrawlerRun, EngineTest, + TestSpider, ) @@ -22,22 +21,39 @@ class BytesReceivedCrawlerRun(CrawlerRun): class BytesReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): - for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): + for spider in ( + TestSpider, + DictItemsSpider, + AttrsItemsSpider, + DataClassItemsSpider, + ): run = BytesReceivedCrawlerRun(spider) with LogCapture() as log: yield run.run() - log.check_present(("scrapy.core.downloader.handlers.http11", - "DEBUG", - f"Download stopped forHello from HTTP2
{val}
-''' - return bytes(response, 'utf-8') +""" + return bytes(response, "utf-8") class DummySpider(Spider): - name = 'dummy' + name = "dummy" start_urls: list = [] def parse(self, response): @@ -51,50 +55,51 @@ class DummySpider(Spider): class Data: SMALL_SIZE = 1024 # 1 KB - LARGE_SIZE = 1024 ** 2 # 1 MB + LARGE_SIZE = 1024**2 # 1 MB STR_SMALL = generate_random_string(SMALL_SIZE) STR_LARGE = generate_random_string(LARGE_SIZE) EXTRA_SMALL = generate_random_string(1024 * 15) - EXTRA_LARGE = generate_random_string((1024 ** 2) * 15) + EXTRA_LARGE = generate_random_string((1024**2) * 15) HTML_SMALL = make_html_body(STR_SMALL) HTML_LARGE = make_html_body(STR_LARGE) - JSON_SMALL = {'data': STR_SMALL} - JSON_LARGE = {'data': STR_LARGE} + JSON_SMALL = {"data": STR_SMALL} + JSON_LARGE = {"data": STR_LARGE} - DATALOSS = b'Dataloss Content' - NO_CONTENT_LENGTH = b'This response do not have any content-length header' + DATALOSS = b"Dataloss Content" + NO_CONTENT_LENGTH = b"This response do not have any content-length header" class GetDataHtmlSmall(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'text/html; charset=UTF-8') + request.setHeader("Content-Type", "text/html; charset=UTF-8") return Data.HTML_SMALL class GetDataHtmlLarge(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'text/html; charset=UTF-8') + request.setHeader("Content-Type", "text/html; charset=UTF-8") return Data.HTML_LARGE class PostDataJsonMixin: @staticmethod def make_response(request: TxRequest, extra_data: str): + assert request.content is not None response = { - 'request-headers': {}, - 'request-body': json.loads(request.content.read()), - 'extra-data': extra_data + "request-headers": {}, + "request-body": json.loads(request.content.read()), + "extra-data": extra_data, } for k, v in request.requestHeaders.getAllRawHeaders(): - response['request-headers'][str(k, 'utf-8')] = str(v[0], 'utf-8') + response["request-headers"][str(k, "utf-8")] = str(v[0], "utf-8") - response_bytes = bytes(json.dumps(response), 'utf-8') - request.setHeader('Content-Type', 'application/json; charset=UTF-8') - request.setHeader('Content-Encoding', 'UTF-8') + response_bytes = bytes(json.dumps(response), "utf-8") + request.setHeader("Content-Type", "application/json; charset=UTF-8") + request.setHeader("Content-Encoding", "UTF-8") return response_bytes @@ -109,7 +114,6 @@ class PostDataJsonLarge(LeafResource, PostDataJsonMixin): class Dataloss(LeafResource): - def render_GET(self, request: TxRequest): request.setHeader(b"Content-Length", b"1024") self.deferRequest(request, 0, self._delayed_render, request) @@ -123,7 +127,7 @@ class Dataloss(LeafResource): class NoContentLengthHeader(LeafResource): def render_GET(self, request: TxRequest): - request.requestHeaders.removeHeader('Content-Length') + request.requestHeaders.removeHeader("Content-Length") self.deferRequest(request, 0, self._delayed_render, request) return NOT_DONE_YET @@ -140,57 +144,62 @@ class TimeoutResponse(LeafResource): class QueryParams(LeafResource): def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json; charset=UTF-8') - request.setHeader('Content-Encoding', 'UTF-8') + request.setHeader("Content-Type", "application/json; charset=UTF-8") + request.setHeader("Content-Encoding", "UTF-8") - query_params = {} + query_params: Dict[str, str] = {} + assert request.args is not None for k, v in request.args.items(): - query_params[str(k, 'utf-8')] = str(v[0], 'utf-8') + query_params[str(k, "utf-8")] = str(v[0], "utf-8") - return bytes(json.dumps(query_params), 'utf-8') + return bytes(json.dumps(query_params), "utf-8") class RequestHeaders(LeafResource): """Sends all the headers received as a response""" def render_GET(self, request: TxRequest): - request.setHeader('Content-Type', 'application/json; charset=UTF-8') - request.setHeader('Content-Encoding', 'UTF-8') + request.setHeader("Content-Type", "application/json; charset=UTF-8") + request.setHeader("Content-Encoding", "UTF-8") headers = {} for k, v in request.requestHeaders.getAllRawHeaders(): - headers[str(k, 'utf-8')] = str(v[0], 'utf-8') + headers[str(k, "utf-8")] = str(v[0], "utf-8") - return bytes(json.dumps(headers), 'utf-8') + return bytes(json.dumps(headers), "utf-8") -def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: - pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8") +def get_client_certificate( + key_file: Path, certificate_file: Path +) -> PrivateCertificate: + pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text( + encoding="utf-8" + ) return PrivateCertificate.loadPEM(pem) @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): - scheme = 'https' - key_file = Path(__file__).parent / 'keys' / 'localhost.key' - certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt' + scheme = "https" + key_file = Path(__file__).parent / "keys" / "localhost.key" + certificate_file = Path(__file__).parent / "keys" / "localhost.crt" def _init_resource(self): self.temp_directory = self.mktemp() Path(self.temp_directory).mkdir() r = File(self.temp_directory) - r.putChild(b'get-data-html-small', GetDataHtmlSmall()) - r.putChild(b'get-data-html-large', GetDataHtmlLarge()) + r.putChild(b"get-data-html-small", GetDataHtmlSmall()) + r.putChild(b"get-data-html-large", GetDataHtmlLarge()) - r.putChild(b'post-data-json-small', PostDataJsonSmall()) - r.putChild(b'post-data-json-large', PostDataJsonLarge()) + r.putChild(b"post-data-json-small", PostDataJsonSmall()) + r.putChild(b"post-data-json-large", PostDataJsonLarge()) - r.putChild(b'dataloss', Dataloss()) - r.putChild(b'no-content-length-header', NoContentLengthHeader()) - r.putChild(b'status', Status()) - r.putChild(b'query-params', QueryParams()) - r.putChild(b'timeout', TimeoutResponse()) - r.putChild(b'request-headers', RequestHeaders()) + r.putChild(b"dataloss", Dataloss()) + r.putChild(b"no-content-length-header", NoContentLengthHeader()) + r.putChild(b"status", Status()) + r.putChild(b"query-params", QueryParams()) + r.putChild(b"timeout", TimeoutResponse()) + r.putChild(b"request-headers", RequestHeaders()) return r @inlineCallbacks @@ -200,26 +209,35 @@ class Https2ClientProtocolTestCase(TestCase): self.site = Site(root, timeout=None) # Start server for testing - self.hostname = 'localhost' - context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file)) + self.hostname = "localhost" + context_factory = ssl_context_factory( + str(self.key_file), str(self.certificate_file) + ) - server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) + server_endpoint = SSL4ServerEndpoint( + reactor, 0, context_factory, interface=self.hostname + ) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port # Connect H2 client with server - self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) + self.client_certificate = get_client_certificate( + self.key_file, self.certificate_file + ) client_options = optionsForClientTLS( hostname=self.hostname, trustRoot=self.client_certificate, - acceptableProtocols=[b'h2'] + acceptableProtocols=[b"h2"], ) - uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) + uri = URI.fromBytes(bytes(self.get_url("/"), "utf-8")) self.conn_closed_deferred = Deferred() from scrapy.core.http2.protocol import H2ClientFactory + h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) - client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) + client_endpoint = SSL4ClientEndpoint( + reactor, self.hostname, self.port_number, client_options + ) self.client = yield client_endpoint.connect(h2_client_factory) @inlineCallbacks @@ -236,8 +254,8 @@ class Https2ClientProtocolTestCase(TestCase): :param path: Should have / at the starting compulsorily if not empty :return: Complete url """ - assert len(path) > 0 and (path[0] == '/' or path[0] == '&') - return f'{self.scheme}://{self.hostname}:{self.port_number}{path}' + assert len(path) > 0 and (path[0] == "/" or path[0] == "&") + return f"{self.scheme}://{self.hostname}:{self.port_number}{path}" def make_request(self, request: Request) -> Deferred: return self.client.request(request, DummySpider()) @@ -251,18 +269,15 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) - def _check_GET( - self, - request: Request, - expected_body, - expected_status - ): + def _check_GET(self, request: Request, expected_body, expected_status): def check_response(response: Response): self.assertEqual(response.status, expected_status) self.assertEqual(response.body, expected_body) self.assertEqual(response.request, request) - content_length = int(response.headers.get('Content-Length')) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) self.assertEqual(len(response.body), content_length) d = self.make_request(request) @@ -271,11 +286,11 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_GET_small_body(self): - request = Request(self.get_url('/get-data-html-small')) + request = Request(self.get_url("/get-data-html-small")) return self._check_GET(request, Data.HTML_SMALL, 200) def test_GET_large_body(self): - request = Request(self.get_url('/get-data-html-large')) + request = Request(self.get_url("/get-data-html-large")) return self._check_GET(request, Data.HTML_LARGE, 200) def _check_GET_x10(self, *args, **kwargs): @@ -286,16 +301,12 @@ class Https2ClientProtocolTestCase(TestCase): def test_GET_small_body_x10(self): return self._check_GET_x10( - Request(self.get_url('/get-data-html-small')), - Data.HTML_SMALL, - 200 + Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) def test_GET_large_body_x10(self): return self._check_GET_x10( - Request(self.get_url('/get-data-html-large')), - Data.HTML_LARGE, - 200 + Request(self.get_url("/get-data-html-large")), Data.HTML_LARGE, 200 ) def _check_POST_json( @@ -303,7 +314,7 @@ class Https2ClientProtocolTestCase(TestCase): request: Request, expected_request_body, expected_extra_data, - expected_status: int + expected_status: int, ): d = self.make_request(request) @@ -311,50 +322,52 @@ class Https2ClientProtocolTestCase(TestCase): self.assertEqual(response.status, expected_status) self.assertEqual(response.request, request) - content_length = int(response.headers.get('Content-Length')) + content_length_header = response.headers.get("Content-Length") + assert content_length_header is not None + content_length = int(content_length_header) self.assertEqual(len(response.body), content_length) # Parse the body - content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") body = json.loads(str(response.body, content_encoding)) - self.assertIn('request-body', body) - self.assertIn('extra-data', body) - self.assertIn('request-headers', body) + self.assertIn("request-body", body) + self.assertIn("extra-data", body) + self.assertIn("request-headers", body) - request_body = body['request-body'] + request_body = body["request-body"] self.assertEqual(request_body, expected_request_body) - extra_data = body['extra-data'] + extra_data = body["extra-data"] self.assertEqual(extra_data, expected_extra_data) # Check if headers were sent successfully - request_headers = body['request-headers'] + request_headers = body["request-headers"] for k, v in request.headers.items(): - k_str = str(k, 'utf-8') + k_str = str(k, "utf-8") self.assertIn(k_str, request_headers) - self.assertEqual(request_headers[k_str], str(v[0], 'utf-8')) + self.assertEqual(request_headers[k_str], str(v[0], "utf-8")) d.addCallback(assert_response) d.addErrback(self.fail) return d def test_POST_small_json(self): - request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) - return self._check_POST_json( - request, - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 + request = JsonRequest( + url=self.get_url("/post-data-json-small"), + method="POST", + data=Data.JSON_SMALL, ) + return self._check_POST_json(request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200) def test_POST_large_json(self): - request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) - return self._check_POST_json( - request, - Data.JSON_LARGE, - Data.EXTRA_LARGE, - 200 + request = JsonRequest( + url=self.get_url("/post-data-json-large"), + method="POST", + data=Data.JSON_LARGE, ) + return self._check_POST_json(request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200) def _check_POST_json_x10(self, *args, **kwargs): def get_deferred(): @@ -363,32 +376,36 @@ class Https2ClientProtocolTestCase(TestCase): return self._check_repeat(get_deferred, 10) def test_POST_small_json_x10(self): - request = JsonRequest(url=self.get_url('/post-data-json-small'), method='POST', data=Data.JSON_SMALL) + request = JsonRequest( + url=self.get_url("/post-data-json-small"), + method="POST", + data=Data.JSON_SMALL, + ) return self._check_POST_json_x10( - request, - Data.JSON_SMALL, - Data.EXTRA_SMALL, - 200 + request, Data.JSON_SMALL, Data.EXTRA_SMALL, 200 ) def test_POST_large_json_x10(self): - request = JsonRequest(url=self.get_url('/post-data-json-large'), method='POST', data=Data.JSON_LARGE) + request = JsonRequest( + url=self.get_url("/post-data-json-large"), + method="POST", + data=Data.JSON_LARGE, + ) return self._check_POST_json_x10( - request, - Data.JSON_LARGE, - Data.EXTRA_LARGE, - 200 + request, Data.JSON_LARGE, Data.EXTRA_LARGE, 200 ) @inlineCallbacks def test_invalid_negotiated_protocol(self): - with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"): - request = Request(url=self.get_url('/status?n=200')) + with mock.patch( + "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2" + ): + request = Request(url=self.get_url("/status?n=200")) with self.assertRaises(ResponseFailed): yield self.make_request(request) def test_cancel_request(self): - request = Request(url=self.get_url('/get-data-html-large')) + request = Request(url=self.get_url("/get-data-html-large")) def assert_response(response: Response): self.assertEqual(response.status, 499) @@ -402,13 +419,15 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_download_maxsize_exceeded(self): - request = Request(url=self.get_url('/get-data-html-large'), meta={'download_maxsize': 1000}) + request = Request( + url=self.get_url("/get-data-html-large"), meta={"download_maxsize": 1000} + ) def assert_cancelled_error(failure): self.assertIsInstance(failure.value, CancelledError) error_pattern = re.compile( - rf'Cancelling download of {request.url}: received response ' - rf'size \(\d*\) larger than download max size \(1000\)' + rf"Cancelling download of {request.url}: received response " + rf"size \(\d*\) larger than download max size \(1000\)" ) self.assertEqual(len(re.findall(error_pattern, str(failure.value))), 1) @@ -420,15 +439,18 @@ class Https2ClientProtocolTestCase(TestCase): def test_received_dataloss_response(self): """In case when value of Header Content-Length != len(Received Data) ProtocolError is raised""" - request = Request(url=self.get_url('/dataloss')) + request = Request(url=self.get_url("/dataloss")) def assert_failure(failure: Failure): self.assertTrue(len(failure.value.reasons) > 0) from h2.exceptions import InvalidBodyLengthError - self.assertTrue(any( - isinstance(error, InvalidBodyLengthError) - for error in failure.value.reasons - )) + + self.assertTrue( + any( + isinstance(error, InvalidBodyLengthError) + for error in failure.value.reasons + ) + ) d = self.make_request(request) d.addCallback(self.fail) @@ -436,13 +458,13 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_missing_content_length_header(self): - request = Request(url=self.get_url('/no-content-length-header')) + request = Request(url=self.get_url("/no-content-length-header")) def assert_content_length(response: Response): self.assertEqual(response.status, 200) self.assertEqual(response.body, Data.NO_CONTENT_LENGTH) self.assertEqual(response.request, request) - self.assertNotIn('Content-Length', response.headers) + self.assertNotIn("Content-Length", response.headers) d = self.make_request(request) d.addCallback(assert_content_length) @@ -450,40 +472,39 @@ class Https2ClientProtocolTestCase(TestCase): return d @inlineCallbacks - def _check_log_warnsize( - self, - request, - warn_pattern, - expected_body - ): - with self.assertLogs('scrapy.core.http2.stream', level='WARNING') as cm: + def _check_log_warnsize(self, request, warn_pattern, expected_body): + with self.assertLogs("scrapy.core.http2.stream", level="WARNING") as cm: response = yield self.make_request(request) self.assertEqual(response.status, 200) self.assertEqual(response.request, request) self.assertEqual(response.body, expected_body) # Check the warning is raised only once for this request - self.assertEqual(sum( - len(re.findall(warn_pattern, log)) - for log in cm.output - ), 1) + self.assertEqual( + sum(len(re.findall(warn_pattern, log)) for log in cm.output), 1 + ) @inlineCallbacks def test_log_expected_warnsize(self): - request = Request(url=self.get_url('/get-data-html-large'), meta={'download_warnsize': 1000}) + request = Request( + url=self.get_url("/get-data-html-large"), meta={"download_warnsize": 1000} + ) warn_pattern = re.compile( - rf'Expected response size \(\d*\) larger than ' - rf'download warn size \(1000\) in request {request}' + rf"Expected response size \(\d*\) larger than " + rf"download warn size \(1000\) in request {request}" ) yield self._check_log_warnsize(request, warn_pattern, Data.HTML_LARGE) @inlineCallbacks def test_log_received_warnsize(self): - request = Request(url=self.get_url('/no-content-length-header'), meta={'download_warnsize': 10}) + request = Request( + url=self.get_url("/no-content-length-header"), + meta={"download_warnsize": 10}, + ) warn_pattern = re.compile( - rf'Received more \(\d*\) bytes than download ' - rf'warn size \(10\) in request {request}' + rf"Received more \(\d*\) bytes than download " + rf"warn size \(10\) in request {request}" ) yield self._check_log_warnsize(request, warn_pattern, Data.NO_CONTENT_LENGTH) @@ -495,9 +516,7 @@ class Https2ClientProtocolTestCase(TestCase): def get_deferred(): return self._check_GET( - Request(self.get_url('/get-data-html-small')), - Data.HTML_SMALL, - 200 + Request(self.get_url("/get-data-html-small")), Data.HTML_SMALL, 200 ) return self._check_repeat(get_deferred, 500) @@ -511,20 +530,20 @@ class Https2ClientProtocolTestCase(TestCase): def assert_inactive_stream(failure): self.assertIsNotNone(failure.check(ResponseFailed)) from scrapy.core.http2.stream import InactiveStreamClosed - self.assertTrue(any( - isinstance(e, InactiveStreamClosed) - for e in failure.value.reasons - )) + + self.assertTrue( + any(isinstance(e, InactiveStreamClosed) for e in failure.value.reasons) + ) # Send 100 request (we do not check the result) for _ in range(100): - d = self.make_request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url("/get-data-html-small"))) d.addBoth(lambda _: None) d_list.append(d) # Now send 10 extra request and save the response deferred in a list for _ in range(10): - d = self.make_request(Request(self.get_url('/get-data-html-small'))) + d = self.make_request(Request(self.get_url("/get-data-html-small"))) d.addCallback(self.fail) d.addErrback(assert_inactive_stream) d_list.append(d) @@ -537,19 +556,21 @@ class Https2ClientProtocolTestCase(TestCase): def test_invalid_request_type(self): with self.assertRaises(TypeError): - self.make_request('https://InvalidDataTypePassed.com') + self.make_request("https://InvalidDataTypePassed.com") def test_query_parameters(self): params = { - 'a': generate_random_string(20), - 'b': generate_random_string(20), - 'c': generate_random_string(20), - 'd': generate_random_string(20) + "a": generate_random_string(20), + "b": generate_random_string(20), + "c": generate_random_string(20), + "d": generate_random_string(20), } - request = Request(self.get_url(f'/query-params?{urlencode(params)}')) + request = Request(self.get_url(f"/query-params?{urlencode(params)}")) def assert_query_params(response: Response): - content_encoding = str(response.headers[b'Content-Encoding'], 'utf-8') + content_encoding_header = response.headers[b"Content-Encoding"] + assert content_encoding_header is not None + content_encoding = str(content_encoding_header, "utf-8") data = json.loads(str(response.body, content_encoding)) self.assertEqual(data, params) @@ -565,7 +586,7 @@ class Https2ClientProtocolTestCase(TestCase): d_list = [] for status in [200, 404]: - request = Request(self.get_url(f'/status?n={status}')) + request = Request(self.get_url(f"/status?n={status}")) d = self.make_request(request) d.addCallback(assert_response_status, status) d.addErrback(self.fail) @@ -574,17 +595,24 @@ class Https2ClientProtocolTestCase(TestCase): return DeferredList(d_list, fireOnOneErrback=True) def test_response_has_correct_certificate_ip_address(self): - request = Request(self.get_url('/status?n=200')) + request = Request(self.get_url("/status?n=200")) def assert_metadata(response: Response): self.assertEqual(response.request, request) self.assertIsInstance(response.certificate, Certificate) + assert response.certificate # typing self.assertIsNotNone(response.certificate.original) - self.assertEqual(response.certificate.getIssuer(), self.client_certificate.getIssuer()) - self.assertTrue(response.certificate.getPublicKey().matches(self.client_certificate.getPublicKey())) + self.assertEqual( + response.certificate.getIssuer(), self.client_certificate.getIssuer() + ) + self.assertTrue( + response.certificate.getPublicKey().matches( + self.client_certificate.getPublicKey() + ) + ) self.assertIsInstance(response.ip_address, IPv4Address) - self.assertEqual(str(response.ip_address), '127.0.0.1') + self.assertEqual(str(response.ip_address), "127.0.0.1") d = self.make_request(request) d.addCallback(assert_metadata) @@ -597,10 +625,11 @@ class Https2ClientProtocolTestCase(TestCase): def assert_invalid_hostname(failure: Failure): from scrapy.core.http2.stream import InvalidHostname + self.assertIsNotNone(failure.check(InvalidHostname)) error_msg = str(failure.value) - self.assertIn('localhost', error_msg) - self.assertIn('127.0.0.1', error_msg) + self.assertIn("localhost", error_msg) + self.assertIn("127.0.0.1", error_msg) self.assertIn(str(request), error_msg) d = self.make_request(request) @@ -609,24 +638,24 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_invalid_hostname(self): - return self._check_invalid_netloc('https://notlocalhost.notlocalhostdomain') + return self._check_invalid_netloc("https://notlocalhost.notlocalhostdomain") def test_invalid_host_port(self): port = self.port_number + 1 - return self._check_invalid_netloc(f'https://127.0.0.1:{port}') + return self._check_invalid_netloc(f"https://127.0.0.1:{port}") def test_connection_stays_with_invalid_requests(self): d_list = [ self.test_invalid_hostname(), self.test_invalid_host_port(), self.test_GET_small_body(), - self.test_POST_small_json() + self.test_POST_small_json(), ] return DeferredList(d_list, fireOnOneErrback=True) def test_connection_timeout(self): - request = Request(self.get_url('/timeout')) + request = Request(self.get_url("/timeout")) d = self.make_request(request) # Update the timer to 1s to test connection timeout @@ -635,8 +664,12 @@ class Https2ClientProtocolTestCase(TestCase): def assert_timeout_error(failure: Failure): for err in failure.value.reasons: from scrapy.core.http2.protocol import H2ClientProtocol + if isinstance(err, TimeoutError): - self.assertIn(f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", str(err)) + self.assertIn( + f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s", + str(err), + ) break else: self.fail() @@ -646,20 +679,20 @@ class Https2ClientProtocolTestCase(TestCase): return d def test_request_headers_received(self): - request = Request(self.get_url('/request-headers'), headers={ - 'header-1': 'header value 1', - 'header-2': 'header value 2' - }) + request = Request( + self.get_url("/request-headers"), + headers={"header-1": "header value 1", "header-2": "header value 2"}, + ) d = self.make_request(request) def assert_request_headers(response: Response): self.assertEqual(response.status, 200) self.assertEqual(response.request, request) - response_headers = json.loads(str(response.body, 'utf-8')) + response_headers = json.loads(str(response.body, "utf-8")) self.assertIsInstance(response_headers, dict) for k, v in request.headers.items(): - k, v = str(k, 'utf-8'), str(v[0], 'utf-8') + k, v = str(k, "utf-8"), str(v[0], "utf-8") self.assertIn(k, response_headers) self.assertEqual(v, response_headers[k]) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 08420332c..9e43b72b0 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,15 +1,15 @@ -from urllib.parse import urlparse from unittest import TestCase +from urllib.parse import urlparse from scrapy.http import Request, Response from scrapy.http.cookies import WrappedRequest, WrappedResponse class WrappedRequestTest(TestCase): - def setUp(self): - self.request = Request("http://www.example.com/page.html", - headers={"Content-Type": "text/html"}) + self.request = Request( + "http://www.example.com/page.html", headers={"Content-Type": "text/html"} + ) self.wrapped = WrappedRequest(self.request) def test_get_full_url(self): @@ -29,35 +29,34 @@ class WrappedRequestTest(TestCase): self.assertFalse(self.wrapped.unverifiable) def test_is_unverifiable2(self): - self.request.meta['is_unverifiable'] = True + self.request.meta["is_unverifiable"] = True self.assertTrue(self.wrapped.is_unverifiable()) self.assertTrue(self.wrapped.unverifiable) def test_get_origin_req_host(self): - self.assertEqual(self.wrapped.origin_req_host, 'www.example.com') + self.assertEqual(self.wrapped.origin_req_host, "www.example.com") def test_has_header(self): - self.assertTrue(self.wrapped.has_header('content-type')) - self.assertFalse(self.wrapped.has_header('xxxxx')) + self.assertTrue(self.wrapped.has_header("content-type")) + self.assertFalse(self.wrapped.has_header("xxxxx")) def test_get_header(self): - self.assertEqual(self.wrapped.get_header('content-type'), 'text/html') - self.assertEqual(self.wrapped.get_header('xxxxx', 'def'), 'def') + self.assertEqual(self.wrapped.get_header("content-type"), "text/html") + self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def") def test_header_items(self): - self.assertEqual(self.wrapped.header_items(), - [('Content-Type', ['text/html'])]) + self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])]) def test_add_unredirected_header(self): - self.wrapped.add_unredirected_header('hello', 'world') - self.assertEqual(self.request.headers['hello'], b'world') + self.wrapped.add_unredirected_header("hello", "world") + self.assertEqual(self.request.headers["hello"], b"world") class WrappedResponseTest(TestCase): - def setUp(self): - self.response = Response("http://www.example.com/page.html", - headers={"Content-TYpe": "text/html"}) + self.response = Response( + "http://www.example.com/page.html", headers={"Content-TYpe": "text/html"} + ) self.wrapped = WrappedResponse(self.response) def test_info(self): @@ -65,4 +64,4 @@ class WrappedResponseTest(TestCase): def test_get_all(self): # get_all result must be native string - self.assertEqual(self.wrapped.get_all('content-type'), ['text/html']) + self.assertEqual(self.wrapped.get_all("content-type"), ["text/html"]) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 1ca936247..7db1eb8c5 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -1,42 +1,43 @@ -import unittest import copy +import unittest from scrapy.http import Headers class HeadersTest(unittest.TestCase): - def assertSortedEqual(self, first, second, msg=None): return self.assertEqual(sorted(first), sorted(second), msg) def test_basics(self): - h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234}) - assert h['Content-Type'] - assert h['Content-Length'] + h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) + assert h["Content-Type"] + assert h["Content-Length"] - self.assertRaises(KeyError, h.__getitem__, 'Accept') - self.assertEqual(h.get('Accept'), None) - self.assertEqual(h.getlist('Accept'), []) + self.assertRaises(KeyError, h.__getitem__, "Accept") + self.assertEqual(h.get("Accept"), None) + self.assertEqual(h.getlist("Accept"), []) - self.assertEqual(h.get('Accept', '*/*'), b'*/*') - self.assertEqual(h.getlist('Accept', '*/*'), [b'*/*']) - self.assertEqual(h.getlist('Accept', ['text/html', 'images/jpeg']), - [b'text/html', b'images/jpeg']) + self.assertEqual(h.get("Accept", "*/*"), b"*/*") + self.assertEqual(h.getlist("Accept", "*/*"), [b"*/*"]) + self.assertEqual( + h.getlist("Accept", ["text/html", "images/jpeg"]), + [b"text/html", b"images/jpeg"], + ) def test_single_value(self): h = Headers() - h['Content-Type'] = 'text/html' - self.assertEqual(h['Content-Type'], b'text/html') - self.assertEqual(h.get('Content-Type'), b'text/html') - self.assertEqual(h.getlist('Content-Type'), [b'text/html']) + h["Content-Type"] = "text/html" + self.assertEqual(h["Content-Type"], b"text/html") + self.assertEqual(h.get("Content-Type"), b"text/html") + self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) def test_multivalue(self): h = Headers() - h['X-Forwarded-For'] = hlist = ['ip1', 'ip2'] - self.assertEqual(h['X-Forwarded-For'], b'ip2') - self.assertEqual(h.get('X-Forwarded-For'), b'ip2') - self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) - assert h.getlist('X-Forwarded-For') is not hlist + h["X-Forwarded-For"] = hlist = ["ip1", "ip2"] + self.assertEqual(h["X-Forwarded-For"], b"ip2") + self.assertEqual(h.get("X-Forwarded-For"), b"ip2") + self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) + assert h.getlist("X-Forwarded-For") is not hlist def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) @@ -45,117 +46,121 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h.getlist("a"), [b"b", b"c"]) def test_encode_utf8(self): - h = Headers({'key': '\xa3'}, encoding='utf-8') + h = Headers({"key": "\xa3"}, encoding="utf-8") key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] - self.assertEqual(val[0], b'\xc2\xa3') + self.assertEqual(val[0], b"\xc2\xa3") def test_encode_latin1(self): - h = Headers({'key': '\xa3'}, encoding='latin1') + h = Headers({"key": "\xa3"}, encoding="latin1") key, val = dict(h).popitem() - self.assertEqual(val[0], b'\xa3') + self.assertEqual(val[0], b"\xa3") def test_encode_multiple(self): - h = Headers({'key': ['\xa3']}, encoding='utf-8') + h = Headers({"key": ["\xa3"]}, encoding="utf-8") key, val = dict(h).popitem() - self.assertEqual(val[0], b'\xc2\xa3') + self.assertEqual(val[0], b"\xc2\xa3") def test_delete_and_contains(self): h = Headers() - h['Content-Type'] = 'text/html' - assert 'Content-Type' in h - del h['Content-Type'] - assert 'Content-Type' not in h + h["Content-Type"] = "text/html" + assert "Content-Type" in h + del h["Content-Type"] + assert "Content-Type" not in h def test_setdefault(self): h = Headers() - hlist = ['ip1', 'ip2'] - olist = h.setdefault('X-Forwarded-For', hlist) - assert h.getlist('X-Forwarded-For') is not hlist - assert h.getlist('X-Forwarded-For') is olist + hlist = ["ip1", "ip2"] + olist = h.setdefault("X-Forwarded-For", hlist) + assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is olist h = Headers() - olist = h.setdefault('X-Forwarded-For', 'ip1') - self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1']) - assert h.getlist('X-Forwarded-For') is olist + olist = h.setdefault("X-Forwarded-For", "ip1") + self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1"]) + assert h.getlist("X-Forwarded-For") is olist def test_iterables(self): - idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']} + idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - self.assertDictEqual(dict(h), - {b'Content-Type': [b'text/html'], - b'X-Forwarded-For': [b'ip1', b'ip2']}) - self.assertSortedEqual(h.keys(), - [b'X-Forwarded-For', b'Content-Type']) - self.assertSortedEqual(h.items(), - [(b'X-Forwarded-For', [b'ip1', b'ip2']), - (b'Content-Type', [b'text/html'])]) - self.assertSortedEqual(h.values(), [b'ip2', b'text/html']) + self.assertDictEqual( + dict(h), + {b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"]}, + ) + self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) + self.assertSortedEqual( + h.items(), + [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])], + ) + self.assertSortedEqual(h.values(), [b"ip2", b"text/html"]) def test_update(self): h = Headers() - h.update({'Content-Type': 'text/html', - 'X-Forwarded-For': ['ip1', 'ip2']}) - self.assertEqual(h.getlist('Content-Type'), [b'text/html']) - self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) + h.update({"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}) + self.assertEqual(h.getlist("Content-Type"), [b"text/html"]) + self.assertEqual(h.getlist("X-Forwarded-For"), [b"ip1", b"ip2"]) def test_copy(self): - h1 = Headers({'header1': ['value1', 'value2']}) + h1 = Headers({"header1": ["value1", "value2"]}) h2 = copy.copy(h1) self.assertEqual(h1, h2) - self.assertEqual(h1.getlist('header1'), h2.getlist('header1')) - assert h1.getlist('header1') is not h2.getlist('header1') + self.assertEqual(h1.getlist("header1"), h2.getlist("header1")) + assert h1.getlist("header1") is not h2.getlist("header1") assert isinstance(h2, Headers) def test_appendlist(self): - h1 = Headers({'header1': 'value1'}) - h1.appendlist('header1', 'value3') - self.assertEqual(h1.getlist('header1'), [b'value1', b'value3']) + h1 = Headers({"header1": "value1"}) + h1.appendlist("header1", "value3") + self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) h1 = Headers() - h1.appendlist('header1', 'value1') - h1.appendlist('header1', 'value3') - self.assertEqual(h1.getlist('header1'), [b'value1', b'value3']) + h1.appendlist("header1", "value1") + h1.appendlist("header1", "value3") + self.assertEqual(h1.getlist("header1"), [b"value1", b"value3"]) def test_setlist(self): - h1 = Headers({'header1': 'value1'}) - self.assertEqual(h1.getlist('header1'), [b'value1']) - h1.setlist('header1', [b'value2', b'value3']) - self.assertEqual(h1.getlist('header1'), [b'value2', b'value3']) + h1 = Headers({"header1": "value1"}) + self.assertEqual(h1.getlist("header1"), [b"value1"]) + h1.setlist("header1", [b"value2", b"value3"]) + self.assertEqual(h1.getlist("header1"), [b"value2", b"value3"]) def test_setlistdefault(self): - h1 = Headers({'header1': 'value1'}) - h1.setlistdefault('header1', ['value2', 'value3']) - h1.setlistdefault('header2', ['value2', 'value3']) - self.assertEqual(h1.getlist('header1'), [b'value1']) - self.assertEqual(h1.getlist('header2'), [b'value2', b'value3']) + h1 = Headers({"header1": "value1"}) + h1.setlistdefault("header1", ["value2", "value3"]) + h1.setlistdefault("header2", ["value2", "value3"]) + self.assertEqual(h1.getlist("header1"), [b"value1"]) + self.assertEqual(h1.getlist("header2"), [b"value2", b"value3"]) def test_none_value(self): h1 = Headers() - h1['foo'] = 'bar' - h1['foo'] = None - h1.setdefault('foo', 'bar') - self.assertEqual(h1.get('foo'), None) - self.assertEqual(h1.getlist('foo'), []) + h1["foo"] = "bar" + h1["foo"] = None + h1.setdefault("foo", "bar") + self.assertEqual(h1.get("foo"), None) + self.assertEqual(h1.getlist("foo"), []) def test_int_value(self): - h1 = Headers({'hey': 5}) - h1['foo'] = 1 - h1.setdefault('bar', 2) - h1.setlist('buz', [1, 'dos', 3]) - self.assertEqual(h1.getlist('foo'), [b'1']) - self.assertEqual(h1.getlist('bar'), [b'2']) - self.assertEqual(h1.getlist('buz'), [b'1', b'dos', b'3']) - self.assertEqual(h1.getlist('hey'), [b'5']) + h1 = Headers({"hey": 5}) + h1["foo"] = 1 + h1.setdefault("bar", 2) + h1.setlist("buz", [1, "dos", 3]) + self.assertEqual(h1.getlist("foo"), [b"1"]) + self.assertEqual(h1.getlist("bar"), [b"2"]) + self.assertEqual(h1.getlist("buz"), [b"1", b"dos", b"3"]) + self.assertEqual(h1.getlist("hey"), [b"5"]) def test_invalid_value(self): - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers, {'foo': object()}) - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers().__setitem__, 'foo', object()) - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers().setdefault, 'foo', object()) - self.assertRaisesRegex(TypeError, 'Unsupported value type', - Headers().setlist, 'foo', [object()]) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers, {"foo": object()} + ) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers().__setitem__, "foo", object() + ) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers().setdefault, "foo", object() + ) + self.assertRaisesRegex( + TypeError, "Unsupported value type", Headers().setlist, "foo", [object()] + ) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9f7f1854f..6dc9ec8b7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,21 +1,29 @@ -import unittest -import re import json -import xmlrpc.client +import re +import unittest import warnings +import xmlrpc.client +from typing import Any, Dict, List from unittest import mock from urllib.parse import parse_qs, unquote_to_bytes, urlparse -from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse +from scrapy.http import ( + FormRequest, + Headers, + HtmlResponse, + JsonRequest, + Request, + XmlRpcRequest, +) +from scrapy.http.request import NO_CALLBACK from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): - request_class = Request - default_method = 'GET' - default_headers = {} - default_meta = {} + default_method = "GET" + default_headers: Dict[bytes, List[bytes]] = {} + default_meta: Dict[str, Any] = {} def test_init(self): # Request requires url in the __init__ method @@ -23,7 +31,7 @@ class RequestTest(unittest.TestCase): # url argument must be basestring self.assertRaises(TypeError, self.request_class, 123) - r = self.request_class('http://www.example.com') + r = self.request_class("http://www.example.com") r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -36,7 +44,9 @@ class RequestTest(unittest.TestCase): meta = {"lala": "lolo"} headers = {b"caca": b"coco"} - r = self.request_class("http://www.example.com", meta=meta, headers=headers, body="a body") + r = self.request_class( + "http://www.example.com", meta=meta, headers=headers, body="a body" + ) assert r.meta is not meta self.assertEqual(r.meta, meta) @@ -45,22 +55,22 @@ class RequestTest(unittest.TestCase): def test_url_scheme(self): # This test passes by not raising any (ValueError) exception - self.request_class('http://example.org') - self.request_class('https://example.org') - self.request_class('s3://example.org') - self.request_class('ftp://example.org') - self.request_class('about:config') - self.request_class('data:,Hello%2C%20World!') + self.request_class("http://example.org") + self.request_class("https://example.org") + self.request_class("s3://example.org") + self.request_class("ftp://example.org") + self.request_class("about:config") + self.request_class("data:,Hello%2C%20World!") def test_url_no_scheme(self): - self.assertRaises(ValueError, self.request_class, 'foo') - self.assertRaises(ValueError, self.request_class, '/foo/') - self.assertRaises(ValueError, self.request_class, '/foo:bar') + self.assertRaises(ValueError, self.request_class, "foo") + self.assertRaises(ValueError, self.request_class, "/foo/") + self.assertRaises(ValueError, self.request_class, "/foo:bar") def test_headers(self): # Different ways of setting headers attribute - url = 'http://www.scrapy.org' - headers = {b'Accept': 'gzip', b'Custom-Header': 'nothing to tell you'} + url = "http://www.scrapy.org" + headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"} r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) @@ -69,15 +79,15 @@ class RequestTest(unittest.TestCase): self.assertFalse(p.headers is r.headers) # headers must not be unicode - h = Headers({'key1': 'val1', 'key2': 'val2'}) - h['newkey'] = 'newval' + h = Headers({"key1": "val1", "key2": "val2"}) + h["newkey"] = "newval" for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: self.assertIsInstance(s, bytes) def test_eq(self): - url = 'http://www.scrapy.org' + url = "http://www.scrapy.org" r1 = self.request_class(url=url) r2 = self.request_class(url=url) self.assertNotEqual(r1, r2) @@ -115,13 +125,17 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") # should be same as above - r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + r2 = self.request_class( + url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8" + ) self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + r3 = self.request_class( + url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1" + ) self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") def test_url_encoding_nonutf8_untouched(self): @@ -154,27 +168,35 @@ class RequestTest(unittest.TestCase): def test_body(self): r1 = self.request_class(url="http://www.example.com/") - assert r1.body == b'' + assert r1.body == b"" r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, 'utf-8') # default encoding + self.assertEqual(r2.encoding, "utf-8") # default encoding - r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8') + r3 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8" + ) assert isinstance(r3.body, bytes) self.assertEqual(r3.body, b"Price: \xc2\xa3100") - r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1') + r4 = self.request_class( + url="http://www.example.com/", body="Price: \xa3100", encoding="latin1" + ) assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") def test_ajax_url(self): # ascii url r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") + self.assertEqual( + r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" + ) # unicode url r = self.request_class(url="http://www.example.com/ajax.html#!key=value") - self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") + self.assertEqual( + r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue" + ) def test_copy(self): """Test Request copy""" @@ -182,10 +204,14 @@ class RequestTest(unittest.TestCase): def somecallback(): pass - r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'], - callback=somecallback, errback=somecallback) - r1.meta['foo'] = 'bar' - r1.cb_kwargs['key'] = 'value' + r1 = self.request_class( + "http://www.example.com", + flags=["f1", "f2"], + callback=somecallback, + errback=somecallback, + ) + r1.meta["foo"] = "bar" + r1.cb_kwargs["key"] = "value" r2 = r1.copy() # make sure copy does not propagate callbacks @@ -199,7 +225,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure cb_kwargs dict is shallow copied - assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical" + assert ( + r1.cb_kwargs is not r2.cb_kwargs + ), "cb_kwargs must be a shallow copy, not identical" self.assertEqual(r1.cb_kwargs, r2.cb_kwargs) # make sure meta dict is shallow copied @@ -207,7 +235,9 @@ class RequestTest(unittest.TestCase): self.assertEqual(r1.meta, r2.meta) # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical" + assert ( + r1.headers is not r2.headers + ), "headers must be a shallow copy, not identical" self.assertEqual(r1.headers, r2.headers) self.assertEqual(r1.encoding, r2.encoding) self.assertEqual(r1.dont_filter, r2.dont_filter) @@ -220,27 +250,31 @@ class RequestTest(unittest.TestCase): class CustomRequest(self.request_class): pass - r1 = CustomRequest('http://www.example.com') + r1 = CustomRequest("http://www.example.com") r2 = r1.copy() assert isinstance(r2, CustomRequest) def test_replace(self): """Test Request.replace() method""" - r1 = self.request_class("http://www.example.com", method='GET') + r1 = self.request_class("http://www.example.com", method="GET") hdrs = Headers(r1.headers) - hdrs[b'key'] = b'value' + hdrs[b"key"] = b"value" r2 = r1.replace(method="POST", body="New body", headers=hdrs) self.assertEqual(r1.url, r2.url) self.assertEqual((r1.method, r2.method), ("GET", "POST")) - self.assertEqual((r1.body, r2.body), (b'', b"New body")) + self.assertEqual((r1.body, r2.body), (b"", b"New body")) self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) # Empty attributes (which may fail if not compared properly) - r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) - r4 = r3.replace(url="http://www.example.com/2", body=b'', meta={}, dont_filter=False) + r3 = self.request_class( + "http://www.example.com", meta={"a": 1}, dont_filter=True + ) + r4 = r3.replace( + url="http://www.example.com/2", body=b"", meta={}, dont_filter=False + ) self.assertEqual(r4.url, "http://www.example.com/2") - self.assertEqual(r4.body, b'') + self.assertEqual(r4.body, b"") self.assertEqual(r4.meta, {}) assert r4.dont_filter is False @@ -250,45 +284,57 @@ class RequestTest(unittest.TestCase): def test_immutable_attributes(self): r = self.request_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') - self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") + self.assertRaises(AttributeError, setattr, r, "body", "xxx") def test_callback_and_errback(self): def a_function(): pass - r1 = self.request_class('http://example.com') + r1 = self.request_class("http://example.com") self.assertIsNone(r1.callback) self.assertIsNone(r1.errback) - r2 = self.request_class('http://example.com', callback=a_function) + r2 = self.request_class("http://example.com", callback=a_function) self.assertIs(r2.callback, a_function) self.assertIsNone(r2.errback) - r3 = self.request_class('http://example.com', errback=a_function) + r3 = self.request_class("http://example.com", errback=a_function) self.assertIsNone(r3.callback) self.assertIs(r3.errback, a_function) r4 = self.request_class( - url='http://example.com', + url="http://example.com", callback=a_function, errback=a_function, ) self.assertIs(r4.callback, a_function) self.assertIs(r4.errback, a_function) + r5 = self.request_class( + url="http://example.com", + callback=NO_CALLBACK, + errback=NO_CALLBACK, + ) + self.assertIs(r5.callback, NO_CALLBACK) + self.assertIs(r5.errback, NO_CALLBACK) + def test_callback_and_errback_type(self): with self.assertRaises(TypeError): - self.request_class('http://example.com', callback='a_function') + self.request_class("http://example.com", callback="a_function") with self.assertRaises(TypeError): - self.request_class('http://example.com', errback='a_function') + self.request_class("http://example.com", errback="a_function") with self.assertRaises(TypeError): self.request_class( - url='http://example.com', - callback='a_function', - errback='a_function', + url="http://example.com", + callback="a_function", + errback="a_function", ) + def test_no_callback(self): + with self.assertRaises(RuntimeError): + NO_CALLBACK() + def test_from_curl(self): # Note: more curated tests regarding curl conversion are in # `test_utils_curl.py` @@ -311,37 +357,48 @@ class RequestTest(unittest.TestCase): r = self.request_class.from_curl(curl_command) self.assertEqual(r.method, "POST") self.assertEqual(r.url, "http://httpbin.org/post") - self.assertEqual(r.body, - b"custname=John+Smith&custtel=500&custemail=jsmith%40" - b"example.org&size=small&topping=cheese&topping=onion" - b"&delivery=12%3A15&comments=") - self.assertEqual(r.cookies, { - '_gauges_unique_year': '1', - '_gauges_unique': '1', - '_gauges_unique_month': '1', - '_gauges_unique_hour': '1', - '_gauges_unique_day': '1' - }) - self.assertEqual(r.headers, { - b'Origin': [b'http://httpbin.org'], - b'Accept-Encoding': [b'gzip, deflate'], - b'Accept-Language': [b'en-US,en;q=0.9,ru;q=0.8,es;q=0.7'], - b'Upgrade-Insecure-Requests': [b'1'], - b'User-Agent': [b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.' - b'36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202' - b'.75 Chrome/62.0.3202.75 Safari/537.36'], - b'Content-Type': [b'application /x-www-form-urlencoded'], - b'Accept': [b'text/html,application/xhtml+xml,application/xml;q=0.' - b'9,image/webp,image/apng,*/*;q=0.8'], - b'Cache-Control': [b'max-age=0'], - b'Referer': [b'http://httpbin.org/forms/post'], - b'Connection': [b'keep-alive']}) + self.assertEqual( + r.body, + b"custname=John+Smith&custtel=500&custemail=jsmith%40" + b"example.org&size=small&topping=cheese&topping=onion" + b"&delivery=12%3A15&comments=", + ) + self.assertEqual( + r.cookies, + { + "_gauges_unique_year": "1", + "_gauges_unique": "1", + "_gauges_unique_month": "1", + "_gauges_unique_hour": "1", + "_gauges_unique_day": "1", + }, + ) + self.assertEqual( + r.headers, + { + b"Origin": [b"http://httpbin.org"], + b"Accept-Encoding": [b"gzip, deflate"], + b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"], + b"Upgrade-Insecure-Requests": [b"1"], + b"User-Agent": [ + b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537." + b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202" + b".75 Chrome/62.0.3202.75 Safari/537.36" + ], + b"Content-Type": [b"application /x-www-form-urlencoded"], + b"Accept": [ + b"text/html,application/xhtml+xml,application/xml;q=0." + b"9,image/webp,image/apng,*/*;q=0.8" + ], + b"Cache-Control": [b"max-age=0"], + b"Referer": [b"http://httpbin.org/forms/post"], + b"Connection": [b"keep-alive"], + }, + ) def test_from_curl_with_kwargs(self): r = self.request_class.from_curl( - 'curl -X PATCH "http://example.org"', - method="POST", - meta={'key': 'value'} + 'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"} ) self.assertEqual(r.method, "POST") self.assertEqual(r.meta, {"key": "value"}) @@ -349,13 +406,13 @@ class RequestTest(unittest.TestCase): def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter('ignore') + warnings.simplefilter("ignore") r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) self.assertEqual(r.method, "DELETE") - # If `ignore_unknon_options` is set to `False` it raises an error with + # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z self.assertRaises( ValueError, @@ -367,7 +424,6 @@ class RequestTest(unittest.TestCase): class FormRequestTest(RequestTest): - request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): @@ -377,68 +433,85 @@ class FormRequestTest(RequestTest): def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) - self.assertEqual(r1.body, b'') + self.assertEqual(r1.body, b"") def test_formdata_overrides_querystring(self): - data = (('a', 'one'), ('a', 'two'), ('b', '2')) - url = self.request_class('http://www.example.com/?a=0&b=1&c=3#fragment', - method='GET', formdata=data).url.split('#')[0] - fs = _qs(self.request_class(url, method='GET', formdata=data)) - self.assertEqual(set(fs[b'a']), {b'one', b'two'}) - self.assertEqual(fs[b'b'], [b'2']) - self.assertIsNone(fs.get(b'c')) + data = (("a", "one"), ("a", "two"), ("b", "2")) + url = self.request_class( + "http://www.example.com/?a=0&b=1&c=3#fragment", method="GET", formdata=data + ).url.split("#")[0] + fs = _qs(self.request_class(url, method="GET", formdata=data)) + self.assertEqual(set(fs[b"a"]), {b"one", b"two"}) + self.assertEqual(fs[b"b"], [b"2"]) + self.assertIsNone(fs.get(b"c")) - data = {'a': '1', 'b': '2'} - fs = _qs(self.request_class('http://www.example.com/', method='GET', formdata=data)) - self.assertEqual(fs[b'a'], [b'1']) - self.assertEqual(fs[b'b'], [b'2']) + data = {"a": "1", "b": "2"} + fs = _qs( + self.request_class("http://www.example.com/", method="GET", formdata=data) + ) + self.assertEqual(fs[b"a"], [b"1"]) + self.assertEqual(fs[b"b"], [b"2"]) def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {b'one': b'two', b'price': b'\xc2\xa3 100'} + data = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'utf-8') - self.assertQueryEqual(r2.body, b'price=%C2%A3+100&one=two') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "utf-8") + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_default_encoding_textual_data(self): # using default encoding (utf-8) - data = {'µ one': 'two', 'price': '£ 100'} + data = {"µ one": "two", "price": "£ 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'utf-8') - self.assertQueryEqual(r2.body, b'price=%C2%A3+100&%C2%B5+one=two') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "utf-8") + self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'} + data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'utf-8') - self.assertQueryEqual(r2.body, b'%C2%B5one=two&price%C2%A3=%C2%A3+100') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "utf-8") + self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_custom_encoding_bytes(self): - data = {b'\xb5 one': b'two', b'price': b'\xa3 100'} - r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') - self.assertEqual(r2.method, 'POST') - self.assertEqual(r2.encoding, 'latin1') - self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two') - self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') + data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + r2 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + self.assertEqual(r2.method, "POST") + self.assertEqual(r2.encoding, "latin1") + self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + self.assertEqual( + r2.headers[b"Content-Type"], b"application/x-www-form-urlencoded" + ) def test_custom_encoding_textual_data(self): - data = {'price': '£ 100'} - r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') - self.assertEqual(r3.encoding, 'latin1') - self.assertEqual(r3.body, b'price=%A3+100') + data = {"price": "£ 100"} + r3 = self.request_class( + "http://www.example.com", formdata=data, encoding="latin1" + ) + self.assertEqual(r3.encoding, "latin1") + self.assertEqual(r3.body, b"price=%A3+100") def test_multi_key_values(self): # using multiples values for a single key - data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']} + data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') + self.assertQueryEqual( + r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" + ) def test_from_response_post(self): response = _buildresponse( @@ -447,17 +520,22 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(set(fs[b'test']), {b'val1', b'val2'}) - self.assertEqual(set(fs[b'one']), {b'two', b'three'}) - self.assertEqual(fs[b'test2'], [b'xxx']) - self.assertEqual(fs[b'six'], [b'seven']) + self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) + self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) + self.assertEqual(fs[b"test2"], [b"xxx"]) + self.assertEqual(fs[b"six"], [b"seven"]) def test_from_response_post_nonascii_bytes_utf8(self): response = _buildresponse( @@ -466,17 +544,22 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs['test £']), {'val1', 'val2'}) - self.assertEqual(set(fs['one']), {'two', 'three'}) - self.assertEqual(fs['test2'], ['xxx µ']) - self.assertEqual(fs['six'], ['seven']) + self.assertEqual(set(fs["test £"]), {"val1", "val2"}) + self.assertEqual(set(fs["one"]), {"two", "three"}) + self.assertEqual(fs["test2"], ["xxx µ"]) + self.assertEqual(fs["six"], ["seven"]) def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -486,18 +569,22 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html", - encoding='latin1', + encoding="latin1", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} ) - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") - fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertEqual(set(fs['test £']), {'val1', 'val2'}) - self.assertEqual(set(fs['one']), {'two', 'three'}) - self.assertEqual(fs['test2'], ['xxx µ']) - self.assertEqual(fs['six'], ['seven']) + fs = _qs(req, to_unicode=True, encoding="latin1") + self.assertEqual(set(fs["test £"]), {"val1", "val2"}) + self.assertEqual(set(fs["one"]), {"two", "three"}) + self.assertEqual(fs["test2"], ["xxx µ"]) + self.assertEqual(fs["six"], ["seven"]) def test_from_response_post_nonascii_unicode(self): response = _buildresponse( @@ -506,42 +593,46 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) + url="http://www.example.com/this/list.html", + ) + req = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers[b"Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs['test £']), {'val1', 'val2'}) - self.assertEqual(set(fs['one']), {'two', 'three'}) - self.assertEqual(fs['test2'], ['xxx µ']) - self.assertEqual(fs['six'], ['seven']) + self.assertEqual(set(fs["test £"]), {"val1", "val2"}) + self.assertEqual(set(fs["one"]), {"two", "three"}) + self.assertEqual(fs["test2"], ["xxx µ"]) + self.assertEqual(fs["six"], ["seven"]) def test_from_response_duplicate_form_key(self): - response = _buildresponse( - '', - url='http://www.example.com') + response = _buildresponse("", url="http://www.example.com") req = self.request_class.from_response( response=response, - method='GET', - formdata=(('foo', 'bar'), ('foo', 'baz')), + method="GET", + formdata=(("foo", "bar"), ("foo", "baz")), ) - self.assertEqual(urlparse(req.url).hostname, 'www.example.com') - self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') + self.assertEqual(urlparse(req.url).hostname, "www.example.com") + self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz") def test_from_response_override_duplicate_form_key(self): response = _buildresponse( """""") + """ + ) req = self.request_class.from_response( - response, - formdata=(('two', '2'), ('two', '4'))) + response, formdata=(("two", "2"), ("two", "4")) + ) fs = _qs(req) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2', b'4']) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2", b"4"]) def test_from_response_extra_headers(self): response = _buildresponse( @@ -549,15 +640,18 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( response=response, - formdata={'one': ['two', 'three'], 'six': 'seven'}, + formdata={"one": ["two", "three"], "six": "seven"}, headers={"Accept-Encoding": "gzip,deflate"}, ) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') - self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) + self.assertEqual(req.headers["Accept-Encoding"], b"gzip,deflate") def test_from_response_get(self): response = _buildresponse( @@ -566,60 +660,67 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") - r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) - self.assertEqual(r1.method, 'GET') + url="http://www.example.com/this/list.html", + ) + r1 = self.request_class.from_response( + response, formdata={"one": ["two", "three"], "six": "seven"} + ) + self.assertEqual(r1.method, "GET") self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") fs = _qs(r1) - self.assertEqual(set(fs[b'test']), {b'val1', b'val2'}) - self.assertEqual(set(fs[b'one']), {b'two', b'three'}) - self.assertEqual(fs[b'test2'], [b'xxx']) - self.assertEqual(fs[b'six'], [b'seven']) + self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"}) + self.assertEqual(set(fs[b"one"]), {b"two", b"three"}) + self.assertEqual(fs[b"test2"], [b"xxx"]) + self.assertEqual(fs[b"six"], [b"seven"]) def test_from_response_override_params(self): response = _buildresponse( """""") - req = self.request_class.from_response(response, formdata={'two': '2'}) + """ + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2']) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2"]) def test_from_response_drop_params(self): response = _buildresponse( """""") - req = self.request_class.from_response(response, formdata={'two': None}) + """ + ) + req = self.request_class.from_response(response, formdata={"two": None}) fs = _qs(req) - self.assertEqual(fs[b'one'], [b'1']) - self.assertNotIn(b'two', fs) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertNotIn(b"two", fs) def test_from_response_override_method(self): response = _buildresponse( - ''' + """ - ''') + """ + ) request = FormRequest.from_response(response) - self.assertEqual(request.method, 'GET') - request = FormRequest.from_response(response, method='POST') - self.assertEqual(request.method, 'POST') + self.assertEqual(request.method, "GET") + request = FormRequest.from_response(response, method="POST") + self.assertEqual(request.method, "POST") def test_from_response_override_url(self): response = _buildresponse( - ''' + """ - ''') + """ + ) request = FormRequest.from_response(response) - self.assertEqual(request.url, 'http://example.com/app') - request = FormRequest.from_response(response, url='http://foo.bar/absolute') - self.assertEqual(request.url, 'http://foo.bar/absolute') - request = FormRequest.from_response(response, url='/relative') - self.assertEqual(request.url, 'http://example.com/relative') + self.assertEqual(request.url, "http://example.com/app") + request = FormRequest.from_response(response, url="http://foo.bar/absolute") + self.assertEqual(request.url, "http://foo.bar/absolute") + request = FormRequest.from_response(response, url="/relative") + self.assertEqual(request.url, "http://example.com/relative") def test_from_response_case_insensitive(self): response = _buildresponse( @@ -627,12 +728,13 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs[b'clickable1'], [b'clicked1']) - self.assertFalse(b'i1' in fs, fs) # xpath in _get_inputs() - self.assertFalse(b'clickable2' in fs, fs) # xpath in _get_clickable() + self.assertEqual(fs[b"clickable1"], [b"clicked1"]) + self.assertFalse(b"i1" in fs, fs) # xpath in _get_inputs() + self.assertFalse(b"clickable2" in fs, fs) # xpath in _get_clickable() def test_from_response_submit_first_clickable(self): response = _buildresponse( @@ -641,13 +743,14 @@ class FormRequestTest(RequestTest): - """) - req = self.request_class.from_response(response, formdata={'two': '2'}) + """ + ) + req = self.request_class.from_response(response, formdata={"two": "2"}) fs = _qs(req) - self.assertEqual(fs[b'clickable1'], [b'clicked1']) - self.assertFalse(b'clickable2' in fs, fs) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2']) + self.assertEqual(fs[b"clickable1"], [b"clicked1"]) + self.assertFalse(b"clickable2" in fs, fs) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2"]) def test_from_response_submit_not_first_clickable(self): response = _buildresponse( @@ -656,15 +759,16 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, formdata={'two': '2'}, clickdata={'name': 'clickable2'} + response, formdata={"two": "2"}, clickdata={"name": "clickable2"} ) fs = _qs(req) - self.assertEqual(fs[b'clickable2'], [b'clicked2']) - self.assertFalse(b'clickable1' in fs, fs) - self.assertEqual(fs[b'one'], [b'1']) - self.assertEqual(fs[b'two'], [b'2']) + self.assertEqual(fs[b"clickable2"], [b"clicked2"]) + self.assertFalse(b"clickable1" in fs, fs) + self.assertEqual(fs[b"one"], [b"1"]) + self.assertEqual(fs[b"two"], [b"2"]) def test_from_response_dont_submit_image_as_input(self): response = _buildresponse( @@ -672,10 +776,11 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v']}) + self.assertEqual(fs, {b"i1": [b"i1v"]}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -684,20 +789,22 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) def test_from_response_clickdata_does_not_ignore_image(self): response = _buildresponse( """""") + """ + ) req = self.request_class.from_response(response) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b"i2v"]}) def test_from_response_multiple_clickdata(self): response = _buildresponse( @@ -706,14 +813,15 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, clickdata={'name': 'clickable', 'value': 'clicked2'} + response, clickdata={"name": "clickable", "value": "clicked2"} ) fs = _qs(req) - self.assertEqual(fs[b'clickable'], [b'clicked2']) - self.assertEqual(fs[b'one'], [b'clicked1']) - self.assertEqual(fs[b'two'], [b'clicked2']) + self.assertEqual(fs[b"clickable"], [b"clicked2"]) + self.assertEqual(fs[b"one"], [b"clicked1"]) + self.assertEqual(fs[b"two"], [b"clicked2"]) def test_from_response_unicode_clickdata(self): response = _buildresponse( @@ -722,12 +830,13 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, clickdata={'name': 'price in \u00a3'} + response, clickdata={"name": "price in \u00a3"} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs['price in \u00a3']) + self.assertTrue(fs["price in \u00a3"]) def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( @@ -737,12 +846,13 @@ class FormRequestTest(RequestTest): """, - encoding='latin1') - req = self.request_class.from_response( - response, clickdata={'name': 'price in \u00a5'} + encoding="latin1", ) - fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertTrue(fs['price in \u00a5']) + req = self.request_class.from_response( + response, clickdata={"name": "price in \u00a5"} + ) + fs = _qs(req, to_unicode=True, encoding="latin1") + self.assertTrue(fs["price in \u00a5"]) def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -754,22 +864,25 @@ class FormRequestTest(RequestTest): - """) + """ + ) req = self.request_class.from_response( - response, formname='form2', clickdata={'name': 'clickable'} + response, formname="form2", clickdata={"name": "clickable"} ) fs = _qs(req) - self.assertEqual(fs[b'clickable'], [b'clicked2']) - self.assertEqual(fs[b'field2'], [b'value2']) - self.assertFalse(b'field1' in fs, fs) + self.assertEqual(fs[b"clickable"], [b"clicked2"]) + self.assertEqual(fs[b"field2"], [b"value2"]) + self.assertFalse(b"field1" in fs, fs) def test_from_response_override_clickable(self): - response = _buildresponse('''''') + response = _buildresponse( + """""" + ) req = self.request_class.from_response( - response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'} + response, formdata={"clickme": "two"}, clickdata={"name": "clickme"} ) fs = _qs(req) - self.assertEqual(fs[b'clickme'], [b'two']) + self.assertEqual(fs[b"clickme"], [b"two"]) def test_from_response_dont_click(self): response = _buildresponse( @@ -778,11 +891,12 @@ class FormRequestTest(RequestTest): - """) + """ + ) r1 = self.request_class.from_response(response, dont_click=True) fs = _qs(r1) - self.assertFalse(b'clickable1' in fs, fs) - self.assertFalse(b'clickable2' in fs, fs) + self.assertFalse(b"clickable1" in fs, fs) + self.assertFalse(b"clickable2" in fs, fs) def test_from_response_ambiguous_clickdata(self): response = _buildresponse( @@ -792,17 +906,27 @@ class FormRequestTest(RequestTest): - """) - self.assertRaises(ValueError, self.request_class.from_response, - response, clickdata={'type': 'submit'}) + """ + ) + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + clickdata={"type": "submit"}, + ) def test_from_response_non_matching_clickdata(self): response = _buildresponse( """""") - self.assertRaises(ValueError, self.request_class.from_response, - response, clickdata={'nonexistent': 'notme'}) + """ + ) + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + clickdata={"nonexistent": "notme"}, + ) def test_from_response_nr_index_clickdata(self): response = _buildresponse( @@ -810,32 +934,37 @@ class FormRequestTest(RequestTest): - """) - req = self.request_class.from_response(response, clickdata={'nr': 1}) + """ + ) + req = self.request_class.from_response(response, clickdata={"nr": 1}) fs = _qs(req) - self.assertIn(b'clickable2', fs) - self.assertNotIn(b'clickable1', fs) + self.assertIn(b"clickable2", fs) + self.assertNotIn(b"clickable1", fs) def test_from_response_invalid_nr_index_clickdata(self): response = _buildresponse( """ - """) - self.assertRaises(ValueError, self.request_class.from_response, - response, clickdata={'nr': 1}) + """ + ) + self.assertRaises( + ValueError, self.request_class.from_response, response, clickdata={"nr": 1} + ) def test_from_response_errors_noform(self): response = _buildresponse("""""") self.assertRaises(ValueError, self.request_class.from_response, response) def test_from_response_invalid_html5(self): - response = _buildresponse("""""") - req = self.request_class.from_response(response, formdata={'bar': 'buz'}) + response = _buildresponse( + """""" + ) + req = self.request_class.from_response(response, formdata={"bar": "buz"}) fs = _qs(req) - self.assertEqual(fs, {b'foo': [b'xxx'], b'bar': [b'buz']}) + self.assertEqual(fs, {b"foo": [b"xxx"], b"bar": [b"buz"]}) def test_from_response_errors_formnumber(self): response = _buildresponse( @@ -843,20 +972,26 @@ class FormRequestTest(RequestTest): - """) - self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1) + """ + ) + self.assertRaises( + IndexError, self.request_class.from_response, response, formnumber=1 + ) def test_from_response_noformname(self): response = _buildresponse( """""") - r1 = self.request_class.from_response(response, formdata={'two': '3'}) - self.assertEqual(r1.method, 'POST') - self.assertEqual(r1.headers['Content-type'], b'application/x-www-form-urlencoded') + """ + ) + r1 = self.request_class.from_response(response, formdata={"two": "3"}) + self.assertEqual(r1.method, "POST") + self.assertEqual( + r1.headers["Content-type"], b"application/x-www-form-urlencoded" + ) fs = _qs(r1) - self.assertEqual(fs, {b'one': [b'1'], b'two': [b'3']}) + self.assertEqual(fs, {b"one": [b"1"], b"two": [b"3"]}) def test_from_response_formname_exists(self): response = _buildresponse( @@ -867,24 +1002,26 @@ class FormRequestTest(RequestTest): """) + """ + ) r1 = self.request_class.from_response(response, formname="form2") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) + self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - def test_from_response_formname_notexist(self): + def test_from_response_formname_nonexistent(self): response = _buildresponse( """ """) + """ + ) r1 = self.request_class.from_response(response, formname="form3") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'one': [b'1']}) + self.assertEqual(fs, {b"one": [b"1"]}) def test_from_response_formname_errors_formnumber(self): response = _buildresponse( @@ -893,9 +1030,15 @@ class FormRequestTest(RequestTest): """) - self.assertRaises(IndexError, self.request_class.from_response, - response, formname="form3", formnumber=2) + """ + ) + self.assertRaises( + IndexError, + self.request_class.from_response, + response, + formname="form3", + formnumber=2, + ) def test_from_response_formid_exists(self): response = _buildresponse( @@ -906,13 +1049,14 @@ class FormRequestTest(RequestTest): """) + """ + ) r1 = self.request_class.from_response(response, formid="form2") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) + self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - def test_from_response_formname_notexists_fallback_formid(self): + def test_from_response_formname_nonexistent_fallback_formid(self): response = _buildresponse( """""") - r1 = self.request_class.from_response(response, formname="form3", formid="form2") - self.assertEqual(r1.method, 'POST') + """ + ) + r1 = self.request_class.from_response( + response, formname="form3", formid="form2" + ) + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'four': [b'4'], b'three': [b'3']}) + self.assertEqual(fs, {b"four": [b"4"], b"three": [b"3"]}) - def test_from_response_formid_notexist(self): + def test_from_response_formid_nonexistent(self): response = _buildresponse( """ """) + """ + ) r1 = self.request_class.from_response(response, formid="form3") - self.assertEqual(r1.method, 'POST') + self.assertEqual(r1.method, "POST") fs = _qs(r1) - self.assertEqual(fs, {b'one': [b'1']}) + self.assertEqual(fs, {b"one": [b"1"]}) def test_from_response_formid_errors_formnumber(self): response = _buildresponse( @@ -947,13 +1095,19 @@ class FormRequestTest(RequestTest): """) - self.assertRaises(IndexError, self.request_class.from_response, - response, formid="form3", formnumber=2) + """ + ) + self.assertRaises( + IndexError, + self.request_class.from_response, + response, + formid="form3", + formnumber=2, + ) def test_from_response_select(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req, to_unicode=True) - self.assertEqual(fs, {'i1': ['i1v2'], 'i2': ['i2v1'], 'i4': ['i4v2', 'i4v3']}) + self.assertEqual(fs, {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}) def test_from_response_radio(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']}) + self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) def test_from_response_checkbox(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'iv2'], b'i2': [b'on']}) + self.assertEqual(fs, {b"i1": [b"iv2"], b"i2": [b"on"]}) def test_from_response_input_text(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b''], b'i4': [b'i4v1']}) + self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""], b"i4": [b"i4v1"]}) def test_from_response_input_hidden(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v1'], b'i2': [b'']}) + self.assertEqual(fs, {b"i1": [b"i1v1"], b"i2": [b""]}) def test_from_response_input_textarea(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b''], b'i3': [b'']}) + self.assertEqual(fs, {b"i1": [b"i1v"], b"i2": [b""], b"i3": [b""]}) def test_from_response_descendants(self): res = _buildresponse( - '''''') + """ + ) req = self.request_class.from_response(res) fs = _qs(req) - self.assertEqual(set(fs), {b'h2', b'i2', b'i1', b'i3', b'h1', b'i5', b'i4'}) + self.assertEqual(set(fs), {b"h2", b"i2", b"i1", b"i3", b"h1", b"i5", b"i4"}) def test_from_response_xpath(self): response = _buildresponse( @@ -1081,28 +1242,43 @@ class FormRequestTest(RequestTest): """) - r1 = self.request_class.from_response(response, formxpath="//form[@action='post.php']") + """ + ) + r1 = self.request_class.from_response( + response, formxpath="//form[@action='post.php']" + ) fs = _qs(r1) - self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b"one"], [b"1"]) - r1 = self.request_class.from_response(response, formxpath="//form/input[@name='four']") + r1 = self.request_class.from_response( + response, formxpath="//form/input[@name='four']" + ) fs = _qs(r1) - self.assertEqual(fs[b'three'], [b'3']) + self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises(ValueError, self.request_class.from_response, - response, formxpath="//form/input[@name='abc']") + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + formxpath="//form/input[@name='abc']", + ) def test_from_response_unicode_xpath(self): response = _buildresponse(b'') - r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']") + r = self.request_class.from_response( + response, formxpath="//form[@name='\u044a']" + ) fs = _qs(r) self.assertEqual(fs, {}) xpath = "//form[@name='\u03b1']" - self.assertRaisesRegex(ValueError, re.escape(xpath), - self.request_class.from_response, - response, formxpath=xpath) + self.assertRaisesRegex( + ValueError, + re.escape(xpath), + self.request_class.from_response, + response, + formxpath=xpath, + ) def test_from_response_button_submit(self): response = _buildresponse( @@ -1111,15 +1287,18 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'submit1']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b"submit1"]) def test_from_response_button_notype(self): response = _buildresponse( @@ -1128,15 +1307,18 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'submit1']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b"submit1"]) def test_from_response_submit_novalue(self): response = _buildresponse( @@ -1145,15 +1327,18 @@ class FormRequestTest(RequestTest): Submit """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b""]) def test_from_response_button_novalue(self): response = _buildresponse( @@ -1162,15 +1347,18 @@ class FormRequestTest(RequestTest): """, - url="http://www.example.com/this/list.html") + url="http://www.example.com/this/list.html", + ) req = self.request_class.from_response(response) - self.assertEqual(req.method, 'POST') - self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') + self.assertEqual(req.method, "POST") + self.assertEqual( + req.headers["Content-type"], b"application/x-www-form-urlencoded" + ) self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req) - self.assertEqual(fs[b'test1'], [b'val1']) - self.assertEqual(fs[b'test2'], [b'val2']) - self.assertEqual(fs[b'button1'], [b'']) + self.assertEqual(fs[b"test1"], [b"val1"]) + self.assertEqual(fs[b"test2"], [b"val2"]) + self.assertEqual(fs[b"button1"], [b""]) def test_html_base_form_action(self): response = _buildresponse( @@ -1185,15 +1373,15 @@ class FormRequestTest(RequestTest): """, - url='http://a.com/' + url="http://a.com/", ) req = self.request_class.from_response(response) - self.assertEqual(req.url, 'http://b.com/test_form') + self.assertEqual(req.url, "http://b.com/test_form") def test_spaces_in_action(self): resp = _buildresponse('') req = self.request_class.from_response(resp) - self.assertEqual(req.url, 'http://example.com/path') + self.assertEqual(req.url, "http://example.com/path") def test_from_response_css(self): response = _buildresponse( @@ -1204,44 +1392,53 @@ class FormRequestTest(RequestTest): """) - r1 = self.request_class.from_response(response, formcss="form[action='post.php']") + """ + ) + r1 = self.request_class.from_response( + response, formcss="form[action='post.php']" + ) fs = _qs(r1) - self.assertEqual(fs[b'one'], [b'1']) + self.assertEqual(fs[b"one"], [b"1"]) r1 = self.request_class.from_response(response, formcss="input[name='four']") fs = _qs(r1) - self.assertEqual(fs[b'three'], [b'3']) + self.assertEqual(fs[b"three"], [b"3"]) - self.assertRaises(ValueError, self.request_class.from_response, - response, formcss="input[name='abc']") + self.assertRaises( + ValueError, + self.request_class.from_response, + response, + formcss="input[name='abc']", + ) def test_from_response_valid_form_methods(self): - form_methods = [[method, method] for method in self.request_class.valid_form_methods] - form_methods.append(['UNKNOWN', 'GET']) + form_methods = [ + [method, method] for method in self.request_class.valid_form_methods + ] + form_methods.append(["UNKNOWN", "GET"]) for method, expected in form_methods: response = _buildresponse( f'' + "" ) r = self.request_class.from_response(response) self.assertEqual(r.method, expected) def _buildresponse(body, **kwargs): - kwargs.setdefault('body', body) - kwargs.setdefault('url', 'http://example.com') - kwargs.setdefault('encoding', 'utf-8') + kwargs.setdefault("body", body) + kwargs.setdefault("url", "http://example.com") + kwargs.setdefault("encoding", "utf-8") return HtmlResponse(**kwargs) -def _qs(req, encoding='utf-8', to_unicode=False): - if req.method == 'POST': +def _qs(req, encoding="utf-8", to_unicode=False): + if req.method == "POST": qs = req.body else: - qs = req.url.partition('?')[2] + qs = req.url.partition("?")[2] uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) @@ -1249,40 +1446,42 @@ def _qs(req, encoding='utf-8', to_unicode=False): class XmlRpcRequestTest(RequestTest): - request_class = XmlRpcRequest - default_method = 'POST' - default_headers = {b'Content-Type': [b'text/xml']} + default_method = "POST" + default_headers = {b"Content-Type": [b"text/xml"]} def _test_request(self, **kwargs): - r = self.request_class('http://scrapytest.org/rpc2', **kwargs) - self.assertEqual(r.headers[b'Content-Type'], b'text/xml') - self.assertEqual(r.body, - to_bytes(xmlrpc.client.dumps(**kwargs), - encoding=kwargs.get('encoding', 'utf-8'))) - self.assertEqual(r.method, 'POST') - self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8')) + r = self.request_class("http://scrapytest.org/rpc2", **kwargs) + self.assertEqual(r.headers[b"Content-Type"], b"text/xml") + self.assertEqual( + r.body, + to_bytes( + xmlrpc.client.dumps(**kwargs), encoding=kwargs.get("encoding", "utf-8") + ), + ) + self.assertEqual(r.method, "POST") + self.assertEqual(r.encoding, kwargs.get("encoding", "utf-8")) self.assertTrue(r.dont_filter, True) def test_xmlrpc_dumps(self): - self._test_request(params=('value',)) - self._test_request(params=('username', 'password'), methodname='login') - self._test_request(params=('response', ), methodresponse='login') - self._test_request(params=('pas£',), encoding='utf-8') + self._test_request(params=("value",)) + self._test_request(params=("username", "password"), methodname="login") + self._test_request(params=("response",), methodresponse="login") + self._test_request(params=("pas£",), encoding="utf-8") self._test_request(params=(None,), allow_none=1) self.assertRaises(TypeError, self._test_request) self.assertRaises(TypeError, self._test_request, params=(None,)) def test_latin1(self): - self._test_request(params=('pas£',), encoding='latin1') + self._test_request(params=("pas£",), encoding="latin1") class JsonRequestTest(RequestTest): request_class = JsonRequest - default_method = 'GET' + default_method = "GET" default_headers = { - b'Content-Type': [b'application/json'], - b'Accept': [b'application/json, text/javascript, */*; q=0.01'], + b"Content-Type": [b"application/json"], + b"Accept": [b"application/json, text/javascript, */*; q=0.01"], } def setUp(self): @@ -1291,14 +1490,14 @@ class JsonRequestTest(RequestTest): def test_data(self): r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.body, b'') + self.assertEqual(r1.body, b"") - body = b'body' + body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) self.assertEqual(r2.body, body) data = { - 'name': 'value', + "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) self.assertEqual(r3.body, to_bytes(json.dumps(data))) @@ -1310,134 +1509,138 @@ class JsonRequestTest(RequestTest): def test_data_method(self): # data is not passed r1 = self.request_class(url="http://www.example.com/") - self.assertEqual(r1.method, 'GET') + self.assertEqual(r1.method, "GET") - body = b'body' + body = b"body" r2 = self.request_class(url="http://www.example.com/", body=body) - self.assertEqual(r2.method, 'GET') + self.assertEqual(r2.method, "GET") data = { - 'name': 'value', + "name": "value", } r3 = self.request_class(url="http://www.example.com/", data=data) - self.assertEqual(r3.method, 'POST') + self.assertEqual(r3.method, "POST") # method passed explicitly - r4 = self.request_class(url="http://www.example.com/", data=data, method='GET') - self.assertEqual(r4.method, 'GET') + r4 = self.request_class(url="http://www.example.com/", data=data, method="GET") + self.assertEqual(r4.method, "GET") r5 = self.request_class(url="http://www.example.com/", data=[]) - self.assertEqual(r5.method, 'POST') + self.assertEqual(r5.method, "POST") def test_body_data(self): - """ passing both body and data should result a warning """ - body = b'body' + """passing both body and data should result a warning""" + body = b"body" data = { - 'name': 'value', + "name": "value", } with warnings.catch_warnings(record=True) as _warnings: r5 = self.request_class(url="http://www.example.com/", body=body, data=data) self.assertEqual(r5.body, body) - self.assertEqual(r5.method, 'GET') + self.assertEqual(r5.method, "GET") self.assertEqual(len(_warnings), 1) - self.assertIn('data will be ignored', str(_warnings[0].message)) + self.assertIn("data will be ignored", str(_warnings[0].message)) def test_empty_body_data(self): - """ passing any body value and data should result a warning """ + """passing any body value and data should result a warning""" data = { - 'name': 'value', + "name": "value", } with warnings.catch_warnings(record=True) as _warnings: - r6 = self.request_class(url="http://www.example.com/", body=b'', data=data) - self.assertEqual(r6.body, b'') - self.assertEqual(r6.method, 'GET') + r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) + self.assertEqual(r6.body, b"") + self.assertEqual(r6.method, "GET") self.assertEqual(len(_warnings), 1) - self.assertIn('data will be ignored', str(_warnings[0].message)) + self.assertIn("data will be ignored", str(_warnings[0].message)) def test_body_none_data(self): data = { - 'name': 'value', + "name": "value", } with warnings.catch_warnings(record=True) as _warnings: r7 = self.request_class(url="http://www.example.com/", body=None, data=data) self.assertEqual(r7.body, to_bytes(json.dumps(data))) - self.assertEqual(r7.method, 'POST') + self.assertEqual(r7.method, "POST") self.assertEqual(len(_warnings), 0) def test_body_data_none(self): with warnings.catch_warnings(record=True) as _warnings: r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - self.assertEqual(r8.method, 'GET') + self.assertEqual(r8.method, "GET") self.assertEqual(len(_warnings), 0) def test_dumps_sort_keys(self): - """ Test that sort_keys=True is passed to json.dumps by default """ + """Test that sort_keys=True is passed to json.dumps by default""" data = { - 'name': 'value', + "name": "value", } - with mock.patch('json.dumps', return_value=b'') as mock_dumps: + with mock.patch("json.dumps", return_value=b"") as mock_dumps: self.request_class(url="http://www.example.com/", data=data) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['sort_keys'], True) + self.assertEqual(kwargs["sort_keys"], True) def test_dumps_kwargs(self): - """ Test that dumps_kwargs are passed to json.dumps """ + """Test that dumps_kwargs are passed to json.dumps""" data = { - 'name': 'value', + "name": "value", } dumps_kwargs = { - 'ensure_ascii': True, - 'allow_nan': True, + "ensure_ascii": True, + "allow_nan": True, } - with mock.patch('json.dumps', return_value=b'') as mock_dumps: - self.request_class(url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: + self.request_class( + url="http://www.example.com/", data=data, dumps_kwargs=dumps_kwargs + ) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['ensure_ascii'], True) - self.assertEqual(kwargs['allow_nan'], True) + self.assertEqual(kwargs["ensure_ascii"], True) + self.assertEqual(kwargs["allow_nan"], True) def test_replace_data(self): data1 = { - 'name1': 'value1', + "name1": "value1", } data2 = { - 'name2': 'value2', + "name2": "value2", } r1 = self.request_class(url="http://www.example.com/", data=data1) r2 = r1.replace(data=data2) self.assertEqual(r2.body, to_bytes(json.dumps(data2))) def test_replace_sort_keys(self): - """ Test that replace provides sort_keys=True to json.dumps """ + """Test that replace provides sort_keys=True to json.dumps""" data1 = { - 'name1': 'value1', + "name1": "value1", } data2 = { - 'name2': 'value2', + "name2": "value2", } r1 = self.request_class(url="http://www.example.com/", data=data1) - with mock.patch('json.dumps', return_value=b'') as mock_dumps: + with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['sort_keys'], True) + self.assertEqual(kwargs["sort_keys"], True) def test_replace_dumps_kwargs(self): - """ Test that dumps_kwargs are provided to json.dumps when replace is called """ + """Test that dumps_kwargs are provided to json.dumps when replace is called""" data1 = { - 'name1': 'value1', + "name1": "value1", } data2 = { - 'name2': 'value2', + "name2": "value2", } dumps_kwargs = { - 'ensure_ascii': True, - 'allow_nan': True, + "ensure_ascii": True, + "allow_nan": True, } - r1 = self.request_class(url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs) - with mock.patch('json.dumps', return_value=b'') as mock_dumps: + r1 = self.request_class( + url="http://www.example.com/", data=data1, dumps_kwargs=dumps_kwargs + ) + with mock.patch("json.dumps", return_value=b"") as mock_dumps: r1.replace(data=data2) kwargs = mock_dumps.call_args[1] - self.assertEqual(kwargs['ensure_ascii'], True) - self.assertEqual(kwargs['allow_nan'], True) + self.assertEqual(kwargs["ensure_ascii"], True) + self.assertEqual(kwargs["allow_nan"], True) def tearDown(self): warnings.resetwarnings() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 80c0279cc..bfa08136f 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -8,8 +8,14 @@ from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import (Headers, HtmlResponse, Request, Response, - TextResponse, XmlResponse) +from scrapy.http import ( + Headers, + HtmlResponse, + Request, + Response, + TextResponse, + XmlResponse, +) from scrapy.link import Link from scrapy.selector import Selector from scrapy.utils.python import to_unicode @@ -17,21 +23,38 @@ from tests import get_testdata class BaseResponseTest(unittest.TestCase): - response_class = Response def test_init(self): # Response requires url in the constructor self.assertRaises(Exception, self.response_class) - self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) + self.assertTrue( + isinstance(self.response_class("http://example.com/"), self.response_class) + ) self.assertRaises(TypeError, self.response_class, b"http://example.com") + self.assertRaises( + TypeError, self.response_class, url="http://example.com", body={} + ) # body can be str or None - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) - self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) + self.assertTrue( + isinstance( + self.response_class("http://example.com/", body=b""), + self.response_class, + ) + ) + self.assertTrue( + isinstance( + self.response_class("http://example.com/", body=b"body"), + self.response_class, + ) + ) # test presence of all optional parameters self.assertTrue( isinstance( - self.response_class('http://example.com/', body=b'', headers={}, status=200), self.response_class + self.response_class( + "http://example.com/", body=b"", headers={}, status=200 + ), + self.response_class, ) ) @@ -52,15 +75,20 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://www.example.com", status=301) self.assertEqual(r.status, 301) - r = self.response_class("http://www.example.com", status='301') + r = self.response_class("http://www.example.com", status="301") self.assertEqual(r.status, 301) - self.assertRaises(ValueError, self.response_class, "http://example.com", status='lala200') + self.assertRaises( + ValueError, + self.response_class, + "http://example.com", + status="lala200", + ) def test_copy(self): """Test Response copy""" r1 = self.response_class("http://www.example.com", body=b"Some body") - r1.flags.append('cached') + r1.flags.append("cached") r2 = r1.copy() self.assertEqual(r1.status, r2.status) @@ -71,29 +99,37 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r1.flags, r2.flags) # make sure headers attribute is shallow copied - assert r1.headers is not r2.headers, "headers must be a shallow copy, not identical" + assert ( + r1.headers is not r2.headers + ), "headers must be a shallow copy, not identical" self.assertEqual(r1.headers, r2.headers) def test_copy_meta(self): req = Request("http://www.example.com") - req.meta['foo'] = 'bar' - r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + req.meta["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) assert r1.meta is req.meta def test_copy_cb_kwargs(self): req = Request("http://www.example.com") - req.cb_kwargs['foo'] = 'bar' - r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + req.cb_kwargs["foo"] = "bar" + r1 = self.response_class( + "http://www.example.com", body=b"Some body", request=req + ) assert r1.cb_kwargs is req.cb_kwargs def test_unavailable_meta(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'): + with self.assertRaisesRegex(AttributeError, r"Response\.meta not available"): r1.meta def test_unavailable_cb_kwargs(self): r1 = self.response_class("http://www.example.com", body=b"Some body") - with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'): + with self.assertRaisesRegex( + AttributeError, r"Response\.cb_kwargs not available" + ): r1.cb_kwargs def test_copy_inherited_classes(self): @@ -102,7 +138,7 @@ class BaseResponseTest(unittest.TestCase): class CustomResponse(self.response_class): pass - r1 = CustomResponse('http://www.example.com') + r1 = CustomResponse("http://www.example.com") r2 = r1.copy() assert isinstance(r2, CustomResponse) @@ -112,16 +148,16 @@ class BaseResponseTest(unittest.TestCase): hdrs = Headers({"key": "value"}) r1 = self.response_class("http://www.example.com") r2 = r1.replace(status=301, body=b"New body", headers=hdrs) - assert r1.body == b'' + assert r1.body == b"" self.assertEqual(r1.url, r2.url) self.assertEqual((r1.status, r2.status), (200, 301)) - self.assertEqual((r1.body, r2.body), (b'', b"New body")) + self.assertEqual((r1.body, r2.body), (b"", b"New body")) self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) # Empty attributes (which may fail if not compared properly) - r3 = self.response_class("http://www.example.com", flags=['cached']) - r4 = r3.replace(body=b'', flags=[]) - self.assertEqual(r4.body, b'') + r3 = self.response_class("http://www.example.com", flags=["cached"]) + r4 = r3.replace(body=b"", flags=[]) + self.assertEqual(r4.body, b"") self.assertEqual(r4.flags, []) def _assert_response_values(self, response, encoding, body): @@ -143,40 +179,40 @@ class BaseResponseTest(unittest.TestCase): def test_immutable_attributes(self): r = self.response_class("http://example.com") - self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') - self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + self.assertRaises(AttributeError, setattr, r, "url", "http://example2.com") + self.assertRaises(AttributeError, setattr, r, "body", "xxx") def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" - joined = self.response_class('http://www.example.com').urljoin('/test') - absolute = 'http://www.example.com/test' + joined = self.response_class("http://www.example.com").urljoin("/test") + absolute = "http://www.example.com/test" self.assertEqual(joined, absolute) def test_shortcut_attributes(self): - r = self.response_class("http://example.com", body=b'hello') + r = self.response_class("http://example.com", body=b"hello") if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text') - self.assertRaisesRegex(NotSupported, msg, r.css, 'body') - self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body') + self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") + self.assertRaisesRegex(NotSupported, msg, r.css, "body") + self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") + self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") else: r.text - r.css('body') - r.xpath('//body') + r.css("body") + r.xpath("//body") # Response.follow def test_follow_url_absolute(self): - self._assert_followed_url('http://foo.example.com', - 'http://foo.example.com') + self._assert_followed_url("http://foo.example.com", "http://foo.example.com") def test_follow_url_relative(self): - self._assert_followed_url('foo', - 'http://example.com/foo') + self._assert_followed_url("foo", "http://example.com/foo") def test_follow_link(self): - self._assert_followed_url(Link('http://example.com/foo'), - 'http://example.com/foo') + self._assert_followed_url( + Link("http://example.com/foo"), "http://example.com/foo" + ) def test_follow_None_url(self): r = self.response_class("http://example.com") @@ -188,8 +224,7 @@ class BaseResponseTest(unittest.TestCase): strict=True, ) def test_follow_whitespace_url(self): - self._assert_followed_url('foo ', - 'http://example.com/foo') + self._assert_followed_url("foo ", "http://example.com/foo") @mark.xfail( parse_version(w3lib_version) < parse_version("2.1.1"), @@ -197,37 +232,42 @@ class BaseResponseTest(unittest.TestCase): strict=True, ) def test_follow_whitespace_link(self): - self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo') + self._assert_followed_url( + Link("http://example.com/foo "), "http://example.com/foo" + ) def test_follow_flags(self): - res = self.response_class('http://example.com/') - fol = res.follow('http://example.com/', flags=['cached', 'allowed']) - self.assertEqual(fol.flags, ['cached', 'allowed']) + res = self.response_class("http://example.com/") + fol = res.follow("http://example.com/", flags=["cached", "allowed"]) + self.assertEqual(fol.flags, ["cached", "allowed"]) # Response.follow_all def test_follow_all_absolute(self): - url_list = ['http://example.org', 'http://www.example.org', - 'http://example.com', 'http://www.example.com'] + url_list = [ + "http://example.org", + "http://www.example.org", + "http://example.com", + "http://www.example.com", + ] self._assert_followed_all_urls(url_list, url_list) def test_follow_all_relative(self): - relative = ['foo', 'bar', 'foo/bar', 'bar/foo'] + relative = ["foo", "bar", "foo/bar", "bar/foo"] absolute = [ - 'http://example.com/foo', - 'http://example.com/bar', - 'http://example.com/foo/bar', - 'http://example.com/bar/foo', + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", ] self._assert_followed_all_urls(relative, absolute) def test_follow_all_links(self): absolute = [ - 'http://example.com/foo', - 'http://example.com/bar', - 'http://example.com/foo/bar', - 'http://example.com/bar/foo', + "http://example.com/foo", + "http://example.com/bar", + "http://example.com/foo/bar", + "http://example.com/bar/foo", ] links = map(Link, absolute) self._assert_followed_all_urls(links, absolute) @@ -254,36 +294,36 @@ class BaseResponseTest(unittest.TestCase): list(r.follow_all(urls=[None])) def test_follow_all_whitespace(self): - relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo '] + relative = ["foo ", "bar ", "foo/bar ", "bar/foo "] absolute = [ - 'http://example.com/foo%20', - 'http://example.com/bar%20', - 'http://example.com/foo/bar%20', - 'http://example.com/bar/foo%20', + "http://example.com/foo%20", + "http://example.com/bar%20", + "http://example.com/foo/bar%20", + "http://example.com/bar/foo%20", ] self._assert_followed_all_urls(relative, absolute) def test_follow_all_whitespace_links(self): absolute = [ - 'http://example.com/foo ', - 'http://example.com/bar ', - 'http://example.com/foo/bar ', - 'http://example.com/bar/foo ', + "http://example.com/foo ", + "http://example.com/bar ", + "http://example.com/foo/bar ", + "http://example.com/bar/foo ", ] links = map(Link, absolute) - expected = [u.replace(' ', '%20') for u in absolute] + expected = [u.replace(" ", "%20") for u in absolute] self._assert_followed_all_urls(links, expected) def test_follow_all_flags(self): - re = self.response_class('http://www.example.com/') + re = self.response_class("http://www.example.com/") urls = [ - 'http://www.example.com/', - 'http://www.example.com/2', - 'http://www.example.com/foo', + "http://www.example.com/", + "http://www.example.com/2", + "http://www.example.com/foo", ] - fol = re.follow_all(urls, flags=['cached', 'allowed']) + fol = re.follow_all(urls, flags=["cached", "allowed"]) for req in fol: - self.assertEqual(req.flags, ['cached', 'allowed']) + self.assertEqual(req.flags, ["cached", "allowed"]) def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: @@ -301,23 +341,24 @@ class BaseResponseTest(unittest.TestCase): yield req def _links_response(self): - body = get_testdata('link_extractor', 'linkextractor.html') - resp = self.response_class('http://example.com/index', body=body) + body = get_testdata("link_extractor", "linkextractor.html") + resp = self.response_class("http://example.com/index", body=body) return resp def _links_response_no_href(self): - body = get_testdata('link_extractor', 'linkextractor_no_href.html') - resp = self.response_class('http://example.com/index', body=body) + body = get_testdata("link_extractor", "linkextractor_no_href.html") + resp = self.response_class("http://example.com/index", body=body) return resp class TextResponseTest(BaseResponseTest): - response_class = TextResponse def test_replace(self): super().test_replace() - r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") + r1 = self.response_class( + "http://www.example.com", body="hello", encoding="cp852" + ) r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") @@ -333,135 +374,208 @@ class TextResponseTest(BaseResponseTest): self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str - resp = self.response_class(url="http://www.example.com/", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/", encoding="utf-8") assert isinstance(resp.url, str) - resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8') - self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1') - self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class("http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}) - self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class("http://www.example.com/price/\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}) - self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="utf-8" + ) + self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + resp = self.response_class( + url="http://www.example.com/price/\xa3", encoding="latin-1" + ) + self.assertEqual(resp.url, "http://www.example.com/price/\xa3") + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + self.assertEqual(resp.url, to_unicode(b"http://www.example.com/price/\xc2\xa3")) + resp = self.response_class( + "http://www.example.com/price/\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) + self.assertEqual(resp.url, "http://www.example.com/price/\xa3") def test_unicode_body(self): - unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' - '\u0442\u0435\u043a\u0441\u0442') - self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body') + unicode_string = ( + "\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 " + "\u0442\u0435\u043a\u0441\u0442" + ) + self.assertRaises( + TypeError, + self.response_class, + "http://www.example.com", + body="unicode body", + ) - original_string = unicode_string.encode('cp1251') - r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') + original_string = unicode_string.encode("cp1251") + r1 = self.response_class( + "http://www.example.com", body=original_string, encoding="cp1251" + ) # check response.text self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) def test_encoding(self): - r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=utf-8"]}) - r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3") - r3 = self.response_class("http://www.example.com", body=b"\xa3", - headers={"Content-type": ["text/html; charset=iso-8859-1"]}) + r1 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=utf-8"]}, + ) + r2 = self.response_class( + "http://www.example.com", encoding="utf-8", body="\xa3" + ) + r3 = self.response_class( + "http://www.example.com", + body=b"\xa3", + headers={"Content-type": ["text/html; charset=iso-8859-1"]}, + ) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") - r5 = self.response_class("http://www.example.com", body=b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=None"]}) - r6 = self.response_class("http://www.example.com", body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gb2312"]}) - r7 = self.response_class("http://www.example.com", body=b"\xa8D", - headers={"Content-type": ["text/html; charset=gbk"]}) - r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3", - headers={"Content-type": ["text/html; charset=cp1251"]}) + r5 = self.response_class( + "http://www.example.com", + body=b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=None"]}, + ) + r6 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gb2312"]}, + ) + r7 = self.response_class( + "http://www.example.com", + body=b"\xa8D", + headers={"Content-type": ["text/html; charset=gbk"]}, + ) + r8 = self.response_class( + "http://www.example.com", + body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}, + ) + r9 = self.response_class( + "http://www.example.com", + body=b"\x80", + headers={ + "Content-type": [b"application/x-download; filename=\x80dummy.txt"] + }, + ) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) - self.assertEqual(r2._declared_encoding(), 'utf-8') - self._assert_response_encoding(r2, 'utf-8') + self.assertEqual(r2._declared_encoding(), "utf-8") + self._assert_response_encoding(r2, "utf-8") self.assertEqual(r3._headers_encoding(), "cp1252") self.assertEqual(r3._declared_encoding(), "cp1252") self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r9._headers_encoding(), None) self.assertEqual(r8._declared_encoding(), "utf-8") + self.assertEqual(r9._declared_encoding(), None) self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") - assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' - self._assert_response_values(r1, 'utf-8', "\xa3") - self._assert_response_values(r2, 'utf-8', "\xa3") - self._assert_response_values(r3, 'iso-8859-1', "\xa3") - self._assert_response_values(r6, 'gb18030', "\u2015") - self._assert_response_values(r7, 'gb18030', "\u2015") + self._assert_response_encoding(r9, "cp1252") + assert ( + r4._body_inferred_encoding() is not None + and r4._body_inferred_encoding() != "ascii" + ) + self._assert_response_values(r1, "utf-8", "\xa3") + self._assert_response_values(r2, "utf-8", "\xa3") + self._assert_response_values(r3, "iso-8859-1", "\xa3") + self._assert_response_values(r6, "gb18030", "\u2015") + self._assert_response_values(r7, "gb18030", "\u2015") + self._assert_response_values(r9, "cp1252", "€") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3") + self.assertRaises( + TypeError, + self.response_class, + "http://www.example.com", + body="\xa3", + ) def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" - r = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=UNKNOWN"]}, - body=b"\xc2\xa3") + r = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=UNKNOWN"]}, + body=b"\xc2\xa3", + ) self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, 'utf-8', "\xa3") + self._assert_response_values(r, "utf-8", "\xa3") def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" - r = self.response_class("http://www.example.com", - body=b'\xff\xfeh\x00i\x00', - encoding='utf-16') - self._assert_response_values(r, 'utf-16', "hi") + r = self.response_class( + "http://www.example.com", + body=b"\xff\xfeh\x00i\x00", + encoding="utf-16", + ) + self._assert_response_values(r, "utf-16", "hi") def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): - r6 = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=utf-8"]}, - body=b"\xef\xbb\xbfWORD\xe3\xab") - self.assertEqual(r6.encoding, 'utf-8') - self.assertIn(r6.text, { - 'WORD\ufffd\ufffd', # w3lib < 1.19.0 - 'WORD\ufffd', # w3lib >= 1.19.0 - }) + r6 = self.response_class( + "http://www.example.com", + headers={"Content-type": ["text/html; charset=utf-8"]}, + body=b"\xef\xbb\xbfWORD\xe3\xab", + ) + self.assertEqual(r6.encoding, "utf-8") + self.assertIn( + r6.text, + { + "WORD\ufffd\ufffd", # w3lib < 1.19.0 + "WORD\ufffd", # w3lib >= 1.19.0 + }, + ) def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and - # response.text in indistint order doesn't affect final + # response.text in indistinct order doesn't affect final + # response.text in indistinct order doesn't affect final # values for encoding and decoded body. - url = 'http://example.com' + url = "http://example.com" body = b"\xef\xbb\xbfWORD" headers = {"Content-type": ["text/html; charset=utf-8"]} # Test response without content-type and BOM encoding response = self.response_class(url, body=body) - self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, 'WORD') + self.assertEqual(response.encoding, "utf-8") + self.assertEqual(response.text, "WORD") response = self.response_class(url, body=body) - self.assertEqual(response.text, 'WORD') - self.assertEqual(response.encoding, 'utf-8') + self.assertEqual(response.text, "WORD") + self.assertEqual(response.encoding, "utf-8") # Body caching sideeffect isn't triggered when encoding is declared in # content-type header but BOM still need to be removed from decoded # body response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, 'WORD') + self.assertEqual(response.encoding, "utf-8") + self.assertEqual(response.text, "WORD") response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, 'WORD') - self.assertEqual(response.encoding, 'utf-8') + self.assertEqual(response.text, "WORD") + self.assertEqual(response.encoding, "utf-8") def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" - r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') + r = self.response_class( + "http://www.example.com", + encoding="utf-8", + body=b"PREFIX\xe3\xabSUFFIX", + ) # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char ('\ufffd') - assert '\ufffd' in r.text, repr(r.text) - assert 'PREFIX' in r.text, repr(r.text) - assert 'SUFFIX' in r.text, repr(r.text) + assert "\ufffd" in r.text, repr(r.text) + assert "PREFIX" in r.text, repr(r.text) + assert "SUFFIX" in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs - r = self.response_class("http://example.com", encoding='utf-8', - body=b'\xf0value') - assert 'value' in r.text, repr(r.text) + r = self.response_class( + "http://example.com", + encoding="utf-8", + body=b"\xf0value", + ) + assert "value" in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') @@ -472,22 +586,15 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertIsInstance(response.selector, Selector) - self.assertEqual(response.selector.type, 'html') + self.assertEqual(response.selector.type, "html") self.assertIs(response.selector, response.selector) # property is cached self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//title/text()").getall(), - ['Some page'] - ) - self.assertEqual( - response.selector.css("title::text").getall(), - ['Some page'] - ) - self.assertEqual( - response.selector.re("Some (.*)"), - ['page'] + response.selector.xpath("//title/text()").getall(), ["Some page"] ) + self.assertEqual(response.selector.css("title::text").getall(), ["Some page"]) + self.assertEqual(response.selector.re("Some (.*)"), ["page"]) def test_selector_shortcuts(self): body = b"A nice paragraph.
" + body = b'A nice paragraph.
' response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(), - response.xpath("normalize-space(//p[@class=\"content\"])").getall(), + response.xpath( + "normalize-space(//p[@class=$pclass])", pclass="content" + ).getall(), + response.xpath('normalize-space(//p[@class="content"])').getall(), ) self.assertEqual( response.xpath( "//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1, + pclass="content", + pcount=1, + ).getall(), + response.xpath( + '//title[count(following::p[@class="content"])=1]/text()' ).getall(), - response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) def test_urljoin_with_base_url(self): """Test urljoin shortcut which also evaluates base-url.""" body = b'>\xbe\xa9<\xb6\xab"""
- response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
+ response = HtmlResponse("http://example.org", body=body, encoding="gb18030")
lx = self.extractor_cls(restrict_xpaths="//div")
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
- fragment='', nofollow=False)])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="http://example.org/foo",
+ text=">\u4eac<\u4e1c",
+ fragment="",
+ nofollow=False,
+ )
+ ],
+ )
def test_restrict_css(self):
- lx = self.extractor_cls(restrict_css=('#subwrapper a',))
- self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample2.html', text='sample 2')
- ])
+ lx = self.extractor_cls(restrict_css=("#subwrapper a",))
+ self.assertEqual(
+ lx.extract_links(self.response),
+ [Link(url="http://example.com/sample2.html", text="sample 2")],
+ )
def test_restrict_css_and_restrict_xpaths_together(self):
- lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
- restrict_css=('#subwrapper + a', ))
- self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=''),
- Link(url='http://example.com/sample2.html', text='sample 2'),
- Link(url='http://example.com/sample3.html', text='sample 3 text'),
- ])
+ lx = self.extractor_cls(
+ restrict_xpaths=('//div[@id="subwrapper"]',),
+ restrict_css=("#subwrapper + a",),
+ )
+ self.assertEqual(
+ [link for link in lx.extract_links(self.response)],
+ [
+ Link(url="http://example.com/sample1.html", text=""),
+ Link(url="http://example.com/sample2.html", text="sample 2"),
+ Link(url="http://example.com/sample3.html", text="sample 3 text"),
+ ],
+ )
def test_area_tag_with_unicode_present(self):
body = b"""\xbe\xa9"""
- response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
+ response = HtmlResponse("http://example.org", body=body, encoding="utf-8")
lx = self.extractor_cls()
lx.extract_links(response)
lx.extract_links(response)
lx.extract_links(response)
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/foo', text='',
- fragment='', nofollow=False)])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="http://example.org/foo",
+ text="",
+ fragment="",
+ nofollow=False,
+ )
+ ],
+ )
def test_encoded_url(self):
body = b"""
BinB"""
- response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+ response = HtmlResponse(
+ "http://known.fm/AC%2FDC/", body=body, encoding="utf8"
+ )
lx = self.extractor_cls()
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="http://known.fm/AC%2FDC/?page=2",
+ text="BinB",
+ fragment="",
+ nofollow=False,
+ ),
+ ],
+ )
def test_encoded_url_in_restricted_xpath(self):
body = b"""
BinB"""
- response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
+ response = HtmlResponse(
+ "http://known.fm/AC%2FDC/", body=body, encoding="utf8"
+ )
lx = self.extractor_cls(restrict_xpaths="//div")
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="http://known.fm/AC%2FDC/?page=2",
+ text="BinB",
+ fragment="",
+ nofollow=False,
+ ),
+ ],
+ )
def test_ignored_extensions(self):
# jpg is ignored by default
html = b"""asd and """
response = HtmlResponse("http://example.org/", body=html)
lx = self.extractor_cls()
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.org/page.html', text='asd'),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(url="http://example.org/page.html", text="asd"),
+ ],
+ )
# override denied extensions
- lx = self.extractor_cls(deny_extensions=['html'])
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.org/photo.jpg'),
- ])
+ lx = self.extractor_cls(deny_extensions=["html"])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(url="http://example.org/photo.jpg"),
+ ],
+ )
def test_process_value(self):
"""Test restrict_xpaths with encodings"""
@@ -280,7 +427,11 @@ class Base:
Text
About us
"""
- response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html",
+ body=html,
+ encoding="windows-1252",
+ )
def process_value(value):
m = re.search(r"javascript:goToPage\('(.*?)'", value)
@@ -288,8 +439,10 @@ class Base:
return m.group(1)
lx = self.extractor_cls(process_value=process_value)
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/other/page.html', text='Text')])
+ self.assertEqual(
+ lx.extract_links(response),
+ [Link(url="http://example.org/other/page.html", text="Text")],
+ )
def test_base_url_with_restrict_xpaths(self):
html = b"""Page title<title><base href="http://otherdomain.com/base/" />
@@ -297,34 +450,50 @@ class Base:
</body></html>"""
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
lx = self.extractor_cls(restrict_xpaths="//p")
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
+ self.assertEqual(
+ lx.extract_links(response),
+ [Link(url="http://otherdomain.com/base/item/12.html", text="Item 12")],
+ )
def test_attrs(self):
lx = self.extractor_cls(attrs="href")
- page4_url = 'http://example.com/page%204.html'
+ page4_url = "http://example.com/page%204.html"
- self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample1.html', text=''),
- Link(url='http://example.com/sample2.html', text='sample 2'),
- Link(url='http://example.com/sample3.html', text='sample 3 text'),
- Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=''),
- Link(url='http://example.com/innertag.html', text='inner tag'),
- Link(url=page4_url, text='href with whitespaces'),
- ])
+ self.assertEqual(
+ lx.extract_links(self.response),
+ [
+ Link(url="http://example.com/sample1.html", text=""),
+ Link(url="http://example.com/sample2.html", text="sample 2"),
+ Link(url="http://example.com/sample3.html", text="sample 3 text"),
+ Link(
+ url="http://example.com/sample3.html#foo",
+ text="sample 3 repetition with fragment",
+ ),
+ Link(url="http://www.google.com/something", text=""),
+ Link(url="http://example.com/innertag.html", text="inner tag"),
+ Link(url=page4_url, text="href with whitespaces"),
+ ],
+ )
- lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
- self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample1.html', text=''),
- Link(url='http://example.com/sample2.html', text='sample 2'),
- Link(url='http://example.com/sample2.jpg', text=''),
- Link(url='http://example.com/sample3.html', text='sample 3 text'),
- Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=''),
- Link(url='http://example.com/innertag.html', text='inner tag'),
- Link(url=page4_url, text='href with whitespaces'),
- ])
+ lx = self.extractor_cls(
+ attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()
+ )
+ self.assertEqual(
+ lx.extract_links(self.response),
+ [
+ Link(url="http://example.com/sample1.html", text=""),
+ Link(url="http://example.com/sample2.html", text="sample 2"),
+ Link(url="http://example.com/sample2.jpg", text=""),
+ Link(url="http://example.com/sample3.html", text="sample 3 text"),
+ Link(
+ url="http://example.com/sample3.html#foo",
+ text="sample 3 repetition with fragment",
+ ),
+ Link(url="http://www.google.com/something", text=""),
+ Link(url="http://example.com/innertag.html", text="inner tag"),
+ Link(url=page4_url, text="href with whitespaces"),
+ ],
+ )
lx = self.extractor_cls(attrs=None)
self.assertEqual(lx.extract_links(self.response), [])
@@ -340,26 +509,40 @@ class Base:
self.assertEqual(lx.extract_links(response), [])
lx = self.extractor_cls()
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample1.html', text=''),
- Link(url='http://example.com/sample2.html', text='sample 2'),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(url="http://example.com/sample1.html", text=""),
+ Link(url="http://example.com/sample2.html", text="sample 2"),
+ ],
+ )
lx = self.extractor_cls(tags="area")
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample1.html', text=''),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(url="http://example.com/sample1.html", text=""),
+ ],
+ )
lx = self.extractor_cls(tags="a")
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample2.html', text='sample 2'),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(url="http://example.com/sample2.html", text="sample 2"),
+ ],
+ )
- lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample2.html', text='sample 2'),
- Link(url='http://example.com/sample2.jpg', text=''),
- ])
+ lx = self.extractor_cls(
+ tags=("a", "img"), attrs=("href", "src"), deny_extensions=()
+ )
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(url="http://example.com/sample2.html", text="sample 2"),
+ Link(url="http://example.com/sample2.jpg", text=""),
+ ],
+ )
def test_tags_attrs(self):
html = b"""
@@ -370,17 +553,43 @@ class Base:
"""
response = HtmlResponse("http://example.com/index.html", body=html)
- lx = self.extractor_cls(tags='div', attrs='data-url')
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
- Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
- ])
+ lx = self.extractor_cls(tags="div", attrs="data-url")
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="http://example.com/get?id=1",
+ text="Item 1",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.com/get?id=2",
+ text="Item 2",
+ fragment="",
+ nofollow=False,
+ ),
+ ],
+ )
- lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
- Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
- ])
+ lx = self.extractor_cls(tags=("div",), attrs=("data-url",))
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="http://example.com/get?id=1",
+ text="Item 1",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.com/get?id=2",
+ text="Item 2",
+ fragment="",
+ nofollow=False,
+ ),
+ ],
+ )
def test_xhtml(self):
xhtml = b"""
@@ -417,14 +626,36 @@ class Base:
self.assertEqual(
lx.extract_links(response),
[
- Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
- Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
- Link(url='http://example.com/nofollow.html', text='Dont follow this one',
- fragment='', nofollow=True),
- Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
- fragment='', nofollow=False),
- Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
- ]
+ Link(
+ url="http://example.com/about.html",
+ text="About us",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.com/follow.html",
+ text="Follow this link",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.com/nofollow.html",
+ text="Dont follow this one",
+ fragment="",
+ nofollow=True,
+ ),
+ Link(
+ url="http://example.com/nofollow2.html",
+ text="Choose to follow or not",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://google.com/something",
+ text="External link not to follow",
+ nofollow=True,
+ ),
+ ],
)
response = XmlResponse("http://example.com/index.xhtml", body=xhtml)
@@ -433,14 +664,36 @@ class Base:
self.assertEqual(
lx.extract_links(response),
[
- Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
- Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
- Link(url='http://example.com/nofollow.html', text='Dont follow this one',
- fragment='', nofollow=True),
- Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
- fragment='', nofollow=False),
- Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
- ]
+ Link(
+ url="http://example.com/about.html",
+ text="About us",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.com/follow.html",
+ text="Follow this link",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.com/nofollow.html",
+ text="Dont follow this one",
+ fragment="",
+ nofollow=True,
+ ),
+ Link(
+ url="http://example.com/nofollow2.html",
+ text="Choose to follow or not",
+ fragment="",
+ nofollow=False,
+ ),
+ Link(
+ url="http://google.com/something",
+ text="External link not to follow",
+ nofollow=True,
+ ),
+ ],
)
def test_link_wrong_href(self):
@@ -451,21 +704,42 @@ class Base:
"""
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
- Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
- ])
+ self.assertEqual(
+ [link for link in lx.extract_links(response)],
+ [
+ Link(
+ url="http://example.org/item1.html",
+ text="Item 1",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.org/item3.html",
+ text="Item 3",
+ nofollow=False,
+ ),
+ ],
+ )
def test_ftp_links(self):
body = b"""
<html><body>
<div><a href="ftp://www.external.com/">An Item</a></div>
</body></html>"""
- response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
+ response = HtmlResponse(
+ "http://www.example.com/index.html", body=body, encoding="utf8"
+ )
lx = self.extractor_cls()
- self.assertEqual(lx.extract_links(response), [
- Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
- ])
+ self.assertEqual(
+ lx.extract_links(response),
+ [
+ Link(
+ url="ftp://www.external.com/",
+ text="An Item",
+ fragment="",
+ nofollow=False,
+ ),
+ ],
+ )
def test_pickle_extractor(self):
lx = self.extractor_cls()
@@ -483,10 +757,17 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
"""
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
- Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
- ])
+ self.assertEqual(
+ [link for link in lx.extract_links(response)],
+ [
+ Link(
+ url="http://example.org/item1.html", text="Item 1", nofollow=False
+ ),
+ Link(
+ url="http://example.org/item3.html", text="Item 3", nofollow=False
+ ),
+ ],
+ )
def test_link_restrict_text(self):
html = b"""
@@ -496,21 +777,77 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
"""
response = HtmlResponse("http://example.org/index.html", body=html)
# Simple text inclusion test
- lx = self.extractor_cls(restrict_text='dog')
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
- ])
+ lx = self.extractor_cls(restrict_text="dog")
+ self.assertEqual(
+ [link for link in lx.extract_links(response)],
+ [
+ Link(
+ url="http://example.org/item2.html",
+ text="Pic of a dog",
+ nofollow=False,
+ ),
+ ],
+ )
# Unique regex test
- lx = self.extractor_cls(restrict_text=r'of.*dog')
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
- ])
+ lx = self.extractor_cls(restrict_text=r"of.*dog")
+ self.assertEqual(
+ [link for link in lx.extract_links(response)],
+ [
+ Link(
+ url="http://example.org/item2.html",
+ text="Pic of a dog",
+ nofollow=False,
+ ),
+ ],
+ )
# Multiple regex test
- lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
- Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
- ])
+ lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"])
+ self.assertEqual(
+ [link for link in lx.extract_links(response)],
+ [
+ Link(
+ url="http://example.org/item1.html",
+ text="Pic of a cat",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.org/item2.html",
+ text="Pic of a dog",
+ nofollow=False,
+ ),
+ ],
+ )
def test_restrict_xpaths_with_html_entities(self):
super().test_restrict_xpaths_with_html_entities()
+
+ @mark.skipif(
+ Version(w3lib_version) < Version("2.0.0"),
+ reason=(
+ "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "
+ "about an invalid port value."
+ ),
+ )
+ def test_skip_bad_links(self):
+ html = b"""
+ <a href="http://example.org:non-port">Why would you do this?</a>
+ <a href="http://example.org/item2.html">Good Link</a>
+ <a href="http://example.org/item3.html">Good Link 2</a>
+ """
+ response = HtmlResponse("http://example.org/index.html", body=html)
+ lx = self.extractor_cls()
+ self.assertEqual(
+ [link for link in lx.extract_links(response)],
+ [
+ Link(
+ url="http://example.org/item2.html",
+ text="Good Link",
+ nofollow=False,
+ ),
+ Link(
+ url="http://example.org/item3.html",
+ text="Good Link 2",
+ nofollow=False,
+ ),
+ ],
+ )
diff --git a/tests/test_loader.py b/tests/test_loader.py
index b3e44d36b..b0b7f8723 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,12 +1,13 @@
-import unittest
import dataclasses
+import unittest
+from typing import Optional
import attr
from itemadapter import ItemAdapter
from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst
from scrapy.http import HtmlResponse, Response
-from scrapy.item import Item, Field
+from scrapy.item import Field, Item
from scrapy.loader import ItemLoader
from scrapy.selector import Selector
@@ -59,119 +60,126 @@ class DefaultedItemLoader(NameItemLoader):
# test processors
def processor_with_args(value, other=None, loader_context=None):
- if 'key' in loader_context:
- return loader_context['key']
+ if "key" in loader_context:
+ return loader_context["key"]
return value
class BasicItemLoaderTest(unittest.TestCase):
-
def test_add_value_on_unknown_field(self):
il = TestItemLoader()
- self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
+ self.assertRaises(KeyError, il.add_value, "wrong_field", ["lala", "lolo"])
def test_load_item_using_default_loader(self):
i = TestItem()
- i['summary'] = 'lala'
+ i["summary"] = "lala"
il = ItemLoader(item=i)
- il.add_value('name', 'marta')
+ il.add_value("name", "marta")
item = il.load_item()
assert item is i
- self.assertEqual(item['summary'], ['lala'])
- self.assertEqual(item['name'], ['marta'])
+ self.assertEqual(item["summary"], ["lala"])
+ self.assertEqual(item["name"], ["marta"])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
- il.add_value('name', 'marta')
+ il.add_value("name", "marta")
item = il.load_item()
- self.assertEqual(item['name'], ['Marta'])
+ self.assertEqual(item["name"], ["Marta"])
class InitializationTestMixin:
-
- item_class = None
+ item_class: Optional[type] = None
def test_keep_single_value(self):
"""Loaded item should contain values from the initial item"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo']})
+ self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]})
def test_keep_list(self):
"""Loaded item should contain values from the initial item"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
+ self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
def test_add_value_singlevalue_singlevalue(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- il.add_value('name', 'bar')
+ il.add_value("name", "bar")
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar']})
+ self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
def test_add_value_singlevalue_list(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- il.add_value('name', ['item', 'loader'])
+ il.add_value("name", ["item", "loader"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'item', 'loader']})
+ self.assertEqual(
+ ItemAdapter(loaded_item).asdict(), {"name": ["foo", "item", "loader"]}
+ )
def test_add_value_list_singlevalue(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- il.add_value('name', 'qwerty')
+ il.add_value("name", "qwerty")
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'qwerty']})
+ self.assertEqual(
+ ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar", "qwerty"]}
+ )
def test_add_value_list_list(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- il.add_value('name', ['item', 'loader'])
+ il.add_value("name", ["item", "loader"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), {'name': ['foo', 'bar', 'item', 'loader']})
+ self.assertEqual(
+ ItemAdapter(loaded_item).asdict(),
+ {"name": ["foo", "bar", "item", "loader"]},
+ )
def test_get_output_value_singlevalue(self):
"""Getting output value must not remove value from item"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- self.assertEqual(il.get_output_value('name'), ['foo'])
+ self.assertEqual(il.get_output_value("name"), ["foo"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo']}))
+ self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]}))
def test_get_output_value_list(self):
"""Getting output value must not remove value from item"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+ self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({'name': ['foo', 'bar']}))
+ self.assertEqual(
+ ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]})
+ )
def test_values_single(self):
"""Values from initial item must be added to loader._values"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- self.assertEqual(il._values.get('name'), ['foo'])
+ self.assertEqual(il._values.get("name"), ["foo"])
def test_values_list(self):
"""Values from initial item must be added to loader._values"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+ self.assertEqual(il._values.get("name"), ["foo", "bar"])
class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase):
@@ -207,42 +215,52 @@ class NoInputReprocessingFromItemTest(unittest.TestCase):
"""
Loaders initialized from loaded items must not reprocess fields (Item instances)
"""
+
def test_avoid_reprocessing_with_initial_values_single(self):
- il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo'))
+ il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title="foo"))
il_loaded = il.load_item()
- self.assertEqual(il_loaded, {'title': 'foo'})
- self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
+ self.assertEqual(il_loaded, {"title": "foo"})
+ self.assertEqual(
+ NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"}
+ )
def test_avoid_reprocessing_with_initial_values_list(self):
- il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title=['foo', 'bar']))
+ il = NoInputReprocessingItemLoader(
+ item=NoInputReprocessingItem(title=["foo", "bar"])
+ )
il_loaded = il.load_item()
- self.assertEqual(il_loaded, {'title': 'foo'})
- self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
+ self.assertEqual(il_loaded, {"title": "foo"})
+ self.assertEqual(
+ NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "foo"}
+ )
def test_avoid_reprocessing_without_initial_values_single(self):
il = NoInputReprocessingItemLoader()
- il.add_value('title', 'FOO')
+ il.add_value("title", "FOO")
il_loaded = il.load_item()
- self.assertEqual(il_loaded, {'title': 'FOO'})
- self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
+ self.assertEqual(il_loaded, {"title": "FOO"})
+ self.assertEqual(
+ NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"}
+ )
def test_avoid_reprocessing_without_initial_values_list(self):
il = NoInputReprocessingItemLoader()
- il.add_value('title', ['foo', 'bar'])
+ il.add_value("title", ["foo", "bar"])
il_loaded = il.load_item()
- self.assertEqual(il_loaded, {'title': 'FOO'})
- self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
+ self.assertEqual(il_loaded, {"title": "FOO"})
+ self.assertEqual(
+ NoInputReprocessingItemLoader(item=il_loaded).load_item(), {"title": "FOO"}
+ )
class TestOutputProcessorItem(unittest.TestCase):
def test_output_processor(self):
-
class TempItem(Item):
temp = Field()
def __init__(self, *args, **kwargs):
super().__init__(self, *args, **kwargs)
- self.setdefault('temp', 0.3)
+ self.setdefault("temp", 0.3)
class TempLoader(ItemLoader):
default_item_class = TempItem
@@ -252,11 +270,14 @@ class TestOutputProcessorItem(unittest.TestCase):
loader = TempLoader()
item = loader.load_item()
self.assertIsInstance(item, TempItem)
- self.assertEqual(dict(item), {'temp': 0.3})
+ self.assertEqual(dict(item), {"temp": 0.3})
class SelectortemLoaderTest(unittest.TestCase):
- response = HtmlResponse(url="", encoding='utf-8', body=b"""
+ response = HtmlResponse(
+ url="",
+ encoding="utf-8",
+ body=b"""
<html>
<body>
<div id="id">marta</div>
@@ -265,7 +286,8 @@ class SelectortemLoaderTest(unittest.TestCase):
<img src="/images/logo.png" width="244" height="65" alt="Scrapy">
</body>
</html>
- """)
+ """,
+ )
def test_init_method(self):
l = TestItemLoader()
@@ -273,28 +295,28 @@ class SelectortemLoaderTest(unittest.TestCase):
def test_init_method_errors(self):
l = TestItemLoader()
- self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href')
- self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href')
- self.assertRaises(RuntimeError, l.get_xpath, '//a/@href')
- self.assertRaises(RuntimeError, l.add_css, 'name', '#name::text')
- self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text')
- self.assertRaises(RuntimeError, l.get_css, '#name::text')
+ self.assertRaises(RuntimeError, l.add_xpath, "url", "//a/@href")
+ self.assertRaises(RuntimeError, l.replace_xpath, "url", "//a/@href")
+ self.assertRaises(RuntimeError, l.get_xpath, "//a/@href")
+ self.assertRaises(RuntimeError, l.add_css, "name", "#name::text")
+ self.assertRaises(RuntimeError, l.replace_css, "name", "#name::text")
+ self.assertRaises(RuntimeError, l.get_css, "#name::text")
def test_init_method_with_selector(self):
sel = Selector(text="<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
- l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
+ l.add_xpath("name", "//div/text()")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
def test_init_method_with_selector_css(self):
sel = Selector(text="<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
- l.add_css('name', 'div::text')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
+ l.add_css("name", "div::text")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
def test_init_method_with_base_response(self):
"""Selector should be None after initialization"""
@@ -306,122 +328,131 @@ class SelectortemLoaderTest(unittest.TestCase):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
- l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
+ l.add_xpath("name", "//div/text()")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
def test_init_method_with_response_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
- l.add_css('name', 'div::text')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
+ l.add_css("name", "div::text")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
- l.add_css('url', 'a::attr(href)')
- self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
+ l.add_css("url", "a::attr(href)")
+ self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
# combining/accumulating CSS selectors and XPath expressions
- l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
+ l.add_xpath("name", "//div/text()")
+ self.assertEqual(l.get_output_value("name"), ["Marta", "Marta"])
- l.add_xpath('url', '//img/@src')
- self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
+ l.add_xpath("url", "//img/@src")
+ self.assertEqual(
+ l.get_output_value("url"), ["http://www.scrapy.org", "/images/logo.png"]
+ )
def test_add_xpath_re(self):
l = TestItemLoader(response=self.response)
- l.add_xpath('name', '//div/text()', re='ma')
- self.assertEqual(l.get_output_value('name'), ['Ma'])
+ l.add_xpath("name", "//div/text()", re="ma")
+ self.assertEqual(l.get_output_value("name"), ["Ma"])
def test_replace_xpath(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
- l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
- l.replace_xpath('name', '//p/text()')
- self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+ l.add_xpath("name", "//div/text()")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
+ l.replace_xpath("name", "//p/text()")
+ self.assertEqual(l.get_output_value("name"), ["Paragraph"])
- l.replace_xpath('name', ['//p/text()', '//div/text()'])
- self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
+ l.replace_xpath("name", ["//p/text()", "//div/text()"])
+ self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
def test_get_xpath(self):
l = TestItemLoader(response=self.response)
- self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
- self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
- self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
+ self.assertEqual(l.get_xpath("//p/text()"), ["paragraph"])
+ self.assertEqual(l.get_xpath("//p/text()", TakeFirst()), "paragraph")
+ self.assertEqual(l.get_xpath("//p/text()", TakeFirst(), re="pa"), "pa")
- self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
+ self.assertEqual(
+ l.get_xpath(["//p/text()", "//div/text()"]), ["paragraph", "marta"]
+ )
def test_replace_xpath_multi_fields(self):
l = TestItemLoader(response=self.response)
- l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), ['Marta'])
- l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+ l.add_xpath(None, "//div/text()", TakeFirst(), lambda x: {"name": x})
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
+ l.replace_xpath(None, "//p/text()", TakeFirst(), lambda x: {"name": x})
+ self.assertEqual(l.get_output_value("name"), ["Paragraph"])
def test_replace_xpath_re(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
- l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
- l.replace_xpath('name', '//div/text()', re='ma')
- self.assertEqual(l.get_output_value('name'), ['Ma'])
+ l.add_xpath("name", "//div/text()")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
+ l.replace_xpath("name", "//div/text()", re="ma")
+ self.assertEqual(l.get_output_value("name"), ["Ma"])
def test_add_css_re(self):
l = TestItemLoader(response=self.response)
- l.add_css('name', 'div::text', re='ma')
- self.assertEqual(l.get_output_value('name'), ['Ma'])
+ l.add_css("name", "div::text", re="ma")
+ self.assertEqual(l.get_output_value("name"), ["Ma"])
- l.add_css('url', 'a::attr(href)', re='http://(.+)')
- self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
+ l.add_css("url", "a::attr(href)", re="http://(.+)")
+ self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
def test_replace_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
- l.add_css('name', 'div::text')
- self.assertEqual(l.get_output_value('name'), ['Marta'])
- l.replace_css('name', 'p::text')
- self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+ l.add_css("name", "div::text")
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
+ l.replace_css("name", "p::text")
+ self.assertEqual(l.get_output_value("name"), ["Paragraph"])
- l.replace_css('name', ['p::text', 'div::text'])
- self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
+ l.replace_css("name", ["p::text", "div::text"])
+ self.assertEqual(l.get_output_value("name"), ["Paragraph", "Marta"])
- l.add_css('url', 'a::attr(href)', re='http://(.+)')
- self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
- l.replace_css('url', 'img::attr(src)')
- self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
+ l.add_css("url", "a::attr(href)", re="http://(.+)")
+ self.assertEqual(l.get_output_value("url"), ["www.scrapy.org"])
+ l.replace_css("url", "img::attr(src)")
+ self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
def test_get_css(self):
l = TestItemLoader(response=self.response)
- self.assertEqual(l.get_css('p::text'), ['paragraph'])
- self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
- self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
+ self.assertEqual(l.get_css("p::text"), ["paragraph"])
+ self.assertEqual(l.get_css("p::text", TakeFirst()), "paragraph")
+ self.assertEqual(l.get_css("p::text", TakeFirst(), re="pa"), "pa")
- self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
- self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
- ['http://www.scrapy.org', '/images/logo.png'])
+ self.assertEqual(l.get_css(["p::text", "div::text"]), ["paragraph", "marta"])
+ self.assertEqual(
+ l.get_css(["a::attr(href)", "img::attr(src)"]),
+ ["http://www.scrapy.org", "/images/logo.png"],
+ )
def test_replace_css_multi_fields(self):
l = TestItemLoader(response=self.response)
- l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), ['Marta'])
- l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), ['Paragraph'])
+ l.add_css(None, "div::text", TakeFirst(), lambda x: {"name": x})
+ self.assertEqual(l.get_output_value("name"), ["Marta"])
+ l.replace_css(None, "p::text", TakeFirst(), lambda x: {"name": x})
+ self.assertEqual(l.get_output_value("name"), ["Paragraph"])
- l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
- self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
- l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
- self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
+ l.add_css(None, "a::attr(href)", TakeFirst(), lambda x: {"url": x})
+ self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
+ l.replace_css(None, "img::attr(src)", TakeFirst(), lambda x: {"url": x})
+ self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
def test_replace_css_re(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
- l.add_css('url', 'a::attr(href)')
- self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
- l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
- self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
+ l.add_css("url", "a::attr(href)")
+ self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
+ l.replace_css("url", "a::attr(href)", re=r"http://www\.(.+)")
+ self.assertEqual(l.get_output_value("url"), ["scrapy.org"])
class SubselectorLoaderTest(unittest.TestCase):
- response = HtmlResponse(url="", encoding='utf-8', body=b"""
+ response = HtmlResponse(
+ url="",
+ encoding="utf-8",
+ body=b"""
<html>
<body>
<header>
@@ -434,76 +465,88 @@ class SubselectorLoaderTest(unittest.TestCase):
</footer>
</body>
</html>
- """)
+ """,
+ )
def test_nested_xpath(self):
l = NestedItemLoader(response=self.response)
nl = l.nested_xpath("//header")
- nl.add_xpath('name', 'div/text()')
- nl.add_css('name_div', '#id')
- nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
+ nl.add_xpath("name", "div/text()")
+ nl.add_css("name_div", "#id")
+ nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
- self.assertEqual(l.get_output_value('name'), ['marta'])
- self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
- self.assertEqual(l.get_output_value('name_value'), ['marta'])
+ self.assertEqual(l.get_output_value("name"), ["marta"])
+ self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>'])
+ self.assertEqual(l.get_output_value("name_value"), ["marta"])
- self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
- self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
- self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+ self.assertEqual(l.get_output_value("name"), nl.get_output_value("name"))
+ self.assertEqual(
+ l.get_output_value("name_div"), nl.get_output_value("name_div")
+ )
+ self.assertEqual(
+ l.get_output_value("name_value"), nl.get_output_value("name_value")
+ )
def test_nested_css(self):
l = NestedItemLoader(response=self.response)
nl = l.nested_css("header")
- nl.add_xpath('name', 'div/text()')
- nl.add_css('name_div', '#id')
- nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
+ nl.add_xpath("name", "div/text()")
+ nl.add_css("name_div", "#id")
+ nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
- self.assertEqual(l.get_output_value('name'), ['marta'])
- self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
- self.assertEqual(l.get_output_value('name_value'), ['marta'])
+ self.assertEqual(l.get_output_value("name"), ["marta"])
+ self.assertEqual(l.get_output_value("name_div"), ['<div id="id">marta</div>'])
+ self.assertEqual(l.get_output_value("name_value"), ["marta"])
- self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
- self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
- self.assertEqual(l.get_output_value('name_value'), nl.get_output_value('name_value'))
+ self.assertEqual(l.get_output_value("name"), nl.get_output_value("name"))
+ self.assertEqual(
+ l.get_output_value("name_div"), nl.get_output_value("name_div")
+ )
+ self.assertEqual(
+ l.get_output_value("name_value"), nl.get_output_value("name_value")
+ )
def test_nested_replace(self):
l = NestedItemLoader(response=self.response)
- nl1 = l.nested_xpath('//footer')
- nl2 = nl1.nested_xpath('a')
+ nl1 = l.nested_xpath("//footer")
+ nl2 = nl1.nested_xpath("a")
- l.add_xpath('url', '//footer/a/@href')
- self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
- nl1.replace_xpath('url', 'img/@src')
- self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
- nl2.replace_xpath('url', '@href')
- self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
+ l.add_xpath("url", "//footer/a/@href")
+ self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
+ nl1.replace_xpath("url", "img/@src")
+ self.assertEqual(l.get_output_value("url"), ["/images/logo.png"])
+ nl2.replace_xpath("url", "@href")
+ self.assertEqual(l.get_output_value("url"), ["http://www.scrapy.org"])
def test_nested_ordering(self):
l = NestedItemLoader(response=self.response)
- nl1 = l.nested_xpath('//footer')
- nl2 = nl1.nested_xpath('a')
+ nl1 = l.nested_xpath("//footer")
+ nl2 = nl1.nested_xpath("a")
- nl1.add_xpath('url', 'img/@src')
- l.add_xpath('url', '//footer/a/@href')
- nl2.add_xpath('url', 'text()')
- l.add_xpath('url', '//footer/a/@href')
+ nl1.add_xpath("url", "img/@src")
+ l.add_xpath("url", "//footer/a/@href")
+ nl2.add_xpath("url", "text()")
+ l.add_xpath("url", "//footer/a/@href")
- self.assertEqual(l.get_output_value('url'), [
- '/images/logo.png',
- 'http://www.scrapy.org',
- 'homepage',
- 'http://www.scrapy.org',
- ])
+ self.assertEqual(
+ l.get_output_value("url"),
+ [
+ "/images/logo.png",
+ "http://www.scrapy.org",
+ "homepage",
+ "http://www.scrapy.org",
+ ],
+ )
def test_nested_load_item(self):
l = NestedItemLoader(response=self.response)
- nl1 = l.nested_xpath('//footer')
- nl2 = nl1.nested_xpath('img')
+ nl1 = l.nested_xpath("//footer")
+ nl2 = nl1.nested_xpath("img")
- l.add_xpath('name', '//header/div/text()')
- nl1.add_xpath('url', 'a/@href')
- nl2.add_xpath('image', '@src')
+ l.add_xpath("name", "//header/div/text()")
+ nl1.add_xpath("url", "a/@href")
+ nl2.add_xpath("image", "@src")
item = l.load_item()
@@ -511,13 +554,14 @@ class SubselectorLoaderTest(unittest.TestCase):
assert item is nl1.item
assert item is nl2.item
- self.assertEqual(item['name'], ['marta'])
- self.assertEqual(item['url'], ['http://www.scrapy.org'])
- self.assertEqual(item['image'], ['/images/logo.png'])
+ self.assertEqual(item["name"], ["marta"])
+ self.assertEqual(item["url"], ["http://www.scrapy.org"])
+ self.assertEqual(item["image"], ["/images/logo.png"])
# Functions as processors
+
def function_processor_strip(iterable):
return [x.strip() for x in iterable]
@@ -538,15 +582,11 @@ class FunctionProcessorItemLoader(ItemLoader):
class FunctionProcessorTestCase(unittest.TestCase):
-
def test_processor_defined_in_item(self):
lo = FunctionProcessorItemLoader()
- lo.add_value('foo', ' bar ')
- lo.add_value('foo', [' asdf ', ' qwerty '])
- self.assertEqual(
- dict(lo.load_item()),
- {'foo': ['BAR', 'ASDF', 'QWERTY']}
- )
+ lo.add_value("foo", " bar ")
+ lo.add_value("foo", [" asdf ", " qwerty "])
+ self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
if __name__ == "__main__":
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 14b3b5568..d7f773d5c 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -4,17 +4,19 @@ Once we remove the references from scrapy, we can remove these tests.
"""
import unittest
-import warnings
from functools import partial
-from itemloaders.processors import (Compose, Identity, Join,
- MapCompose, SelectJmes, TakeFirst)
+from itemloaders.processors import (
+ Compose,
+ Identity,
+ Join,
+ MapCompose,
+ SelectJmes,
+ TakeFirst,
+)
-from scrapy.item import Item, Field
+from scrapy.item import Field, Item
from scrapy.loader import ItemLoader
-from scrapy.loader.common import wrap_loader_context
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.misc import extract_regex
# test items
@@ -42,28 +44,27 @@ class DefaultedItemLoader(NameItemLoader):
# test processors
def processor_with_args(value, other=None, loader_context=None):
- if 'key' in loader_context:
- return loader_context['key']
+ if "key" in loader_context:
+ return loader_context["key"]
return value
class BasicItemLoaderTest(unittest.TestCase):
-
def test_load_item_using_default_loader(self):
i = TestItem()
- i['summary'] = 'lala'
+ i["summary"] = "lala"
il = ItemLoader(item=i)
- il.add_value('name', 'marta')
+ il.add_value("name", "marta")
item = il.load_item()
assert item is i
- self.assertEqual(item['summary'], ['lala'])
- self.assertEqual(item['name'], ['marta'])
+ self.assertEqual(item["summary"], ["lala"])
+ self.assertEqual(item["name"], ["marta"])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
- il.add_value('name', 'marta')
+ il.add_value("name", "marta")
item = il.load_item()
- self.assertEqual(item['name'], ['Marta'])
+ self.assertEqual(item["name"], ["Marta"])
def test_load_item_ignore_none_field_values(self):
def validate_sku(value):
@@ -76,206 +77,223 @@ class BasicItemLoaderTest(unittest.TestCase):
price_out = Compose(TakeFirst(), float)
sku_out = Compose(TakeFirst(), validate_sku)
- valid_fragment = 'SKU: 1234'
- invalid_fragment = 'SKU: not available'
- sku_re = 'SKU: (.+)'
+ valid_fragment = "SKU: 1234"
+ invalid_fragment = "SKU: not available"
+ sku_re = "SKU: (.+)"
il = MyLoader(item={})
# Should not return "sku: None".
- il.add_value('sku', [invalid_fragment], re=sku_re)
+ il.add_value("sku", [invalid_fragment], re=sku_re)
# Should not ignore empty values.
- il.add_value('name', '')
- il.add_value('price', ['0'])
- self.assertEqual(il.load_item(), {
- 'name': '',
- 'price': 0.0,
- })
+ il.add_value("name", "")
+ il.add_value("price", ["0"])
+ self.assertEqual(
+ il.load_item(),
+ {
+ "name": "",
+ "price": 0.0,
+ },
+ )
- il.replace_value('sku', [valid_fragment], re=sku_re)
- self.assertEqual(il.load_item()['sku'], '1234')
+ il.replace_value("sku", [valid_fragment], re=sku_re)
+ self.assertEqual(il.load_item()["sku"], "1234")
def test_self_referencing_loader(self):
class MyLoader(ItemLoader):
url_out = TakeFirst()
def img_url_out(self, values):
- return (self.get_output_value('url') or '') + values[0]
+ return (self.get_output_value("url") or "") + values[0]
il = MyLoader(item={})
- il.add_value('url', 'http://example.com/')
- il.add_value('img_url', '1234.png')
- self.assertEqual(il.load_item(), {
- 'url': 'http://example.com/',
- 'img_url': 'http://example.com/1234.png',
- })
+ il.add_value("url", "http://example.com/")
+ il.add_value("img_url", "1234.png")
+ self.assertEqual(
+ il.load_item(),
+ {
+ "url": "http://example.com/",
+ "img_url": "http://example.com/1234.png",
+ },
+ )
il = MyLoader(item={})
- il.add_value('img_url', '1234.png')
- self.assertEqual(il.load_item(), {
- 'img_url': '1234.png',
- })
+ il.add_value("img_url", "1234.png")
+ self.assertEqual(
+ il.load_item(),
+ {
+ "img_url": "1234.png",
+ },
+ )
def test_add_value(self):
il = TestItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_collected_values('name'), ['Marta'])
- self.assertEqual(il.get_output_value('name'), ['Marta'])
- il.add_value('name', 'pepe')
- self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
- self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_collected_values("name"), ["Marta"])
+ self.assertEqual(il.get_output_value("name"), ["Marta"])
+ il.add_value("name", "pepe")
+ self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe"])
+ self.assertEqual(il.get_output_value("name"), ["Marta", "Pepe"])
# test add object value
- il.add_value('summary', {'key': 1})
- self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
+ il.add_value("summary", {"key": 1})
+ self.assertEqual(il.get_collected_values("summary"), [{"key": 1}])
- il.add_value(None, 'Jim', lambda x: {'name': x})
- self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
+ il.add_value(None, "Jim", lambda x: {"name": x})
+ self.assertEqual(il.get_collected_values("name"), ["Marta", "Pepe", "Jim"])
def test_add_zero(self):
il = NameItemLoader()
- il.add_value('name', 0)
- self.assertEqual(il.get_collected_values('name'), [0])
+ il.add_value("name", 0)
+ self.assertEqual(il.get_collected_values("name"), [0])
def test_replace_value(self):
il = TestItemLoader()
- il.replace_value('name', 'marta')
- self.assertEqual(il.get_collected_values('name'), ['Marta'])
- self.assertEqual(il.get_output_value('name'), ['Marta'])
- il.replace_value('name', 'pepe')
- self.assertEqual(il.get_collected_values('name'), ['Pepe'])
- self.assertEqual(il.get_output_value('name'), ['Pepe'])
+ il.replace_value("name", "marta")
+ self.assertEqual(il.get_collected_values("name"), ["Marta"])
+ self.assertEqual(il.get_output_value("name"), ["Marta"])
+ il.replace_value("name", "pepe")
+ self.assertEqual(il.get_collected_values("name"), ["Pepe"])
+ self.assertEqual(il.get_output_value("name"), ["Pepe"])
- il.replace_value(None, 'Jim', lambda x: {'name': x})
- self.assertEqual(il.get_collected_values('name'), ['Jim'])
+ il.replace_value(None, "Jim", lambda x: {"name": x})
+ self.assertEqual(il.get_collected_values("name"), ["Jim"])
def test_get_value(self):
il = NameItemLoader()
- self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
- self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
- self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
+ self.assertEqual("FOO", il.get_value(["foo", "bar"], TakeFirst(), str.upper))
+ self.assertEqual(
+ ["foo", "bar"], il.get_value(["name:foo", "name:bar"], re="name:(.*)$")
+ )
+ self.assertEqual(
+ "foo", il.get_value(["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$")
+ )
- il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
- self.assertEqual(['foo'], il.get_collected_values('name'))
- il.replace_value('name', 'name:bar', re='name:(.*)$')
- self.assertEqual(['bar'], il.get_collected_values('name'))
+ il.add_value("name", ["name:foo", "name:bar"], TakeFirst(), re="name:(.*)$")
+ self.assertEqual(["foo"], il.get_collected_values("name"))
+ il.replace_value("name", "name:bar", re="name:(.*)$")
+ self.assertEqual(["bar"], il.get_collected_values("name"))
def test_iter_on_input_processor_input(self):
class NameFirstItemLoader(NameItemLoader):
name_in = TakeFirst()
il = NameFirstItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_collected_values('name'), ['marta'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_collected_values("name"), ["marta"])
il = NameFirstItemLoader()
- il.add_value('name', ['marta', 'jose'])
- self.assertEqual(il.get_collected_values('name'), ['marta'])
+ il.add_value("name", ["marta", "jose"])
+ self.assertEqual(il.get_collected_values("name"), ["marta"])
il = NameFirstItemLoader()
- il.replace_value('name', 'marta')
- self.assertEqual(il.get_collected_values('name'), ['marta'])
+ il.replace_value("name", "marta")
+ self.assertEqual(il.get_collected_values("name"), ["marta"])
il = NameFirstItemLoader()
- il.replace_value('name', ['marta', 'jose'])
- self.assertEqual(il.get_collected_values('name'), ['marta'])
+ il.replace_value("name", ["marta", "jose"])
+ self.assertEqual(il.get_collected_values("name"), ["marta"])
il = NameFirstItemLoader()
- il.add_value('name', 'marta')
- il.add_value('name', ['jose', 'pedro'])
- self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
+ il.add_value("name", "marta")
+ il.add_value("name", ["jose", "pedro"])
+ self.assertEqual(il.get_collected_values("name"), ["marta", "jose"])
def test_map_compose_filter(self):
def filter_world(x):
- return None if x == 'world' else x
+ return None if x == "world" else x
proc = MapCompose(filter_world, str.upper)
- self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
- ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+ self.assertEqual(
+ proc(["hello", "world", "this", "is", "scrapy"]),
+ ["HELLO", "THIS", "IS", "SCRAPY"],
+ )
def test_map_compose_filter_multil(self):
class TestItemLoader(NameItemLoader):
name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['Mart'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["Mart"])
item = il.load_item()
- self.assertEqual(item['name'], ['Mart'])
+ self.assertEqual(item["name"], ["Mart"])
def test_default_input_processor(self):
il = DefaultedItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['mart'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["mart"])
def test_inherited_default_input_processor(self):
class InheritDefaultedItemLoader(DefaultedItemLoader):
pass
il = InheritDefaultedItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['mart'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["mart"])
def test_input_processor_inheritance(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(lambda v: v.lower())
il = ChildItemLoader()
- il.add_value('url', 'HTTP://scrapy.ORG')
- self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['Marta'])
+ il.add_value("url", "HTTP://scrapy.ORG")
+ self.assertEqual(il.get_output_value("url"), ["http://scrapy.org"])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["Marta"])
class ChildChildItemLoader(ChildItemLoader):
url_in = MapCompose(lambda v: v.upper())
summary_in = MapCompose(lambda v: v)
il = ChildChildItemLoader()
- il.add_value('url', 'http://scrapy.org')
- self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['Marta'])
+ il.add_value("url", "http://scrapy.org")
+ self.assertEqual(il.get_output_value("url"), ["HTTP://SCRAPY.ORG"])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["Marta"])
def test_empty_map_compose(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose()
il = IdentityDefaultedItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['marta'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["marta"])
def test_identity_input_processor(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = Identity()
il = IdentityDefaultedItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['marta'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["marta"])
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['mARTA'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["mARTA"])
def test_extend_default_input_processors(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
- name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
+ name_in = MapCompose(
+ DefaultedItemLoader.default_input_processor, str.swapcase
+ )
il = ChildDefaultedItemLoader()
- il.add_value('name', 'marta')
- self.assertEqual(il.get_output_value('name'), ['MART'])
+ il.add_value("name", "marta")
+ self.assertEqual(il.get_output_value("name"), ["MART"])
def test_output_processor_using_function(self):
il = TestItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
class TakeFirstItemLoader(TestItemLoader):
name_out = " ".join
il = TakeFirstItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), 'Mar Ta')
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), "Mar Ta")
def test_output_processor_error(self):
class TestItemLoader(ItemLoader):
@@ -283,9 +301,9 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = MapCompose(float)
il = TestItemLoader()
- il.add_value('name', ['$10'])
+ il.add_value("name", ["$10"])
try:
- float('$10')
+ float("$10")
except Exception as e:
expected_exc_str = str(e)
@@ -296,118 +314,118 @@ class BasicItemLoaderTest(unittest.TestCase):
exc = e
assert isinstance(exc, ValueError)
s = str(exc)
- assert 'name' in s, s
- assert '$10' in s, s
- assert 'ValueError' in s, s
+ assert "name" in s, s
+ assert "$10" in s, s
+ assert "ValueError" in s, s
assert expected_exc_str in s, s
def test_output_processor_using_classes(self):
il = TestItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
class TakeFirstItemLoader(TestItemLoader):
name_out = Join()
il = TakeFirstItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), 'Mar Ta')
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), "Mar Ta")
class TakeFirstItemLoader(TestItemLoader):
name_out = Join("<br>")
il = TakeFirstItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), "Mar<br>Ta")
def test_default_output_processor(self):
il = TestItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
class LalaItemLoader(TestItemLoader):
default_output_processor = Identity()
il = LalaItemLoader()
- il.add_value('name', ['mar', 'ta'])
- self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
+ il.add_value("name", ["mar", "ta"])
+ self.assertEqual(il.get_output_value("name"), ["Mar", "Ta"])
def test_loader_context_on_declaration(self):
class ChildItemLoader(TestItemLoader):
- url_in = MapCompose(processor_with_args, key='val')
+ url_in = MapCompose(processor_with_args, key="val")
il = ChildItemLoader()
- il.add_value('url', 'text')
- self.assertEqual(il.get_output_value('url'), ['val'])
- il.replace_value('url', 'text2')
- self.assertEqual(il.get_output_value('url'), ['val'])
+ il.add_value("url", "text")
+ self.assertEqual(il.get_output_value("url"), ["val"])
+ il.replace_value("url", "text2")
+ self.assertEqual(il.get_output_value("url"), ["val"])
def test_loader_context_on_instantiation(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(processor_with_args)
- il = ChildItemLoader(key='val')
- il.add_value('url', 'text')
- self.assertEqual(il.get_output_value('url'), ['val'])
- il.replace_value('url', 'text2')
- self.assertEqual(il.get_output_value('url'), ['val'])
+ il = ChildItemLoader(key="val")
+ il.add_value("url", "text")
+ self.assertEqual(il.get_output_value("url"), ["val"])
+ il.replace_value("url", "text2")
+ self.assertEqual(il.get_output_value("url"), ["val"])
def test_loader_context_on_assign(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(processor_with_args)
il = ChildItemLoader()
- il.context['key'] = 'val'
- il.add_value('url', 'text')
- self.assertEqual(il.get_output_value('url'), ['val'])
- il.replace_value('url', 'text2')
- self.assertEqual(il.get_output_value('url'), ['val'])
+ il.context["key"] = "val"
+ il.add_value("url", "text")
+ self.assertEqual(il.get_output_value("url"), ["val"])
+ il.replace_value("url", "text2")
+ self.assertEqual(il.get_output_value("url"), ["val"])
def test_item_passed_to_input_processor_functions(self):
def processor(value, loader_context):
- return loader_context['item']['name']
+ return loader_context["item"]["name"]
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(processor)
- it = TestItem(name='marta')
+ it = TestItem(name="marta")
il = ChildItemLoader(item=it)
- il.add_value('url', 'text')
- self.assertEqual(il.get_output_value('url'), ['marta'])
- il.replace_value('url', 'text2')
- self.assertEqual(il.get_output_value('url'), ['marta'])
+ il.add_value("url", "text")
+ self.assertEqual(il.get_output_value("url"), ["marta"])
+ il.replace_value("url", "text2")
+ self.assertEqual(il.get_output_value("url"), ["marta"])
def test_compose_processor(self):
class TestItemLoader(NameItemLoader):
name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
- il.add_value('name', ['marta', 'other'])
- self.assertEqual(il.get_output_value('name'), 'Mart')
+ il.add_value("name", ["marta", "other"])
+ self.assertEqual(il.get_output_value("name"), "Mart")
item = il.load_item()
- self.assertEqual(item['name'], 'Mart')
+ self.assertEqual(item["name"], "Mart")
def test_partial_processor(self):
def join(values, sep=None, loader_context=None, ignored=None):
if sep is not None:
return sep.join(values)
- if loader_context and 'sep' in loader_context:
- return loader_context['sep'].join(values)
- return ''.join(values)
+ if loader_context and "sep" in loader_context:
+ return loader_context["sep"].join(values)
+ return "".join(values)
class TestItemLoader(NameItemLoader):
- name_out = Compose(partial(join, sep='+'))
- url_out = Compose(partial(join, loader_context={'sep': '.'}))
- summary_out = Compose(partial(join, ignored='foo'))
+ name_out = Compose(partial(join, sep="+"))
+ url_out = Compose(partial(join, loader_context={"sep": "."}))
+ summary_out = Compose(partial(join, ignored="foo"))
il = TestItemLoader()
- il.add_value('name', ['rabbit', 'hole'])
- il.add_value('url', ['rabbit', 'hole'])
- il.add_value('summary', ['rabbit', 'hole'])
+ il.add_value("name", ["rabbit", "hole"])
+ il.add_value("url", ["rabbit", "hole"])
+ il.add_value("summary", ["rabbit", "hole"])
item = il.load_item()
- self.assertEqual(item['name'], 'rabbit+hole')
- self.assertEqual(item['url'], 'rabbit.hole')
- self.assertEqual(item['summary'], 'rabbithole')
+ self.assertEqual(item["name"], "rabbit+hole")
+ self.assertEqual(item["url"], "rabbit.hole")
+ self.assertEqual(item["summary"], "rabbithole")
def test_error_input_processor(self):
class TestItem(Item):
@@ -418,8 +436,7 @@ class BasicItemLoaderTest(unittest.TestCase):
name_in = MapCompose(float)
il = TestItemLoader()
- self.assertRaises(ValueError, il.add_value, 'name',
- ['marta', 'other'])
+ self.assertRaises(ValueError, il.add_value, "name", ["marta", "other"])
def test_error_output_processor(self):
class TestItem(Item):
@@ -430,7 +447,7 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = Compose(Join(), float)
il = TestItemLoader()
- il.add_value('name', 'marta')
+ il.add_value("name", "marta")
with self.assertRaises(ValueError):
il.load_item()
@@ -442,95 +459,95 @@ class BasicItemLoaderTest(unittest.TestCase):
default_item_class = TestItem
il = TestItemLoader()
- self.assertRaises(ValueError, il.add_value, 'name',
- ['marta', 'other'], Compose(float))
+ self.assertRaises(
+ ValueError, il.add_value, "name", ["marta", "other"], Compose(float)
+ )
class InitializationFromDictTest(unittest.TestCase):
-
item_class = dict
def test_keep_single_value(self):
"""Loaded item should contain values from the initial item"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(dict(loaded_item), {'name': ['foo']})
+ self.assertEqual(dict(loaded_item), {"name": ["foo"]})
def test_keep_list(self):
"""Loaded item should contain values from the initial item"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+ self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]})
def test_add_value_singlevalue_singlevalue(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- il.add_value('name', 'bar')
+ il.add_value("name", "bar")
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
+ self.assertEqual(dict(loaded_item), {"name": ["foo", "bar"]})
def test_add_value_singlevalue_list(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- il.add_value('name', ['item', 'loader'])
+ il.add_value("name", ["item", "loader"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
+ self.assertEqual(dict(loaded_item), {"name": ["foo", "item", "loader"]})
def test_add_value_list_singlevalue(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- il.add_value('name', 'qwerty')
+ il.add_value("name", "qwerty")
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
+ self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "qwerty"]})
def test_add_value_list_list(self):
"""Values added after initialization should be appended"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- il.add_value('name', ['item', 'loader'])
+ il.add_value("name", ["item", "loader"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
+ self.assertEqual(dict(loaded_item), {"name": ["foo", "bar", "item", "loader"]})
def test_get_output_value_singlevalue(self):
"""Getting output value must not remove value from item"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- self.assertEqual(il.get_output_value('name'), ['foo'])
+ self.assertEqual(il.get_output_value("name"), ["foo"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(loaded_item, dict({'name': ['foo']}))
+ self.assertEqual(loaded_item, dict({"name": ["foo"]}))
def test_get_output_value_list(self):
"""Getting output value must not remove value from item"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
+ self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
- self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
+ self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]}))
def test_values_single(self):
"""Values from initial item must be added to loader._values"""
- input_item = self.item_class(name='foo')
+ input_item = self.item_class(name="foo")
il = ItemLoader(item=input_item)
- self.assertEqual(il._values.get('name'), ['foo'])
+ self.assertEqual(il._values.get("name"), ["foo"])
def test_values_list(self):
"""Values from initial item must be added to loader._values"""
- input_item = self.item_class(name=['foo', 'bar'])
+ input_item = self.item_class(name=["foo", "bar"])
il = ItemLoader(item=input_item)
- self.assertEqual(il._values.get('name'), ['foo', 'bar'])
+ self.assertEqual(il._values.get("name"), ["foo", "bar"])
class BaseNoInputReprocessingLoader(ItemLoader):
@@ -546,40 +563,48 @@ class NoInputReprocessingFromDictTest(unittest.TestCase):
"""
Loaders initialized from loaded items must not reprocess fields (dict instances)
"""
+
def test_avoid_reprocessing_with_initial_values_single(self):
- il = NoInputReprocessingDictLoader(item=dict(title='foo'))
+ il = NoInputReprocessingDictLoader(item=dict(title="foo"))
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title='foo'))
- self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+ self.assertEqual(il_loaded, dict(title="foo"))
+ self.assertEqual(
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo")
+ )
def test_avoid_reprocessing_with_initial_values_list(self):
- il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
+ il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"]))
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title='foo'))
- self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
+ self.assertEqual(il_loaded, dict(title="foo"))
+ self.assertEqual(
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo")
+ )
def test_avoid_reprocessing_without_initial_values_single(self):
il = NoInputReprocessingDictLoader()
- il.add_value('title', 'foo')
+ il.add_value("title", "foo")
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title='FOO'))
- self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+ self.assertEqual(il_loaded, dict(title="FOO"))
+ self.assertEqual(
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO")
+ )
def test_avoid_reprocessing_without_initial_values_list(self):
il = NoInputReprocessingDictLoader()
- il.add_value('title', ['foo', 'bar'])
+ il.add_value("title", ["foo", "bar"])
il_loaded = il.load_item()
- self.assertEqual(il_loaded, dict(title='FOO'))
- self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
+ self.assertEqual(il_loaded, dict(title="FOO"))
+ self.assertEqual(
+ NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO")
+ )
class TestOutputProcessorDict(unittest.TestCase):
def test_output_processor(self):
-
class TempDict(dict):
def __init__(self, *args, **kwargs):
super().__init__(self, *args, **kwargs)
- self.setdefault('temp', 0.3)
+ self.setdefault("temp", 0.3)
class TempLoader(ItemLoader):
default_item_class = TempDict
@@ -589,64 +614,67 @@ class TestOutputProcessorDict(unittest.TestCase):
loader = TempLoader()
item = loader.load_item()
self.assertIsInstance(item, TempDict)
- self.assertEqual(dict(item), {'temp': 0.3})
+ self.assertEqual(dict(item), {"temp": 0.3})
class ProcessorsTest(unittest.TestCase):
-
def test_take_first(self):
proc = TakeFirst()
- self.assertEqual(proc([None, '', 'hello', 'world']), 'hello')
- self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0)
+ self.assertEqual(proc([None, "", "hello", "world"]), "hello")
+ self.assertEqual(proc([None, "", 0, "hello", "world"]), 0)
def test_identity(self):
proc = Identity()
- self.assertEqual(proc([None, '', 'hello', 'world']),
- [None, '', 'hello', 'world'])
+ self.assertEqual(
+ proc([None, "", "hello", "world"]), [None, "", "hello", "world"]
+ )
def test_join(self):
proc = Join()
- self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
- self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
- self.assertEqual(proc(['hello', 'world']), 'hello world')
- self.assertIsInstance(proc(['hello', 'world']), str)
+ self.assertRaises(TypeError, proc, [None, "", "hello", "world"])
+ self.assertEqual(proc(["", "hello", "world"]), " hello world")
+ self.assertEqual(proc(["hello", "world"]), "hello world")
+ self.assertIsInstance(proc(["hello", "world"]), str)
def test_compose(self):
proc = Compose(lambda v: v[0], str.upper)
- self.assertEqual(proc(['hello', 'world']), 'HELLO')
+ self.assertEqual(proc(["hello", "world"]), "HELLO")
proc = Compose(str.upper)
self.assertEqual(proc(None), None)
proc = Compose(str.upper, stop_on_none=False)
self.assertRaises(ValueError, proc, None)
proc = Compose(str.upper, lambda x: x + 1)
- self.assertRaises(ValueError, proc, 'hello')
+ self.assertRaises(ValueError, proc, "hello")
def test_mapcompose(self):
def filter_world(x):
- return None if x == 'world' else x
+ return None if x == "world" else x
+
proc = MapCompose(filter_world, str.upper)
- self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
- ['HELLO', 'THIS', 'IS', 'SCRAPY'])
+ self.assertEqual(
+ proc(["hello", "world", "this", "is", "scrapy"]),
+ ["HELLO", "THIS", "IS", "SCRAPY"],
+ )
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
proc = MapCompose(filter_world, str.upper)
self.assertRaises(ValueError, proc, [1])
proc = MapCompose(filter_world, lambda x: x + 1)
- self.assertRaises(ValueError, proc, 'hello')
+ self.assertRaises(ValueError, proc, "hello")
class SelectJmesTestCase(unittest.TestCase):
test_list_equals = {
- 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
- 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
- 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
- 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
- 'dict': (
- 'foo.bar[*].name',
+ "simple": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"),
+ "invalid": ("foo.bar.baz", {"foo": {"bar": "baz"}}, None),
+ "top_level": ("foo", {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+ "double_vs_single_quote_string": ("foo.bar", {"foo": {"bar": "baz"}}, "baz"),
+ "dict": (
+ "foo.bar[*].name",
{"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
- ['one', 'two']
+ ["one", "two"],
),
- 'list': ('[1]', [1, 2], 2)
+ "list": ("[1]", [1, 2], 2),
}
def test_output(self):
@@ -654,14 +682,13 @@ class SelectJmesTestCase(unittest.TestCase):
expr, test_list, expected = self.test_list_equals[tl]
test = SelectJmes(expr)(test_list)
self.assertEqual(
- test,
- expected,
- msg=f'test "{tl}" got {test} expected {expected}'
+ test, expected, msg=f'test "{tl}" got {test} expected {expected}'
)
# Functions as processors
+
def function_processor_strip(iterable):
return [x.strip() for x in iterable]
@@ -684,35 +711,11 @@ class FunctionProcessorDictLoader(ItemLoader):
class FunctionProcessorTestCase(unittest.TestCase):
-
def test_processor_defined_in_item_loader(self):
lo = FunctionProcessorDictLoader()
- lo.add_value('foo', ' bar ')
- lo.add_value('foo', [' asdf ', ' qwerty '])
- self.assertEqual(
- dict(lo.load_item()),
- {'foo': ['BAR', 'ASDF', 'QWERTY']}
- )
-
-
-class DeprecatedUtilityFunctionsTestCase(unittest.TestCase):
-
- def test_deprecated_wrap_loader_context(self):
- def function(*args):
- return None
-
- with warnings.catch_warnings(record=True) as w:
- wrap_loader_context(function, context={})
-
- assert len(w) == 1
- assert issubclass(w[0].category, ScrapyDeprecationWarning)
-
- def test_deprecated_extract_regex(self):
- with warnings.catch_warnings(record=True) as w:
- extract_regex(r'\w+', 'this is a test')
-
- assert len(w) == 1
- assert issubclass(w[0].category, ScrapyDeprecationWarning)
+ lo.add_value("foo", " bar ")
+ lo.add_value("foo", [" asdf ", " qwerty "])
+ self.assertEqual(dict(lo.load_item()), {"foo": ["BAR", "ASDF", "QWERTY"]})
if __name__ == "__main__":
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index f3bb23bda..56810f2ff 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -6,17 +6,16 @@ from twisted.python.failure import Failure
from twisted.trial.unittest import TestCase as TwistedTestCase
from scrapy.exceptions import DropItem
-from scrapy.utils.test import get_crawler
from scrapy.http import Request, Response
-from scrapy.item import Item, Field
+from scrapy.item import Field, Item
from scrapy.logformatter import LogFormatter
from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
from tests.spiders import ItemSpider
class CustomItem(Item):
-
name = Field()
def __str__(self):
@@ -24,67 +23,74 @@ class CustomItem(Item):
class LogFormatterTestCase(unittest.TestCase):
-
def setUp(self):
self.formatter = LogFormatter()
- self.spider = Spider('default')
+ self.spider = Spider("default")
def test_crawled_with_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
- logline = logkws['msg'] % logkws['args']
- self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
+ logline = logkws["msg"] % logkws["args"]
+ self.assertEqual(
+ logline, "Crawled (200) <GET http://www.example.com> (referer: None)"
+ )
def test_crawled_without_referer(self):
- req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
- res = Response("http://www.example.com", flags=['cached'])
+ req = Request(
+ "http://www.example.com", headers={"referer": "http://example.com"}
+ )
+ res = Response("http://www.example.com", flags=["cached"])
logkws = self.formatter.crawled(req, res, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(
logline,
- "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+ "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']",
+ )
def test_flags_in_request(self):
- req = Request("http://www.example.com", flags=['test', 'flag'])
+ req = Request("http://www.example.com", flags=["test", "flag"])
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(
logline,
- "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
+ "Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)",
+ )
def test_dropped(self):
item = {}
exception = Exception("\u2018")
response = Response("http://www.example.com")
logkws = self.formatter.dropped(item, exception, response, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
lines = logline.splitlines()
assert all(isinstance(x, str) for x in lines)
- self.assertEqual(lines, ["Dropped: \u2018", '{}'])
+ self.assertEqual(lines, ["Dropped: \u2018", "{}"])
def test_item_error(self):
# In practice, the complete traceback is shown by passing the
# 'exc_info' argument to the logging function
- item = {'key': 'value'}
+ item = {"key": "value"}
exception = Exception()
response = Response("http://www.example.com")
logkws = self.formatter.item_error(item, exception, response, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(logline, "Error processing {'key': 'value'}")
def test_spider_error(self):
# In practice, the complete traceback is shown by passing the
# 'exc_info' argument to the logging function
failure = Failure(Exception())
- request = Request("http://www.example.com", headers={'Referer': 'http://example.org'})
+ request = Request(
+ "http://www.example.com", headers={"Referer": "http://example.org"}
+ )
response = Response("http://www.example.com", request=request)
logkws = self.formatter.spider_error(failure, request, response, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(
logline,
- "Spider error processing <GET http://www.example.com> (referer: http://example.org)"
+ "Spider error processing <GET http://www.example.com> (referer: http://example.org)",
)
def test_download_error_short(self):
@@ -93,7 +99,7 @@ class LogFormatterTestCase(unittest.TestCase):
failure = Failure(Exception())
request = Request("http://www.example.com")
logkws = self.formatter.download_error(failure, request, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(logline, "Error downloading <GET http://www.example.com>")
def test_download_error_long(self):
@@ -101,67 +107,77 @@ class LogFormatterTestCase(unittest.TestCase):
# 'exc_info' argument to the logging function
failure = Failure(Exception())
request = Request("http://www.example.com")
- logkws = self.formatter.download_error(failure, request, self.spider, "Some message")
- logline = logkws['msg'] % logkws['args']
- self.assertEqual(logline, "Error downloading <GET http://www.example.com>: Some message")
+ logkws = self.formatter.download_error(
+ failure, request, self.spider, "Some message"
+ )
+ logline = logkws["msg"] % logkws["args"]
+ self.assertEqual(
+ logline, "Error downloading <GET http://www.example.com>: Some message"
+ )
def test_scraped(self):
item = CustomItem()
- item['name'] = '\xa3'
+ item["name"] = "\xa3"
response = Response("http://www.example.com")
logkws = self.formatter.scraped(item, response, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
lines = logline.splitlines()
assert all(isinstance(x, str) for x in lines)
- self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
+ self.assertEqual(
+ lines, ["Scraped from <200 http://www.example.com>", "name: \xa3"]
+ )
class LogFormatterSubclass(LogFormatter):
def crawled(self, request, response, spider):
kwargs = super().crawled(request, response, spider)
- CRAWLEDMSG = (
- "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
- )
- log_args = kwargs['args']
- log_args['flags'] = str(request.flags)
+ CRAWLEDMSG = "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
+ log_args = kwargs["args"]
+ log_args["flags"] = str(request.flags)
return {
- 'level': kwargs['level'],
- 'msg': CRAWLEDMSG,
- 'args': log_args,
+ "level": kwargs["level"],
+ "msg": CRAWLEDMSG,
+ "args": log_args,
}
class LogformatterSubclassTest(LogFormatterTestCase):
def setUp(self):
self.formatter = LogFormatterSubclass()
- self.spider = Spider('default')
+ self.spider = Spider("default")
def test_crawled_with_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(
- logline,
- "Crawled (200) <GET http://www.example.com> (referer: None) []")
+ logline, "Crawled (200) <GET http://www.example.com> (referer: None) []"
+ )
def test_crawled_without_referer(self):
- req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached'])
+ req = Request(
+ "http://www.example.com",
+ headers={"referer": "http://example.com"},
+ flags=["cached"],
+ )
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(
logline,
- "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+ "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']",
+ )
def test_flags_in_request(self):
- req = Request("http://www.example.com", flags=['test', 'flag'])
+ req = Request("http://www.example.com", flags=["test", "flag"])
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
- logline = logkws['msg'] % logkws['args']
+ logline = logkws["msg"] % logkws["args"]
self.assertEqual(
logline,
- "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
+ "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']",
+ )
class SkipMessagesLogFormatter(LogFormatter):
@@ -191,8 +207,8 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
self.mockserver = MockServer()
self.mockserver.__enter__()
self.base_settings = {
- 'LOG_LEVEL': 'DEBUG',
- 'ITEM_PIPELINES': {
+ "LOG_LEVEL": "DEBUG",
+ "ITEM_PIPELINES": {
DropSomeItemsPipeline: 300,
},
}
@@ -212,7 +228,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
@defer.inlineCallbacks
def test_skip_messages(self):
settings = self.base_settings.copy()
- settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
+ settings["LOG_FORMATTER"] = SkipMessagesLogFormatter
crawler = get_crawler(ItemSpider, settings)
with LogCapture() as lc:
yield crawler.crawl(mockserver=self.mockserver)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 9b248fbfa..2535e58db 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -1,125 +1,168 @@
-# coding=utf-8
-
import unittest
-from io import BytesIO
from email.charset import Charset
+from io import BytesIO
+
+from twisted import version as twisted_version
+from twisted.internet import defer
+from twisted.internet._sslverify import ClientTLSOptions
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.versions import Version
from scrapy.mail import MailSender
class MailSenderTest(unittest.TestCase):
-
def test_send(self):
mailsender = MailSender(debug=True)
- mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
- _callback=self._catch_mail_sent)
+ mailsender.send(
+ to=["test@scrapy.org"],
+ subject="subject",
+ body="body",
+ _callback=self._catch_mail_sent,
+ )
assert self.catched_msg
- self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
- self.assertEqual(self.catched_msg['subject'], 'subject')
- self.assertEqual(self.catched_msg['body'], 'body')
+ self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"])
+ self.assertEqual(self.catched_msg["subject"], "subject")
+ self.assertEqual(self.catched_msg["body"], "body")
- msg = self.catched_msg['msg']
- self.assertEqual(msg['to'], 'test@scrapy.org')
- self.assertEqual(msg['subject'], 'subject')
- self.assertEqual(msg.get_payload(), 'body')
- self.assertEqual(msg.get('Content-Type'), 'text/plain')
+ msg = self.catched_msg["msg"]
+ self.assertEqual(msg["to"], "test@scrapy.org")
+ self.assertEqual(msg["subject"], "subject")
+ self.assertEqual(msg.get_payload(), "body")
+ self.assertEqual(msg.get("Content-Type"), "text/plain")
def test_send_single_values_to_and_cc(self):
mailsender = MailSender(debug=True)
- mailsender.send(to='test@scrapy.org', subject='subject', body='body',
- cc='test@scrapy.org', _callback=self._catch_mail_sent)
+ mailsender.send(
+ to="test@scrapy.org",
+ subject="subject",
+ body="body",
+ cc="test@scrapy.org",
+ _callback=self._catch_mail_sent,
+ )
def test_send_html(self):
mailsender = MailSender(debug=True)
- mailsender.send(to=['test@scrapy.org'], subject='subject',
- body='<p>body</p>', mimetype='text/html',
- _callback=self._catch_mail_sent)
+ mailsender.send(
+ to=["test@scrapy.org"],
+ subject="subject",
+ body="<p>body</p>",
+ mimetype="text/html",
+ _callback=self._catch_mail_sent,
+ )
- msg = self.catched_msg['msg']
- self.assertEqual(msg.get_payload(), '<p>body</p>')
- self.assertEqual(msg.get('Content-Type'), 'text/html')
+ msg = self.catched_msg["msg"]
+ self.assertEqual(msg.get_payload(), "<p>body</p>")
+ self.assertEqual(msg.get("Content-Type"), "text/html")
def test_send_attach(self):
attach = BytesIO()
- attach.write(b'content')
+ attach.write(b"content")
attach.seek(0)
- attachs = [('attachment', 'text/plain', attach)]
+ attachs = [("attachment", "text/plain", attach)]
mailsender = MailSender(debug=True)
- mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
- attachs=attachs, _callback=self._catch_mail_sent)
+ mailsender.send(
+ to=["test@scrapy.org"],
+ subject="subject",
+ body="body",
+ attachs=attachs,
+ _callback=self._catch_mail_sent,
+ )
assert self.catched_msg
- self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
- self.assertEqual(self.catched_msg['subject'], 'subject')
- self.assertEqual(self.catched_msg['body'], 'body')
+ self.assertEqual(self.catched_msg["to"], ["test@scrapy.org"])
+ self.assertEqual(self.catched_msg["subject"], "subject")
+ self.assertEqual(self.catched_msg["body"], "body")
- msg = self.catched_msg['msg']
- self.assertEqual(msg['to'], 'test@scrapy.org')
- self.assertEqual(msg['subject'], 'subject')
+ msg = self.catched_msg["msg"]
+ self.assertEqual(msg["to"], "test@scrapy.org")
+ self.assertEqual(msg["subject"], "subject")
payload = msg.get_payload()
assert isinstance(payload, list)
self.assertEqual(len(payload), 2)
text, attach = payload
- self.assertEqual(text.get_payload(decode=True), b'body')
- self.assertEqual(text.get_charset(), Charset('us-ascii'))
- self.assertEqual(attach.get_payload(decode=True), b'content')
+ self.assertEqual(text.get_payload(decode=True), b"body")
+ self.assertEqual(text.get_charset(), Charset("us-ascii"))
+ self.assertEqual(attach.get_payload(decode=True), b"content")
def _catch_mail_sent(self, **kwargs):
self.catched_msg = dict(**kwargs)
def test_send_utf8(self):
- subject = 'sübjèçt'
- body = 'bödÿ-àéïöñß'
+ subject = "sübjèçt"
+ body = "bödÿ-àéïöñß"
mailsender = MailSender(debug=True)
- mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
- charset='utf-8', _callback=self._catch_mail_sent)
+ mailsender.send(
+ to=["test@scrapy.org"],
+ subject=subject,
+ body=body,
+ charset="utf-8",
+ _callback=self._catch_mail_sent,
+ )
assert self.catched_msg
- self.assertEqual(self.catched_msg['subject'], subject)
- self.assertEqual(self.catched_msg['body'], body)
+ self.assertEqual(self.catched_msg["subject"], subject)
+ self.assertEqual(self.catched_msg["body"], body)
- msg = self.catched_msg['msg']
- self.assertEqual(msg['subject'], subject)
- self.assertEqual(msg.get_payload(), body)
- self.assertEqual(msg.get_charset(), Charset('utf-8'))
- self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
+ msg = self.catched_msg["msg"]
+ self.assertEqual(msg["subject"], subject)
+ self.assertEqual(msg.get_payload(decode=True).decode("utf-8"), body)
+ self.assertEqual(msg.get_charset(), Charset("utf-8"))
+ self.assertEqual(msg.get("Content-Type"), 'text/plain; charset="utf-8"')
def test_send_attach_utf8(self):
- subject = 'sübjèçt'
- body = 'bödÿ-àéïöñß'
+ subject = "sübjèçt"
+ body = "bödÿ-àéïöñß"
attach = BytesIO()
- attach.write(body.encode('utf-8'))
+ attach.write(body.encode("utf-8"))
attach.seek(0)
- attachs = [('attachment', 'text/plain', attach)]
+ attachs = [("attachment", "text/plain", attach)]
mailsender = MailSender(debug=True)
- mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
- attachs=attachs, charset='utf-8',
- _callback=self._catch_mail_sent)
+ mailsender.send(
+ to=["test@scrapy.org"],
+ subject=subject,
+ body=body,
+ attachs=attachs,
+ charset="utf-8",
+ _callback=self._catch_mail_sent,
+ )
assert self.catched_msg
- self.assertEqual(self.catched_msg['subject'], subject)
- self.assertEqual(self.catched_msg['body'], body)
+ self.assertEqual(self.catched_msg["subject"], subject)
+ self.assertEqual(self.catched_msg["body"], body)
- msg = self.catched_msg['msg']
- self.assertEqual(msg['subject'], subject)
- self.assertEqual(msg.get_charset(), Charset('utf-8'))
- self.assertEqual(msg.get('Content-Type'),
- 'multipart/mixed; charset="utf-8"')
+ msg = self.catched_msg["msg"]
+ self.assertEqual(msg["subject"], subject)
+ self.assertEqual(msg.get_charset(), Charset("utf-8"))
+ self.assertEqual(msg.get("Content-Type"), 'multipart/mixed; charset="utf-8"')
payload = msg.get_payload()
assert isinstance(payload, list)
self.assertEqual(len(payload), 2)
text, attach = payload
- self.assertEqual(text.get_payload(decode=True).decode('utf-8'), body)
- self.assertEqual(text.get_charset(), Charset('utf-8'))
- self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
+ self.assertEqual(text.get_payload(decode=True).decode("utf-8"), body)
+ self.assertEqual(text.get_charset(), Charset("utf-8"))
+ self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body)
+
+ def test_create_sender_factory_with_host(self):
+ mailsender = MailSender(debug=False, smtphost="smtp.testhost.com")
+
+ factory = mailsender._create_sender_factory(
+ to_addrs=["test@scrapy.org"], msg="test", d=defer.Deferred()
+ )
+
+ context = factory.buildProtocol("test@scrapy.org").context
+ if twisted_version >= Version("twisted", 21, 2, 0):
+ self.assertIsInstance(context, ClientTLSOptions)
+ else:
+ self.assertIsInstance(context, ClientContextFactory)
if __name__ == "__main__":
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index 8651431b5..a42c7b3d1 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -1,12 +1,11 @@
from twisted.trial import unittest
-from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured
from scrapy.middleware import MiddlewareManager
+from scrapy.settings import Settings
class M1:
-
def open_spider(self, spider):
pass
@@ -18,7 +17,6 @@ class M1:
class M2:
-
def open_spider(self, spider):
pass
@@ -29,13 +27,11 @@ class M2:
class M3:
-
def process(self, response, request, spider):
pass
class MOff:
-
def open_spider(self, spider):
pass
@@ -43,41 +39,43 @@ class MOff:
pass
def __init__(self):
- raise NotConfigured
+ raise NotConfigured("foo")
class TestMiddlewareManager(MiddlewareManager):
-
@classmethod
def _get_mwlist_from_settings(cls, settings):
return [M1, MOff, M3]
def _add_middleware(self, mw):
super()._add_middleware(mw)
- if hasattr(mw, 'process'):
- self.methods['process'].append(mw.process)
+ if hasattr(mw, "process"):
+ self.methods["process"].append(mw.process)
class MiddlewareManagerTest(unittest.TestCase):
-
def test_init(self):
m1, m2, m3 = M1(), M2(), M3()
mwman = TestMiddlewareManager(m1, m2, m3)
- self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider])
- self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider])
- self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process])
+ self.assertEqual(
+ list(mwman.methods["open_spider"]), [m1.open_spider, m2.open_spider]
+ )
+ self.assertEqual(
+ list(mwman.methods["close_spider"]), [m2.close_spider, m1.close_spider]
+ )
+ self.assertEqual(list(mwman.methods["process"]), [m1.process, m3.process])
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())
self.assertEqual(
- [x.__self__.__class__ for x in mwman.methods['open_spider']],
- [M1, M2])
+ [x.__self__.__class__ for x in mwman.methods["open_spider"]], [M1, M2]
+ )
self.assertEqual(
- [x.__self__.__class__ for x in mwman.methods['close_spider']],
- [M2, M1])
+ [x.__self__.__class__ for x in mwman.methods["close_spider"]], [M2, M1]
+ )
self.assertEqual(
- [x.__self__.__class__ for x in mwman.methods['process']],
- [M1, M3])
+ [x.__self__.__class__ for x in mwman.methods["process"]], [M1, M3]
+ )
def test_enabled(self):
m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 39bca7bf2..c41ab483f 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -1,19 +1,21 @@
import shutil
from pathlib import Path
+from typing import Optional, Set
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from w3lib.url import add_or_replace_parameter
-from scrapy.crawler import CrawlerRunner
from scrapy import signals
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.misc import load_object
from tests.mockserver import MockServer
from tests.spiders import SimpleSpider
class MediaDownloadSpider(SimpleSpider):
- name = 'mediadownload'
+ name = "mediadownload"
def _process_url(self, url):
return url
@@ -27,34 +29,38 @@ class MediaDownloadSpider(SimpleSpider):
self._process_url(response.urljoin(href))
for href in response.xpath(
'//table[thead/tr/th="Filename"]/tbody//a/@href'
- ).getall()],
+ ).getall()
+ ],
}
yield item
class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):
- name = 'brokenmedia'
+ name = "brokenmedia"
def _process_url(self, url):
- return url + '.foo'
+ return url + ".foo"
class RedirectedMediaDownloadSpider(MediaDownloadSpider):
- name = 'redirectedmedia'
+ name = "redirectedmedia"
def _process_url(self, url):
- return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url)
+ return add_or_replace_parameter(
+ self.mockserver.url("/redirect-to"), "goto", url
+ )
class FileDownloadCrawlTestCase(TestCase):
- pipeline_class = 'scrapy.pipelines.files.FilesPipeline'
- store_setting_key = 'FILES_STORE'
- media_key = 'files'
- media_urls_key = 'file_urls'
- expected_checksums = {
- '5547178b89448faf0015a13f904c936e',
- 'c2281c83670e31d8aaab7cb642b824db',
- 'ed3f6538dc15d4d9179dae57319edc5f'}
+ pipeline_class = "scrapy.pipelines.files.FilesPipeline"
+ store_setting_key = "FILES_STORE"
+ media_key = "files"
+ media_urls_key = "file_urls"
+ expected_checksums: Optional[Set[str]] = {
+ "5547178b89448faf0015a13f904c936e",
+ "c2281c83670e31d8aaab7cb642b824db",
+ "ed3f6538dc15d4d9179dae57319edc5f",
+ }
def setUp(self):
self.mockserver = MockServer()
@@ -64,8 +70,8 @@ class FileDownloadCrawlTestCase(TestCase):
self.tmpmediastore = Path(self.mktemp())
self.tmpmediastore.mkdir()
self.settings = {
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
- 'ITEM_PIPELINES': {self.pipeline_class: 1},
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+ "ITEM_PIPELINES": {self.pipeline_class: 1},
self.store_setting_key: str(self.tmpmediastore),
}
self.runner = CrawlerRunner(self.settings)
@@ -91,43 +97,46 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertIn(self.media_key, items[0])
# check that logs show the expected number of successful file downloads
- file_dl_success = 'File (downloaded): Downloaded file from'
+ file_dl_success = "File (downloaded): Downloaded file from"
self.assertEqual(logs.count(file_dl_success), 3)
# check that the images/files status is `downloaded`
for item in items:
for i in item[self.media_key]:
- self.assertEqual(i['status'], 'downloaded')
+ self.assertEqual(i["status"], "downloaded")
# check that the images/files checksums are what we know they should be
if self.expected_checksums is not None:
checksums = set(
- i['checksum']
- for item in items
- for i in item[self.media_key]
+ i["checksum"] for item in items for i in item[self.media_key]
)
self.assertEqual(checksums, self.expected_checksums)
# check that the image files where actually written to the media store
for item in items:
for i in item[self.media_key]:
- self.assertTrue((self.tmpmediastore / i['path']).exists())
+ self.assertTrue((self.tmpmediastore / i["path"]).exists())
def _assert_files_download_failure(self, crawler, items, code, logs):
-
# check that the item does NOT have the "images/files" field populated
self.assertEqual(len(items), 1)
self.assertIn(self.media_key, items[0])
self.assertFalse(items[0][self.media_key])
# check that there was 1 successful fetch and 3 other responses with non-200 code
- self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4)
- self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4)
- self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1)
- self.assertEqual(crawler.stats.get_value(f'downloader/response_status_count/{code}'), 3)
+ self.assertEqual(
+ crawler.stats.get_value("downloader/request_method_count/GET"), 4
+ )
+ self.assertEqual(crawler.stats.get_value("downloader/response_count"), 4)
+ self.assertEqual(
+ crawler.stats.get_value("downloader/response_status_count/200"), 1
+ )
+ self.assertEqual(
+ crawler.stats.get_value(f"downloader/response_status_count/{code}"), 3
+ )
# check that logs do show the failure on the file downloads
- file_dl_failure = f'File (code: {code}): Error downloading file from'
+ file_dl_failure = f"File (code: {code}): Error downloading file from"
self.assertEqual(logs.count(file_dl_failure), 3)
# check that no files were written to the media store
@@ -140,7 +149,8 @@ class FileDownloadCrawlTestCase(TestCase):
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
- media_urls_key=self.media_urls_key)
+ media_urls_key=self.media_urls_key,
+ )
self._assert_files_downloaded(self.items, str(log))
@defer.inlineCallbacks
@@ -150,7 +160,8 @@ class FileDownloadCrawlTestCase(TestCase):
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
- media_urls_key=self.media_urls_key)
+ media_urls_key=self.media_urls_key,
+ )
self._assert_files_download_failure(crawler, self.items, 404, str(log))
@defer.inlineCallbacks
@@ -161,13 +172,14 @@ class FileDownloadCrawlTestCase(TestCase):
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
- mockserver=self.mockserver)
+ mockserver=self.mockserver,
+ )
self._assert_files_download_failure(crawler, self.items, 302, str(log))
@defer.inlineCallbacks
def test_download_media_redirected_allowed(self):
settings = dict(self.settings)
- settings.update({'MEDIA_ALLOW_REDIRECTS': True})
+ settings.update({"MEDIA_ALLOW_REDIRECTS": True})
runner = CrawlerRunner(settings)
crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
with LogCapture() as log:
@@ -175,27 +187,55 @@ class FileDownloadCrawlTestCase(TestCase):
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
- mockserver=self.mockserver)
+ mockserver=self.mockserver,
+ )
self._assert_files_downloaded(self.items, str(log))
- self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
+ self.assertEqual(
+ crawler.stats.get_value("downloader/response_status_count/302"), 3
+ )
+
+ @defer.inlineCallbacks
+ def test_download_media_file_path_error(self):
+ cls = load_object(self.pipeline_class)
+
+ class ExceptionRaisingMediaPipeline(cls):
+ def file_path(self, request, response=None, info=None, *, item=None):
+ return 1 / 0
+
+ settings = {
+ **self.settings,
+ "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
+ }
+ runner = CrawlerRunner(settings)
+ crawler = self._create_crawler(MediaDownloadSpider, runner=runner)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ self.mockserver.url("/files/images/"),
+ media_key=self.media_key,
+ media_urls_key=self.media_urls_key,
+ mockserver=self.mockserver,
+ )
+ self.assertIn("ZeroDivisionError", str(log))
+skip_pillow: Optional[str]
try:
from PIL import Image # noqa: imported just to check for the import error
except ImportError:
- skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+ skip_pillow = (
+ "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+ )
else:
skip_pillow = None
class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
-
skip = skip_pillow
- pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
- store_setting_key = 'IMAGES_STORE'
- media_key = 'images'
- media_urls_key = 'image_urls'
+ pipeline_class = "scrapy.pipelines.images.ImagesPipeline"
+ store_setting_key = "IMAGES_STORE"
+ media_key = "images"
+ media_urls_key = "image_urls"
# somehow checksums for images are different for Python 3.3
expected_checksums = None
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4acd29bf7..e7000e314 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,3 +1,4 @@
+import dataclasses
import os
import random
import time
@@ -6,9 +7,9 @@ from io import BytesIO
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
+from typing import Dict, List
from unittest import mock
from urllib.parse import urlparse
-import dataclasses
import attr
from itemadapter import ItemAdapter
@@ -32,18 +33,15 @@ from scrapy.utils.test import (
get_gcs_content_and_delete,
skip_if_no_boto,
)
+from tests.mockserver import MockFTPServer
-
-def _mocked_download_func(request, info):
- response = request.meta.get('response')
- return response() if callable(response) else response
+from .test_pipeline_media import _mocked_download_func
class FilesPipelineTestCase(unittest.TestCase):
-
def setUp(self):
self.tempdir = mkdtemp()
- settings_dict = {'FILES_STORE': self.tempdir}
+ settings_dict = {"FILES_STORE": self.tempdir}
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func
@@ -56,41 +54,70 @@ class FilesPipelineTestCase(unittest.TestCase):
file_path = self.pipeline.file_path
self.assertEqual(
file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
- 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
+ "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf",
+ )
self.assertEqual(
- file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt")),
- 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
+ file_path(
+ Request(
+ "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"
+ )
+ ),
+ "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt",
+ )
self.assertEqual(
- file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")),
- 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
+ file_path(
+ Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")
+ ),
+ "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc",
+ )
self.assertEqual(
- file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
- 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+ file_path(
+ Request(
+ "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"
+ )
+ ),
+ "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg",
+ )
self.assertEqual(
file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
- 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
+ "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2",
+ )
self.assertEqual(
file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
- 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+ "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1",
+ )
self.assertEqual(
- file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
- response=Response("http://www.dorma.co.uk/images/product_details/2532"),
- info=object()),
- 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+ file_path(
+ Request("http://www.dorma.co.uk/images/product_details/2532"),
+ response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+ info=object(),
+ ),
+ "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1",
+ )
self.assertEqual(
- file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
- 'full/76c00cef2ef669ae65052661f68d451162829507')
+ file_path(
+ Request(
+ "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha"
+ )
+ ),
+ "full/76c00cef2ef669ae65052661f68d451162829507",
+ )
self.assertEqual(
- file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
- //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
- 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+ file_path(
+ Request(
+ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y"
+ )
+ ),
+ "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png",
+ )
def test_fs_store(self):
assert isinstance(self.pipeline.store, FSFilesStore)
self.assertEqual(self.pipeline.store.basedir, self.tempdir)
- path = 'some/image/key.jpg'
- fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg')
+ path = "some/image/key.jpg"
+ fullpath = Path(self.tempdir, "some", "image", "key.jpg")
self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath)
@defer.inlineCallbacks
@@ -98,18 +125,24 @@ class FilesPipelineTestCase(unittest.TestCase):
item_url = "http://example.com/file.pdf"
item = _create_item_with_files(item_url)
patchers = [
- mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
- mock.patch.object(FSFilesStore, 'stat_file', return_value={
- 'checksum': 'abc', 'last_modified': time.time()}),
- mock.patch.object(FilesPipeline, 'get_media_requests',
- return_value=[_prepare_request_object(item_url)])
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(
+ FSFilesStore,
+ "stat_file",
+ return_value={"checksum": "abc", "last_modified": time.time()},
+ ),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
]
for p in patchers:
p.start()
result = yield self.pipeline.process_item(item, None)
- self.assertEqual(result['files'][0]['checksum'], 'abc')
- self.assertEqual(result['files'][0]['status'], 'uptodate')
+ self.assertEqual(result["files"][0]["checksum"], "abc")
+ self.assertEqual(result["files"][0]["status"], "uptodate")
for p in patchers:
p.stop()
@@ -119,19 +152,28 @@ class FilesPipelineTestCase(unittest.TestCase):
item_url = "http://example.com/file2.pdf"
item = _create_item_with_files(item_url)
patchers = [
- mock.patch.object(FSFilesStore, 'stat_file', return_value={
- 'checksum': 'abc',
- 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
- mock.patch.object(FilesPipeline, 'get_media_requests',
- return_value=[_prepare_request_object(item_url)]),
- mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)
+ mock.patch.object(
+ FSFilesStore,
+ "stat_file",
+ return_value={
+ "checksum": "abc",
+ "last_modified": time.time()
+ - (self.pipeline.expires * 60 * 60 * 24 * 2),
+ },
+ ),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
]
for p in patchers:
p.start()
result = yield self.pipeline.process_item(item, None)
- self.assertNotEqual(result['files'][0]['checksum'], 'abc')
- self.assertEqual(result['files'][0]['status'], 'downloaded')
+ self.assertNotEqual(result["files"][0]["checksum"], "abc")
+ self.assertEqual(result["files"][0]["status"], "downloaded")
for p in patchers:
p.stop()
@@ -141,19 +183,28 @@ class FilesPipelineTestCase(unittest.TestCase):
item_url = "http://example.com/file3.pdf"
item = _create_item_with_files(item_url)
patchers = [
- mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
- mock.patch.object(FSFilesStore, 'stat_file', return_value={
- 'checksum': 'abc',
- 'last_modified': time.time() - (self.pipeline.expires * 60 * 60 * 24 * 2)}),
- mock.patch.object(FilesPipeline, 'get_media_requests',
- return_value=[_prepare_request_object(item_url, flags=['cached'])])
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(
+ FSFilesStore,
+ "stat_file",
+ return_value={
+ "checksum": "abc",
+ "last_modified": time.time()
+ - (self.pipeline.expires * 60 * 60 * 24 * 2),
+ },
+ ),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url, flags=["cached"])],
+ ),
]
for p in patchers:
p.start()
result = yield self.pipeline.process_item(item, None)
- self.assertNotEqual(result['files'][0]['checksum'], 'abc')
- self.assertEqual(result['files'][0]['status'], 'cached')
+ self.assertNotEqual(result["files"][0]["checksum"], "abc")
+ self.assertEqual(result["files"][0]["status"], "cached")
for p in patchers:
p.stop()
@@ -162,48 +213,62 @@ class FilesPipelineTestCase(unittest.TestCase):
"""
Custom file path based on item data, overriding default implementation
"""
+
class CustomFilesPipeline(FilesPipeline):
def file_path(self, request, response=None, info=None, item=None):
return f'full/{item.get("path")}'
- file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path
- item = dict(path='path-to-store-file')
+ file_path = CustomFilesPipeline.from_settings(
+ Settings({"FILES_STORE": self.tempdir})
+ ).file_path
+ item = dict(path="path-to-store-file")
request = Request("http://example.com")
- self.assertEqual(file_path(request, item=item), 'full/path-to-store-file')
+ self.assertEqual(file_path(request, item=item), "full/path-to-store-file")
class FilesPipelineTestCaseFieldsMixin:
+ def setUp(self):
+ self.tempdir = mkdtemp()
+
+ def tearDown(self):
+ rmtree(self.tempdir)
def test_item_fields_default(self):
- url = 'http://www.example.com/files/1.txt'
- item = self.item_class(name='item1', file_urls=[url])
- pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': 's3://example/files/'}))
+ url = "http://www.example.com/files/1.txt"
+ item = self.item_class(name="item1", file_urls=[url])
+ pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
- results = [(True, {'url': url})]
+ results = [(True, {"url": url})]
item = pipeline.item_completed(results, item, None)
files = ItemAdapter(item).get("files")
self.assertEqual(files, [results[0][1]])
self.assertIsInstance(item, self.item_class)
def test_item_fields_override_settings(self):
- url = 'http://www.example.com/files/1.txt'
- item = self.item_class(name='item1', custom_file_urls=[url])
- pipeline = FilesPipeline.from_settings(Settings({
- 'FILES_STORE': 's3://example/files/',
- 'FILES_URLS_FIELD': 'custom_file_urls',
- 'FILES_RESULT_FIELD': 'custom_files'
- }))
+ url = "http://www.example.com/files/1.txt"
+ item = self.item_class(name="item1", custom_file_urls=[url])
+ pipeline = FilesPipeline.from_settings(
+ Settings(
+ {
+ "FILES_STORE": self.tempdir,
+ "FILES_URLS_FIELD": "custom_file_urls",
+ "FILES_RESULT_FIELD": "custom_files",
+ }
+ )
+ )
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
- results = [(True, {'url': url})]
+ results = [(True, {"url": url})]
item = pipeline.item_completed(results, item, None)
custom_files = ItemAdapter(item).get("custom_files")
self.assertEqual(custom_files, [results[0][1]])
self.assertIsInstance(item, self.item_class)
-class FilesPipelineTestCaseFieldsDict(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsDict(
+ FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = dict
@@ -217,7 +282,9 @@ class FilesPipelineTestItem(Item):
custom_files = Field()
-class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsItem(
+ FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = FilesPipelineTestItem
@@ -232,7 +299,9 @@ class FilesPipelineTestDataClass:
custom_files: list = dataclasses.field(default_factory=list)
-class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsDataClass(
+ FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = FilesPipelineTestDataClass
@@ -240,14 +309,16 @@ class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, uni
class FilesPipelineTestAttrsItem:
name = attr.ib(default="")
# default fields
- file_urls = attr.ib(default=lambda: [])
- files = attr.ib(default=lambda: [])
+ file_urls: List[str] = attr.ib(default=lambda: [])
+ files: List[Dict[str, str]] = attr.ib(default=lambda: [])
# overridden fields
- custom_file_urls = attr.ib(default=lambda: [])
- custom_files = attr.ib(default=lambda: [])
+ custom_file_urls: List[str] = attr.ib(default=lambda: [])
+ custom_files: List[Dict[str, str]] = attr.ib(default=lambda: [])
-class FilesPipelineTestCaseFieldsAttrsItem(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class FilesPipelineTestCaseFieldsAttrsItem(
+ FilesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = FilesPipelineTestAttrsItem
@@ -255,12 +326,12 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
default_cls_settings = {
"EXPIRES": 90,
"FILES_URLS_FIELD": "file_urls",
- "FILES_RESULT_FIELD": "files"
+ "FILES_RESULT_FIELD": "files",
}
file_cls_attr_settings_map = {
("EXPIRES", "FILES_EXPIRES", "expires"),
("FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"),
- ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field")
+ ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),
}
def setUp(self):
@@ -270,7 +341,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
rmtree(self.tempdir)
def _generate_fake_settings(self, prefix=None):
-
def random_string():
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
@@ -278,15 +348,17 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
"FILES_EXPIRES": random.randint(100, 1000),
"FILES_URLS_FIELD": random_string(),
"FILES_RESULT_FIELD": random_string(),
- "FILES_STORE": self.tempdir
+ "FILES_STORE": self.tempdir,
}
if not prefix:
return settings
- return {prefix.upper() + "_" + k if k != "FILES_STORE" else k: v for k, v in settings.items()}
+ return {
+ prefix.upper() + "_" + k if k != "FILES_STORE" else k: v
+ for k, v in settings.items()
+ }
def _generate_fake_pipeline(self):
-
class UserDefinedFilePipeline(FilesPipeline):
EXPIRES = 1001
FILES_URLS_FIELD = "alfa"
@@ -339,10 +411,13 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
If there are no settings for subclass and no subclass attributes, pipeline should use
attributes of base class.
"""
+
class UserDefinedFilesPipeline(FilesPipeline):
pass
- user_pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
+ user_pipeline = UserDefinedFilesPipeline.from_settings(
+ Settings({"FILES_STORE": self.tempdir})
+ )
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_cls_settings.get(pipe_attr.upper())
@@ -353,6 +428,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
If there are custom settings for subclass and NO class attributes, pipeline should use custom
settings.
"""
+
class UserDefinedFilesPipeline(FilesPipeline):
pass
@@ -374,7 +450,11 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(prefix=prefix)
user_pipeline = pipeline_cls.from_settings(Settings(settings))
- for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
+ for (
+ pipe_cls_attr,
+ settings_attr,
+ pipe_inst_attr,
+ ) in self.file_cls_attr_settings_map:
custom_value = settings.get(prefix + "_" + settings_attr)
self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr])
self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
@@ -384,9 +464,16 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
DEFAULT_FILES_RESULT_FIELD = "this"
DEFAULT_FILES_URLS_FIELD = "that"
- pipeline = UserDefinedFilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
- self.assertEqual(pipeline.files_result_field, "this")
- self.assertEqual(pipeline.files_urls_field, "that")
+ pipeline = UserDefinedFilesPipeline.from_settings(
+ Settings({"FILES_STORE": self.tempdir})
+ )
+ self.assertEqual(
+ pipeline.files_result_field,
+ UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD,
+ )
+ self.assertEqual(
+ pipeline.files_urls_field, UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD
+ )
def test_user_defined_subclass_default_key_names(self):
"""Test situation when user defines subclass of FilesPipeline,
@@ -402,37 +489,52 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
expected_value = settings.get(settings_attr)
- self.assertEqual(getattr(pipeline_cls, pipe_inst_attr),
- expected_value)
+ self.assertEqual(getattr(pipeline_cls, pipe_inst_attr), expected_value)
+
+ def test_file_pipeline_using_pathlike_objects(self):
+ class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
+ def file_path(self, request, response=None, info=None, *, item=None):
+ return Path("subdir") / Path(request.url).name
+
+ pipeline = CustomFilesPipelineWithPathLikeDir.from_settings(
+ Settings({"FILES_STORE": Path("./Temp")})
+ )
+ request = Request("http://example.com/image01.jpg")
+ self.assertEqual(pipeline.file_path(request), Path("subdir/image01.jpg"))
+
+ def test_files_store_constructor_with_pathlike_object(self):
+ path = Path("./FileDir")
+ fs_store = FSFilesStore(path)
+ self.assertEqual(fs_store.basedir, str(path))
class TestS3FilesStore(unittest.TestCase):
-
@defer.inlineCallbacks
def test_persist(self):
skip_if_no_boto()
- bucket = 'mybucket'
- key = 'export.csv'
- uri = f's3://{bucket}/{key}'
+ bucket = "mybucket"
+ key = "export.csv"
+ uri = f"s3://{bucket}/{key}"
buffer = mock.MagicMock()
- meta = {'foo': 'bar'}
- path = ''
- content_type = 'image/png'
+ meta = {"foo": "bar"}
+ path = ""
+ content_type = "image/png"
store = S3FilesStore(uri)
from botocore.stub import Stubber
+
with Stubber(store.s3_client) as stub:
stub.add_response(
- 'put_object',
+ "put_object",
expected_params={
- 'ACL': S3FilesStore.POLICY,
- 'Body': buffer,
- 'Bucket': bucket,
- 'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
- 'ContentType': content_type,
- 'Key': key,
- 'Metadata': meta,
+ "ACL": S3FilesStore.POLICY,
+ "Body": buffer,
+ "Bucket": bucket,
+ "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+ "ContentType": content_type,
+ "Key": key,
+ "Metadata": meta,
},
service_response={},
)
@@ -442,7 +544,7 @@ class TestS3FilesStore(unittest.TestCase):
buffer,
info=None,
meta=meta,
- headers={'Content-Type': content_type},
+ headers={"Content-Type": content_type},
)
stub.assert_no_pending_responses()
@@ -451,40 +553,41 @@ class TestS3FilesStore(unittest.TestCase):
[
mock.call.seek(0),
# The call to read does not happen with Stubber
- ]
+ ],
)
@defer.inlineCallbacks
def test_stat(self):
skip_if_no_boto()
- bucket = 'mybucket'
- key = 'export.csv'
- uri = f's3://{bucket}/{key}'
- checksum = '3187896a9657a28163abb31667df64c8'
+ bucket = "mybucket"
+ key = "export.csv"
+ uri = f"s3://{bucket}/{key}"
+ checksum = "3187896a9657a28163abb31667df64c8"
last_modified = datetime(2019, 12, 1)
store = S3FilesStore(uri)
from botocore.stub import Stubber
+
with Stubber(store.s3_client) as stub:
stub.add_response(
- 'head_object',
+ "head_object",
expected_params={
- 'Bucket': bucket,
- 'Key': key,
+ "Bucket": bucket,
+ "Key": key,
},
service_response={
- 'ETag': f'"{checksum}"',
- 'LastModified': last_modified,
+ "ETag": f'"{checksum}"',
+ "LastModified": last_modified,
},
)
- file_stats = yield store.stat_file('', info=None)
+ file_stats = yield store.stat_file("", info=None)
self.assertEqual(
file_stats,
{
- 'checksum': checksum,
- 'last_modified': last_modified.timestamp(),
+ "checksum": checksum,
+ "last_modified": last_modified.timestamp(),
},
)
@@ -495,27 +598,27 @@ class TestGCSFilesStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
assert_gcs_environ()
- uri = os.environ.get('GCS_TEST_FILE_URI')
+ uri = os.environ.get("GCS_TEST_FILE_URI")
if not uri:
raise unittest.SkipTest("No GCS URI available for testing")
data = b"TestGCSFilesStore: \xe2\x98\x83"
buf = BytesIO(data)
- meta = {'foo': 'bar'}
- path = 'full/filename'
+ meta = {"foo": "bar"}
+ path = "full/filename"
store = GCSFilesStore(uri)
- store.POLICY = 'authenticatedRead'
- expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'}
+ store.POLICY = "authenticatedRead"
+ expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"}
yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
s = yield store.stat_file(path, info=None)
- self.assertIn('last_modified', s)
- self.assertIn('checksum', s)
- self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
+ self.assertIn("last_modified", s)
+ self.assertIn("checksum", s)
+ self.assertEqual(s["checksum"], "cdcda85605e46d0af6110752770dce3c")
u = urlparse(uri)
content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path)
self.assertEqual(content, data)
- self.assertEqual(blob.metadata, {'foo': 'bar'})
+ self.assertEqual(blob.metadata, {"foo": "bar"})
self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
- self.assertEqual(blob.content_type, 'application/octet-stream')
+ self.assertEqual(blob.content_type, "application/octet-stream")
self.assertIn(expected_policy, acl)
@defer.inlineCallbacks
@@ -525,17 +628,19 @@ class TestGCSFilesStore(unittest.TestCase):
"""
assert_gcs_environ()
try:
- import google.cloud.storage # noqa
+ import google.cloud.storage # noqa
except ModuleNotFoundError:
raise unittest.SkipTest("google-cloud-storage is not installed")
else:
- with mock.patch('google.cloud.storage') as _:
- with mock.patch('scrapy.pipelines.files.time') as _:
- uri = 'gs://my_bucket/my_prefix/'
+ with mock.patch("google.cloud.storage") as _:
+ with mock.patch("scrapy.pipelines.files.time") as _:
+ uri = "gs://my_bucket/my_prefix/"
store = GCSFilesStore(uri)
store.bucket = mock.Mock()
- path = 'full/my_data.txt'
- yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None)
+ path = "full/my_data.txt"
+ yield store.persist_file(
+ path, mock.Mock(), info=None, meta=None, headers=None
+ )
yield store.stat_file(path, info=None)
expected_blob_path = store.prefix + path
store.bucket.blob.assert_called_with(expected_blob_path)
@@ -545,26 +650,29 @@ class TestGCSFilesStore(unittest.TestCase):
class TestFTPFileStore(unittest.TestCase):
@defer.inlineCallbacks
def test_persist(self):
- uri = os.environ.get('FTP_TEST_FILE_URI')
- if not uri:
- raise unittest.SkipTest("No FTP URI available for testing")
data = b"TestFTPFilesStore: \xe2\x98\x83"
buf = BytesIO(data)
- meta = {'foo': 'bar'}
- path = 'full/filename'
- store = FTPFilesStore(uri)
- empty_dict = yield store.stat_file(path, info=None)
- self.assertEqual(empty_dict, {})
- yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
- stat = yield store.stat_file(path, info=None)
- self.assertIn('last_modified', stat)
- self.assertIn('checksum', stat)
- self.assertEqual(stat['checksum'], 'd113d66b2ec7258724a268bd88eef6b6')
- path = f'{store.basedir}/{path}'
- content = get_ftp_content_and_delete(
- path, store.host, store.port,
- store.username, store.password, store.USE_ACTIVE_MODE)
- self.assertEqual(data.decode(), content)
+ meta = {"foo": "bar"}
+ path = "full/filename"
+ with MockFTPServer() as ftp_server:
+ store = FTPFilesStore(ftp_server.url("/"))
+ empty_dict = yield store.stat_file(path, info=None)
+ self.assertEqual(empty_dict, {})
+ yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
+ stat = yield store.stat_file(path, info=None)
+ self.assertIn("last_modified", stat)
+ self.assertIn("checksum", stat)
+ self.assertEqual(stat["checksum"], "d113d66b2ec7258724a268bd88eef6b6")
+ path = f"{store.basedir}/{path}"
+ content = get_ftp_content_and_delete(
+ path,
+ store.host,
+ store.port,
+ store.username,
+ store.password,
+ store.USE_ACTIVE_MODE,
+ )
+ self.assertEqual(data, content)
class ItemWithFiles(Item):
@@ -574,15 +682,12 @@ class ItemWithFiles(Item):
def _create_item_with_files(*files):
item = ItemWithFiles()
- item['file_urls'] = files
+ item["file_urls"] = files
return item
def _prepare_request_object(item_url, flags=None):
return Request(
item_url,
- meta={'response': Response(item_url, status=200, body=b'data', flags=flags)})
-
-
-if __name__ == "__main__":
- unittest.main()
+ meta={"response": Response(item_url, status=200, body=b"data", flags=flags)},
+ )
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index f98d40fda..2e2e06b89 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -5,6 +5,7 @@ import random
import warnings
from shutil import rmtree
from tempfile import mkdtemp
+from typing import Dict, List, Optional
from unittest.mock import patch
import attr
@@ -18,31 +19,27 @@ from scrapy.pipelines.images import ImageException, ImagesPipeline, NoimagesDrop
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
-
+skip_pillow: Optional[str]
try:
from PIL import Image
except ImportError:
- skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+ skip_pillow = (
+ "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+ )
else:
- encoders = {'jpeg_encoder', 'jpeg_decoder'}
- if not encoders.issubset(set(Image.core.__dict__)):
- skip_pillow = 'Missing JPEG encoders'
+ encoders = {"jpeg_encoder", "jpeg_decoder"}
+ if not encoders.issubset(set(Image.core.__dict__)): # type: ignore[attr-defined]
+ skip_pillow = "Missing JPEG encoders"
else:
skip_pillow = None
-def _mocked_download_func(request, info):
- response = request.meta.get('response')
- return response() if callable(response) else response
-
-
class ImagesPipelineTestCase(unittest.TestCase):
-
skip = skip_pillow
def setUp(self):
self.tempdir = mkdtemp()
- self.pipeline = ImagesPipeline(self.tempdir, download_func=_mocked_download_func)
+ self.pipeline = ImagesPipeline(self.tempdir)
def tearDown(self):
rmtree(self.tempdir)
@@ -51,43 +48,75 @@ class ImagesPipelineTestCase(unittest.TestCase):
file_path = self.pipeline.file_path
self.assertEqual(
file_path(Request("https://dev.mydeco.com/mydeco.gif")),
- 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+ "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
+ )
self.assertEqual(
- file_path(Request("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg")),
- 'full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg')
+ file_path(
+ Request(
+ "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"
+ )
+ ),
+ "full/0ffcd85d563bca45e2f90becd0ca737bc58a00b2.jpg",
+ )
self.assertEqual(
- file_path(Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")),
- 'full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg')
+ file_path(
+ Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.gif")
+ ),
+ "full/b250e3a74fff2e4703e310048a5b13eba79379d2.jpg",
+ )
self.assertEqual(
- file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg")),
- 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+ file_path(
+ Request(
+ "http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"
+ )
+ ),
+ "full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg",
+ )
self.assertEqual(
file_path(Request("http://www.dorma.co.uk/images/product_details/2532/")),
- 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg')
+ "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2.jpg",
+ )
self.assertEqual(
file_path(Request("http://www.dorma.co.uk/images/product_details/2532")),
- 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+ "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg",
+ )
self.assertEqual(
- file_path(Request("http://www.dorma.co.uk/images/product_details/2532"),
- response=Response("http://www.dorma.co.uk/images/product_details/2532"),
- info=object()),
- 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg')
+ file_path(
+ Request("http://www.dorma.co.uk/images/product_details/2532"),
+ response=Response("http://www.dorma.co.uk/images/product_details/2532"),
+ info=object(),
+ ),
+ "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg",
+ )
def test_thumbnail_name(self):
thumb_path = self.pipeline.thumb_path
- name = '50'
- self.assertEqual(thumb_path(Request("file:///tmp/foo.jpg"), name),
- 'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
- self.assertEqual(thumb_path(Request("file://foo.png"), name),
- 'thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg')
- self.assertEqual(thumb_path(Request("file:///tmp/foo"), name),
- 'thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg')
- self.assertEqual(thumb_path(Request("file:///tmp/some.name/foo"), name),
- 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
- self.assertEqual(thumb_path(Request("file:///tmp/some.name/foo"), name,
- response=Response("file:///tmp/some.name/foo"),
- info=object()),
- 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
+ name = "50"
+ self.assertEqual(
+ thumb_path(Request("file:///tmp/foo.jpg"), name),
+ "thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg",
+ )
+ self.assertEqual(
+ thumb_path(Request("file://foo.png"), name),
+ "thumbs/50/e55b765eba0ec7348e50a1df496040449071b96a.jpg",
+ )
+ self.assertEqual(
+ thumb_path(Request("file:///tmp/foo"), name),
+ "thumbs/50/0329ad83ebb8e93ea7c7906d46e9ed55f7349a50.jpg",
+ )
+ self.assertEqual(
+ thumb_path(Request("file:///tmp/some.name/foo"), name),
+ "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg",
+ )
+ self.assertEqual(
+ thumb_path(
+ Request("file:///tmp/some.name/foo"),
+ name,
+ response=Response("file:///tmp/some.name/foo"),
+ info=object(),
+ ),
+ "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg",
+ )
def test_thumbnail_name_from_item(self):
"""
@@ -95,23 +124,27 @@ class ImagesPipelineTestCase(unittest.TestCase):
"""
class CustomImagesPipeline(ImagesPipeline):
- def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+ def thumb_path(
+ self, request, thumb_id, response=None, info=None, item=None
+ ):
return f"thumb/{thumb_id}/{item.get('path')}"
- thumb_path = CustomImagesPipeline.from_settings(Settings(
- {'IMAGES_STORE': self.tempdir}
- )).thumb_path
- item = dict(path='path-to-store-file')
+ thumb_path = CustomImagesPipeline.from_settings(
+ Settings({"IMAGES_STORE": self.tempdir})
+ ).thumb_path
+ item = dict(path="path-to-store-file")
request = Request("http://example.com")
- self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
+ self.assertEqual(
+ thumb_path(request, "small", item=item), "thumb/small/path-to-store-file"
+ )
def test_get_images_exception(self):
self.pipeline.min_width = 100
self.pipeline.min_height = 100
- _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
- _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0))
- _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0))
+ _, buf1 = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+ _, buf2 = _create_image("JPEG", "RGB", (150, 50), (0, 0, 0))
+ _, buf3 = _create_image("JPEG", "RGB", (50, 150), (0, 0, 0))
resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue())
resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue())
@@ -128,121 +161,154 @@ class ImagesPipelineTestCase(unittest.TestCase):
def test_get_images_new(self):
self.pipeline.min_width = 0
self.pipeline.min_height = 0
- self.pipeline.thumbs = {'small': (20, 20)}
+ self.pipeline.thumbs = {"small": (20, 20)}
- orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
- orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0))
+ orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+ orig_thumb, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0))
resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
req = Request(url="https://dev.mydeco.com/mydeco.gif")
- get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object())
+ get_images_gen = self.pipeline.get_images(
+ response=resp, request=req, info=object()
+ )
path, new_im, new_buf = next(get_images_gen)
- self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+ self.assertEqual(path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg")
self.assertEqual(orig_im, new_im)
self.assertEqual(buf.getvalue(), new_buf.getvalue())
thumb_path, thumb_img, thumb_buf = next(get_images_gen)
- self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+ self.assertEqual(
+ thumb_path, "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+ )
self.assertEqual(thumb_img, thumb_img)
self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue())
def test_get_images_old(self):
- self.pipeline.thumbs = {'small': (20, 20)}
- orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
+ self.pipeline.thumbs = {"small": (20, 20)}
+ orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
req = Request(url="https://dev.mydeco.com/mydeco.gif")
def overridden_convert_image(image, size=None):
- im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0))
+ im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
return im, buf
- with patch.object(self.pipeline, 'convert_image', overridden_convert_image):
+ with patch.object(self.pipeline, "convert_image", overridden_convert_image):
with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter('always')
- get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object())
+ warnings.simplefilter("always")
+ get_images_gen = self.pipeline.get_images(
+ response=resp, request=req, info=object()
+ )
path, new_im, new_buf = next(get_images_gen)
- self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+ self.assertEqual(
+ path, "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+ )
self.assertEqual(orig_im.mode, new_im.mode)
self.assertEqual(orig_im.getcolors(), new_im.getcolors())
self.assertEqual(buf.getvalue(), new_buf.getvalue())
thumb_path, thumb_img, thumb_buf = next(get_images_gen)
- self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
+ self.assertEqual(
+ thumb_path,
+ "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg",
+ )
self.assertEqual(orig_im.mode, thumb_img.mode)
self.assertEqual(orig_im.getcolors(), thumb_img.getcolors())
self.assertEqual(buf.getvalue(), thumb_buf.getvalue())
- expected_warning_msg = ('.convert_image() method overriden in a deprecated way, '
- 'overriden method does not accept response_body argument.')
- self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1)
+ expected_warning_msg = (
+ ".convert_image() method overridden in a deprecated way, "
+ "overridden method does not accept response_body argument."
+ )
+ self.assertEqual(
+ len(
+ [
+ warning
+ for warning in w
+ if expected_warning_msg in str(warning.message)
+ ]
+ ),
+ 1,
+ )
def test_convert_image_old(self):
# tests for old API
with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter('always')
+ warnings.simplefilter("always")
SIZE = (100, 100)
- # straigh forward case: RGB and JPEG
+ # straight forward case: RGB and JPEG
COLOUR = (0, 127, 255)
- im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR)
+ im, _ = _create_image("JPEG", "RGB", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im)
- self.assertEqual(converted.mode, 'RGB')
+ self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
# check that thumbnail keep image ratio
thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25))
- self.assertEqual(thumbnail.mode, 'RGB')
+ self.assertEqual(thumbnail.mode, "RGB")
self.assertEqual(thumbnail.size, (10, 10))
# transparency case: RGBA and PNG
COLOUR = (0, 127, 255, 50)
- im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR)
+ im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im)
- self.assertEqual(converted.mode, 'RGB')
+ self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
# transparency case with palette: P and PNG
COLOUR = (0, 127, 255, 50)
- im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR)
- im = im.convert('P')
+ im, _ = _create_image("PNG", "RGBA", SIZE, COLOUR)
+ im = im.convert("P")
converted, _ = self.pipeline.convert_image(im)
- self.assertEqual(converted.mode, 'RGB')
+ self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
- # ensure that we recieved deprecation warnings
- expected_warning_msg = '.convert_image() method called in a deprecated way'
- self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4)
+ # ensure that we received deprecation warnings
+ expected_warning_msg = ".convert_image() method called in a deprecated way"
+ self.assertTrue(
+ len(
+ [
+ warning
+ for warning in w
+ if expected_warning_msg in str(warning.message)
+ ]
+ )
+ == 4
+ )
def test_convert_image_new(self):
# tests for new API
SIZE = (100, 100)
- # straigh forward case: RGB and JPEG
+ # straight forward case: RGB and JPEG
COLOUR = (0, 127, 255)
- im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR)
+ im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)
converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
- self.assertEqual(converted.mode, 'RGB')
+ self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, COLOUR)])
# check that we don't convert JPEGs again
self.assertEqual(converted_buf, buf)
# check that thumbnail keep image ratio
- thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf)
- self.assertEqual(thumbnail.mode, 'RGB')
+ thumbnail, _ = self.pipeline.convert_image(
+ converted, size=(10, 25), response_body=converted_buf
+ )
+ self.assertEqual(thumbnail.mode, "RGB")
self.assertEqual(thumbnail.size, (10, 10))
# transparency case: RGBA and PNG
COLOUR = (0, 127, 255, 50)
- im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR)
+ im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
converted, _ = self.pipeline.convert_image(im, response_body=buf)
- self.assertEqual(converted.mode, 'RGB')
+ self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
# transparency case with palette: P and PNG
COLOUR = (0, 127, 255, 50)
- im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR)
- im = im.convert('P')
+ im, buf = _create_image("PNG", "RGBA", SIZE, COLOUR)
+ im = im.convert("P")
converted, _ = self.pipeline.convert_image(im, response_body=buf)
- self.assertEqual(converted.mode, 'RGB')
+ self.assertEqual(converted.mode, "RGB")
self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))])
@@ -252,47 +318,54 @@ class DeprecatedImagesPipeline(ImagesPipeline):
def image_key(self, url):
image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
- return f'empty/{image_guid}.jpg'
+ return f"empty/{image_guid}.jpg"
def thumb_key(self, url, thumb_id):
thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
- return f'thumbsup/{thumb_id}/{thumb_guid}.jpg'
+ return f"thumbsup/{thumb_id}/{thumb_guid}.jpg"
class ImagesPipelineTestCaseFieldsMixin:
-
skip = skip_pillow
def test_item_fields_default(self):
- url = 'http://www.example.com/images/1.jpg'
- item = self.item_class(name='item1', image_urls=[url])
- pipeline = ImagesPipeline.from_settings(Settings({'IMAGES_STORE': 's3://example/images/'}))
+ url = "http://www.example.com/images/1.jpg"
+ item = self.item_class(name="item1", image_urls=[url])
+ pipeline = ImagesPipeline.from_settings(
+ Settings({"IMAGES_STORE": "s3://example/images/"})
+ )
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
- results = [(True, {'url': url})]
+ results = [(True, {"url": url})]
item = pipeline.item_completed(results, item, None)
images = ItemAdapter(item).get("images")
self.assertEqual(images, [results[0][1]])
self.assertIsInstance(item, self.item_class)
def test_item_fields_override_settings(self):
- url = 'http://www.example.com/images/1.jpg'
- item = self.item_class(name='item1', custom_image_urls=[url])
- pipeline = ImagesPipeline.from_settings(Settings({
- 'IMAGES_STORE': 's3://example/images/',
- 'IMAGES_URLS_FIELD': 'custom_image_urls',
- 'IMAGES_RESULT_FIELD': 'custom_images'
- }))
+ url = "http://www.example.com/images/1.jpg"
+ item = self.item_class(name="item1", custom_image_urls=[url])
+ pipeline = ImagesPipeline.from_settings(
+ Settings(
+ {
+ "IMAGES_STORE": "s3://example/images/",
+ "IMAGES_URLS_FIELD": "custom_image_urls",
+ "IMAGES_RESULT_FIELD": "custom_images",
+ }
+ )
+ )
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
- results = [(True, {'url': url})]
+ results = [(True, {"url": url})]
item = pipeline.item_completed(results, item, None)
custom_images = ItemAdapter(item).get("custom_images")
self.assertEqual(custom_images, [results[0][1]])
self.assertIsInstance(item, self.item_class)
-class ImagesPipelineTestCaseFieldsDict(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsDict(
+ ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = dict
@@ -306,7 +379,9 @@ class ImagesPipelineTestItem(Item):
custom_images = Field()
-class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsItem(
+ ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = ImagesPipelineTestItem
@@ -321,7 +396,9 @@ class ImagesPipelineTestDataClass:
custom_images: list = dataclasses.field(default_factory=list)
-class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsDataClass(
+ ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = ImagesPipelineTestDataClass
@@ -329,19 +406,20 @@ class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, u
class ImagesPipelineTestAttrsItem:
name = attr.ib(default="")
# default fields
- image_urls = attr.ib(default=lambda: [])
- images = attr.ib(default=lambda: [])
+ image_urls: List[str] = attr.ib(default=lambda: [])
+ images: List[Dict[str, str]] = attr.ib(default=lambda: [])
# overridden fields
- custom_image_urls = attr.ib(default=lambda: [])
- custom_images = attr.ib(default=lambda: [])
+ custom_image_urls: List[str] = attr.ib(default=lambda: [])
+ custom_images: List[Dict[str, str]] = attr.ib(default=lambda: [])
-class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+class ImagesPipelineTestCaseFieldsAttrsItem(
+ ImagesPipelineTestCaseFieldsMixin, unittest.TestCase
+):
item_class = ImagesPipelineTestAttrsItem
class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
-
skip = skip_pillow
img_cls_attribute_names = [
@@ -351,7 +429,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
("MIN_HEIGHT", "IMAGES_MIN_HEIGHT"),
("IMAGES_URLS_FIELD", "IMAGES_URLS_FIELD"),
("IMAGES_RESULT_FIELD", "IMAGES_RESULT_FIELD"),
- ("THUMBS", "IMAGES_THUMBS")
+ ("THUMBS", "IMAGES_THUMBS"),
]
# This should match what is defined in ImagesPipeline.
@@ -360,8 +438,8 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
MIN_HEIGHT=0,
EXPIRES=90,
THUMBS={},
- IMAGES_URLS_FIELD='image_urls',
- IMAGES_RESULT_FIELD='images'
+ IMAGES_URLS_FIELD="image_urls",
+ IMAGES_RESULT_FIELD="images",
)
def setUp(self):
@@ -387,26 +465,30 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
"IMAGES_MIN_WIDTH": random.randint(1, 1000),
"IMAGES_MIN_HEIGHT": random.randint(1, 1000),
"IMAGES_THUMBS": {
- 'small': (random.randint(1, 1000), random.randint(1, 1000)),
- 'big': (random.randint(1, 1000), random.randint(1, 1000))
- }
+ "small": (random.randint(1, 1000), random.randint(1, 1000)),
+ "big": (random.randint(1, 1000), random.randint(1, 1000)),
+ },
}
if not prefix:
return settings
- return {prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v for k, v in settings.items()}
+ return {
+ prefix.upper() + "_" + k if k != "IMAGES_STORE" else k: v
+ for k, v in settings.items()
+ }
def _generate_fake_pipeline_subclass(self):
"""
:return: ImagePipeline class will all uppercase attributes set.
"""
+
class UserDefinedImagePipeline(ImagesPipeline):
# Values should be in different range than fake_settings.
MIN_WIDTH = random.randint(1000, 2000)
MIN_HEIGHT = random.randint(1000, 2000)
THUMBS = {
- 'small': (random.randint(1000, 2000), random.randint(1000, 2000)),
- 'big': (random.randint(1000, 2000), random.randint(1000, 2000))
+ "small": (random.randint(1000, 2000), random.randint(1000, 2000)),
+ "big": (random.randint(1000, 2000), random.randint(1000, 2000)),
}
EXPIRES = random.randint(1000, 2000)
IMAGES_URLS_FIELD = "field_one"
@@ -427,7 +509,9 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
expected_default_value = self.default_pipeline_settings.get(pipe_attr)
custom_value = custom_settings.get(settings_attr)
self.assertNotEqual(expected_default_value, custom_value)
- self.assertEqual(getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value)
+ self.assertEqual(
+ getattr(default_sts_pipe, pipe_attr.lower()), expected_default_value
+ )
self.assertEqual(getattr(user_sts_pipe, pipe_attr.lower()), custom_value)
def test_subclass_attrs_preserved_default_settings(self):
@@ -464,10 +548,13 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
If there are no settings for subclass and no subclass attributes, pipeline should use
attributes of base class.
"""
+
class UserDefinedImagePipeline(ImagesPipeline):
pass
- user_pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
+ user_pipeline = UserDefinedImagePipeline.from_settings(
+ Settings({"IMAGES_STORE": self.tempdir})
+ )
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_pipeline_settings.get(pipe_attr.upper())
@@ -478,6 +565,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
If there are custom settings for subclass and NO class attributes, pipeline should use custom
settings.
"""
+
class UserDefinedImagePipeline(ImagesPipeline):
pass
@@ -508,9 +596,18 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
class UserDefinedImagePipeline(ImagesPipeline):
DEFAULT_IMAGES_URLS_FIELD = "something"
DEFAULT_IMAGES_RESULT_FIELD = "something_else"
- pipeline = UserDefinedImagePipeline.from_settings(Settings({"IMAGES_STORE": self.tempdir}))
- self.assertEqual(pipeline.images_result_field, "something_else")
- self.assertEqual(pipeline.images_urls_field, "something")
+
+ pipeline = UserDefinedImagePipeline.from_settings(
+ Settings({"IMAGES_STORE": self.tempdir})
+ )
+ self.assertEqual(
+ pipeline.images_result_field,
+ UserDefinedImagePipeline.DEFAULT_IMAGES_RESULT_FIELD,
+ )
+ self.assertEqual(
+ pipeline.images_urls_field,
+ UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD,
+ )
def test_user_defined_subclass_default_key_names(self):
"""Test situation when user defines subclass of ImagePipeline,
@@ -526,12 +623,10 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_value = settings.get(settings_attr)
- self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()),
- expected_value)
+ self.assertEqual(getattr(pipeline_cls, pipe_attr.lower()), expected_value)
class NoimagesDropTestCase(unittest.TestCase):
-
def test_deprecation_warning(self):
arg = str()
with warnings.catch_warnings(record=True) as w:
@@ -539,8 +634,10 @@ class NoimagesDropTestCase(unittest.TestCase):
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
with warnings.catch_warnings(record=True) as w:
+
class SubclassedNoimagesDrop(NoimagesDrop):
pass
+
SubclassedNoimagesDrop(arg)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
@@ -551,7 +648,3 @@ def _create_image(format, *a, **kw):
Image.new(*a, **kw).save(buf, format)
buf.seek(0)
return Image.open(buf), buf
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 0a94ae699..820484565 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,46 +1,47 @@
-from typing import Optional
import io
+from typing import Optional
from testfixtures import LogCapture
-from twisted.trial import unittest
-from twisted.python.failure import Failure
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.python.failure import Failure
+from twisted.trial import unittest
from scrapy import signals
from scrapy.http import Request, Response
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
+from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.images import ImagesPipeline
from scrapy.pipelines.media import MediaPipeline
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.settings import Settings
+from scrapy.spiders import Spider
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.test import get_crawler
-
try:
from PIL import Image # noqa: imported just to check for the import error
except ImportError:
- skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+ skip_pillow: Optional[
+ str
+ ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
else:
skip_pillow = None
def _mocked_download_func(request, info):
- response = request.meta.get('response')
+ assert request.callback is NO_CALLBACK
+ response = request.meta.get("response")
return response() if callable(response) else response
class BaseMediaPipelineTestCase(unittest.TestCase):
-
pipeline_class = MediaPipeline
settings = None
def setUp(self):
spider_cls = Spider
- self.spider = spider_cls('media.com')
+ self.spider = spider_cls("media.com")
crawler = get_crawler(spider_cls, self.settings)
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
@@ -50,29 +51,29 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
def tearDown(self):
for name, signal in vars(signals).items():
- if not name.startswith('_'):
+ if not name.startswith("_"):
disconnect_all(signal)
def test_default_media_to_download(self):
- request = Request('http://url')
+ request = Request("http://url")
assert self.pipe.media_to_download(request, self.info) is None
def test_default_get_media_requests(self):
- item = dict(name='name')
+ item = dict(name="name")
assert self.pipe.get_media_requests(item, self.info) is None
def test_default_media_downloaded(self):
- request = Request('http://url')
- response = Response('http://url', body=b'')
+ request = Request("http://url")
+ response = Response("http://url", body=b"")
assert self.pipe.media_downloaded(response, request, self.info) is response
def test_default_media_failed(self):
- request = Request('http://url')
+ request = Request("http://url")
fail = Failure(Exception())
assert self.pipe.media_failed(fail, request, self.info) is fail
def test_default_item_completed(self):
- item = dict(name='name')
+ item = dict(name="name")
assert self.pipe.item_completed([], item, self.info) is item
# Check that failures are logged by default
@@ -85,7 +86,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
assert new_item is item
assert len(log.records) == 1
record = log.records[0]
- assert record.levelname == 'ERROR'
+ assert record.levelname == "ERROR"
self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
# disable failure logging and check again
@@ -97,14 +98,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
@inlineCallbacks
def test_default_process_item(self):
- item = dict(name='name')
+ item = dict(name="name")
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
def test_modify_media_request(self):
- request = Request('http://url')
+ request = Request("http://url")
self.pipe._modify_media_request(request)
- assert request.meta == {'handle_httpstatus_all': True}
+ assert request.meta == {"handle_httpstatus_all": True}
def test_should_remove_req_res_references_before_caching_the_results(self):
"""Regression test case to prevent a memory leak in the Media Pipeline.
@@ -134,8 +135,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
Chaining (https://www.python.org/dev/peps/pep-3134/).
"""
# Create sample pair of Request and Response objects
- request = Request('http://url')
- response = Response('http://url', body=b'', request=request)
+ request = Request("http://url")
+ response = Response("http://url", body=b"", request=request)
# Simulate the Media Pipeline behavior to produce a Twisted Failure
try:
@@ -146,7 +147,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
try:
# Simulate the media_downloaded callback raising a FileException
# This usually happens when the status code is not 200 OK
- raise FileException('download-error')
+ raise FileException("download-error")
except Exception as exc:
file_exc = exc
# Simulate Twisted capturing the FileException
@@ -171,116 +172,136 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
# ... encapsulating the original FileException ...
self.assertEqual(info.downloaded[fp].value, file_exc)
# ... but it should not store the StopIteration exception on its context
- context = getattr(info.downloaded[fp].value, '__context__', None)
+ context = getattr(info.downloaded[fp].value, "__context__", None)
self.assertIsNone(context)
class MockedMediaPipeline(MediaPipeline):
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._mockcalled = []
def download(self, request, info):
- self._mockcalled.append('download')
+ self._mockcalled.append("download")
return super().download(request, info)
def media_to_download(self, request, info, *, item=None):
- self._mockcalled.append('media_to_download')
- if 'result' in request.meta:
- return request.meta.get('result')
+ self._mockcalled.append("media_to_download")
+ if "result" in request.meta:
+ return request.meta.get("result")
return super().media_to_download(request, info)
def get_media_requests(self, item, info):
- self._mockcalled.append('get_media_requests')
- return item.get('requests')
+ self._mockcalled.append("get_media_requests")
+ return item.get("requests")
def media_downloaded(self, response, request, info, *, item=None):
- self._mockcalled.append('media_downloaded')
+ self._mockcalled.append("media_downloaded")
return super().media_downloaded(response, request, info)
def media_failed(self, failure, request, info):
- self._mockcalled.append('media_failed')
+ self._mockcalled.append("media_failed")
return super().media_failed(failure, request, info)
def item_completed(self, results, item, info):
- self._mockcalled.append('item_completed')
+ self._mockcalled.append("item_completed")
item = super().item_completed(results, item, info)
- item['results'] = results
+ item["results"] = results
return item
class MediaPipelineTestCase(BaseMediaPipelineTestCase):
-
pipeline_class = MockedMediaPipeline
def _callback(self, result):
- self.pipe._mockcalled.append('request_callback')
+ self.pipe._mockcalled.append("request_callback")
return result
def _errback(self, result):
- self.pipe._mockcalled.append('request_errback')
+ self.pipe._mockcalled.append("request_errback")
return result
@inlineCallbacks
def test_result_succeed(self):
- rsp = Response('http://url1')
- req = Request('http://url1', meta=dict(response=rsp),
- callback=self._callback, errback=self._errback)
+ rsp = Response("http://url1")
+ req = Request(
+ "http://url1",
+ meta=dict(response=rsp),
+ callback=self._callback,
+ errback=self._errback,
+ )
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
- self.assertEqual(new_item['results'], [(True, rsp)])
+ self.assertEqual(new_item["results"], [(True, rsp)])
self.assertEqual(
self.pipe._mockcalled,
- ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
+ [
+ "get_media_requests",
+ "media_to_download",
+ "media_downloaded",
+ "request_callback",
+ "item_completed",
+ ],
+ )
@inlineCallbacks
def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
fail = Failure(Exception())
- req = Request('http://url1', meta=dict(response=fail),
- callback=self._callback, errback=self._errback)
+ req = Request(
+ "http://url1",
+ meta=dict(response=fail),
+ callback=self._callback,
+ errback=self._errback,
+ )
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
- self.assertEqual(new_item['results'], [(False, fail)])
+ self.assertEqual(new_item["results"], [(False, fail)])
self.assertEqual(
self.pipe._mockcalled,
- ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
+ [
+ "get_media_requests",
+ "media_to_download",
+ "media_failed",
+ "request_errback",
+ "item_completed",
+ ],
+ )
@inlineCallbacks
def test_mix_of_success_and_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
- rsp1 = Response('http://url1')
- req1 = Request('http://url1', meta=dict(response=rsp1))
+ rsp1 = Response("http://url1")
+ req1 = Request("http://url1", meta=dict(response=rsp1))
fail = Failure(Exception())
- req2 = Request('http://url2', meta=dict(response=fail))
+ req2 = Request("http://url2", meta=dict(response=fail))
item = dict(requests=[req1, req2])
new_item = yield self.pipe.process_item(item, self.spider)
- self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
+ self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)])
m = self.pipe._mockcalled
# only once
- self.assertEqual(m[0], 'get_media_requests') # first hook called
- self.assertEqual(m.count('get_media_requests'), 1)
- self.assertEqual(m.count('item_completed'), 1)
- self.assertEqual(m[-1], 'item_completed') # last hook called
+ self.assertEqual(m[0], "get_media_requests") # first hook called
+ self.assertEqual(m.count("get_media_requests"), 1)
+ self.assertEqual(m.count("item_completed"), 1)
+ self.assertEqual(m[-1], "item_completed") # last hook called
# twice, one per request
- self.assertEqual(m.count('media_to_download'), 2)
+ self.assertEqual(m.count("media_to_download"), 2)
# one to handle success and other for failure
- self.assertEqual(m.count('media_downloaded'), 1)
- self.assertEqual(m.count('media_failed'), 1)
+ self.assertEqual(m.count("media_downloaded"), 1)
+ self.assertEqual(m.count("media_failed"), 1)
@inlineCallbacks
def test_get_media_requests(self):
# returns single Request (without callback)
- req = Request('http://url')
+ req = Request("http://url")
item = dict(requests=req) # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
self.assertIn(self.fingerprint(req), self.info.downloaded)
# returns iterable of Requests
- req1 = Request('http://url1')
- req2 = Request('http://url2')
+ req1 = Request("http://url1")
+ req2 = Request("http://url2")
item = dict(requests=iter([req1, req2]))
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
@@ -289,30 +310,34 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_results_are_cached_across_multiple_items(self):
- rsp1 = Response('http://url1')
- req1 = Request('http://url1', meta=dict(response=rsp1))
+ rsp1 = Response("http://url1")
+ req1 = Request("http://url1", meta=dict(response=rsp1))
item = dict(requests=req1)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
- self.assertEqual(new_item['results'], [(True, rsp1)])
+ self.assertEqual(new_item["results"], [(True, rsp1)])
# rsp2 is ignored, rsp1 must be in results because request fingerprints are the same
- req2 = Request(req1.url, meta=dict(response=Response('http://donot.download.me')))
+ req2 = Request(
+ req1.url, meta=dict(response=Response("http://donot.download.me"))
+ )
item = dict(requests=req2)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
- self.assertEqual(new_item['results'], [(True, rsp1)])
+ self.assertEqual(new_item["results"], [(True, rsp1)])
@inlineCallbacks
def test_results_are_cached_for_requests_of_single_item(self):
- rsp1 = Response('http://url1')
- req1 = Request('http://url1', meta=dict(response=rsp1))
- req2 = Request(req1.url, meta=dict(response=Response('http://donot.download.me')))
+ rsp1 = Response("http://url1")
+ req1 = Request("http://url1", meta=dict(response=rsp1))
+ req2 = Request(
+ req1.url, meta=dict(response=Response("http://donot.download.me"))
+ )
item = dict(requests=[req1, req2])
new_item = yield self.pipe.process_item(item, self.spider)
self.assertTrue(new_item is item)
- self.assertEqual(new_item['results'], [(True, rsp1), (True, rsp1)])
+ self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
@inlineCallbacks
def test_wait_if_request_is_downloading(self):
@@ -324,227 +349,122 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
self.assertEqual(len(self.info.waiting[fp]), 2)
return response
- rsp1 = Response('http://url')
+ rsp1 = Response("http://url")
def rsp1_func():
dfd = Deferred().addCallback(_check_downloading)
- reactor.callLater(.1, dfd.callback, rsp1)
+ reactor.callLater(0.1, dfd.callback, rsp1)
return dfd
def rsp2_func():
- self.fail('it must cache rsp1 result and must not try to redownload')
+ self.fail("it must cache rsp1 result and must not try to redownload")
- req1 = Request('http://url', meta=dict(response=rsp1_func))
+ req1 = Request("http://url", meta=dict(response=rsp1_func))
req2 = Request(req1.url, meta=dict(response=rsp2_func))
item = dict(requests=[req1, req2])
new_item = yield self.pipe.process_item(item, self.spider)
- self.assertEqual(new_item['results'], [(True, rsp1), (True, rsp1)])
+ self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
@inlineCallbacks
def test_use_media_to_download_result(self):
- req = Request('http://url', meta=dict(result='ITSME', response=self.fail))
+ req = Request("http://url", meta=dict(result="ITSME", response=self.fail))
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
- self.assertEqual(new_item['results'], [(True, 'ITSME')])
+ self.assertEqual(new_item["results"], [(True, "ITSME")])
self.assertEqual(
self.pipe._mockcalled,
- ['get_media_requests', 'media_to_download', 'item_completed'])
+ ["get_media_requests", "media_to_download", "item_completed"],
+ )
class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
-
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._mockcalled = []
def get_media_requests(self, item, info):
- item_url = item['image_urls'][0]
+ item_url = item["image_urls"][0]
output_img = io.BytesIO()
- img = Image.new('RGB', (60, 30), color='red')
- img.save(output_img, format='JPEG')
+ img = Image.new("RGB", (60, 30), color="red")
+ img.save(output_img, format="JPEG")
return Request(
item_url,
- meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
+ meta={
+ "response": Response(item_url, status=200, body=output_img.getvalue())
+ },
)
def inc_stats(self, *args, **kwargs):
return True
def media_to_download(self, request, info):
- self._mockcalled.append('media_to_download')
+ self._mockcalled.append("media_to_download")
return super().media_to_download(request, info)
def media_downloaded(self, response, request, info):
- self._mockcalled.append('media_downloaded')
+ self._mockcalled.append("media_downloaded")
return super().media_downloaded(response, request, info)
def file_downloaded(self, response, request, info):
- self._mockcalled.append('file_downloaded')
+ self._mockcalled.append("file_downloaded")
return super().file_downloaded(response, request, info)
def file_path(self, request, response=None, info=None):
- self._mockcalled.append('file_path')
+ self._mockcalled.append("file_path")
return super().file_path(request, response, info)
def thumb_path(self, request, thumb_id, response=None, info=None):
- self._mockcalled.append('thumb_path')
+ self._mockcalled.append("thumb_path")
return super().thumb_path(request, thumb_id, response, info)
def get_images(self, response, request, info):
- self._mockcalled.append('get_images')
+ self._mockcalled.append("get_images")
return super().get_images(response, request, info)
def image_downloaded(self, response, request, info):
- self._mockcalled.append('image_downloaded')
+ self._mockcalled.append("image_downloaded")
return super().image_downloaded(response, request, info)
-class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
- skip = skip_pillow
-
- def setUp(self):
- settings_dict = {
- 'IMAGES_STORE': 'store-uri',
- 'IMAGES_THUMBS': {'small': (50, 50)},
- }
- crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
- self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
- self.pipe.download_func = _mocked_download_func
- self.pipe.open_spider(None)
- self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
-
- def _assert_method_called_with_warnings(self, method, message, warnings):
- self.assertIn(method, self.pipe._mockcalled)
- warningShown = False
- for warning in warnings:
- if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning:
- warningShown = True
- self.assertTrue(warningShown)
-
- @inlineCallbacks
- def test_media_to_download_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'media_to_download(self, request, info) is deprecated, '
- 'please use media_to_download(self, request, info, *, item=None)'
- )
- self._assert_method_called_with_warnings('media_to_download', message, warnings)
-
- @inlineCallbacks
- def test_media_downloaded_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'media_downloaded(self, response, request, info) is deprecated, '
- 'please use media_downloaded(self, response, request, info, *, item=None)'
- )
- self._assert_method_called_with_warnings('media_downloaded', message, warnings)
-
- @inlineCallbacks
- def test_file_downloaded_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'file_downloaded(self, response, request, info) is deprecated, '
- 'please use file_downloaded(self, response, request, info, *, item=None)'
- )
- self._assert_method_called_with_warnings('file_downloaded', message, warnings)
-
- @inlineCallbacks
- def test_file_path_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'file_path(self, request, response=None, info=None) is deprecated, '
- 'please use file_path(self, request, response=None, info=None, *, item=None)'
- )
- self._assert_method_called_with_warnings('file_path', message, warnings)
-
- @inlineCallbacks
- def test_thumb_path_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
- 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
- )
- self._assert_method_called_with_warnings('thumb_path', message, warnings)
-
- @inlineCallbacks
- def test_get_images_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'get_images(self, response, request, info) is deprecated, '
- 'please use get_images(self, response, request, info, *, item=None)'
- )
- self._assert_method_called_with_warnings('get_images', message, warnings)
-
- @inlineCallbacks
- def test_image_downloaded_called(self):
- yield self.pipe.process_item(self.item, None)
- warnings = self.flushWarnings([MediaPipeline._compatible])
- message = (
- 'image_downloaded(self, response, request, info) is deprecated, '
- 'please use image_downloaded(self, response, request, info, *, item=None)'
- )
- self._assert_method_called_with_warnings('image_downloaded', message, warnings)
-
-
class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
-
def _assert_request_no3xx(self, pipeline_class, settings):
pipe = pipeline_class(settings=Settings(settings))
- request = Request('http://url')
+ request = Request("http://url")
pipe._modify_media_request(request)
- self.assertIn('handle_httpstatus_list', request.meta)
+ self.assertIn("handle_httpstatus_list", request.meta)
for status, check in [
- (200, True),
-
- # These are the status codes we want
- # the downloader to handle itself
- (301, False),
- (302, False),
- (302, False),
- (307, False),
- (308, False),
-
- # we still want to get 4xx and 5xx
- (400, True),
- (404, True),
- (500, True)]:
+ (200, True),
+ # These are the status codes we want
+ # the downloader to handle itself
+ (301, False),
+ (302, False),
+ (302, False),
+ (307, False),
+ (308, False),
+ # we still want to get 4xx and 5xx
+ (400, True),
+ (404, True),
+ (500, True),
+ ]:
if check:
- self.assertIn(status, request.meta['handle_httpstatus_list'])
+ self.assertIn(status, request.meta["handle_httpstatus_list"])
else:
- self.assertNotIn(status, request.meta['handle_httpstatus_list'])
+ self.assertNotIn(status, request.meta["handle_httpstatus_list"])
def test_standard_setting(self):
- self._assert_request_no3xx(
- MediaPipeline,
- {
- 'MEDIA_ALLOW_REDIRECTS': True
- })
+ self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
def test_subclass_standard_setting(self):
-
class UserDefinedPipeline(MediaPipeline):
pass
- self._assert_request_no3xx(
- UserDefinedPipeline,
- {
- 'MEDIA_ALLOW_REDIRECTS': True
- })
+ self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
def test_subclass_specific_setting(self):
-
class UserDefinedPipeline(MediaPipeline):
pass
self._assert_request_no3xx(
- UserDefinedPipeline,
- {
- 'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True
- })
+ UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True}
+ )
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 8e432b913..5ab288c1a 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -5,22 +5,21 @@ from twisted.internet import defer
from twisted.internet.defer import Deferred
from twisted.trial import unittest
-from scrapy import Spider, signals, Request
-from scrapy.utils.defer import maybe_deferred_to_future, deferred_to_future
+from scrapy import Request, Spider, signals
+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-
from tests.mockserver import MockServer
class SimplePipeline:
def process_item(self, item, spider):
- item['pipeline_passed'] = True
+ item["pipeline_passed"] = True
return item
class DeferredPipeline:
def cb(self, item):
- item['pipeline_passed'] = True
+ item["pipeline_passed"] = True
return item
def process_item(self, item, spider):
@@ -34,9 +33,10 @@ class AsyncDefPipeline:
async def process_item(self, item, spider):
d = Deferred()
from twisted.internet import reactor
+
reactor.callLater(0, d.callback, None)
await maybe_deferred_to_future(d)
- item['pipeline_passed'] = True
+ item["pipeline_passed"] = True
return item
@@ -44,10 +44,11 @@ class AsyncDefAsyncioPipeline:
async def process_item(self, item, spider):
d = Deferred()
from twisted.internet import reactor
+
reactor.callLater(0, d.callback, None)
await deferred_to_future(d)
await asyncio.sleep(0.2)
- item['pipeline_passed'] = await get_from_asyncio_queue(True)
+ item["pipeline_passed"] = await get_from_asyncio_queue(True)
return item
@@ -55,23 +56,24 @@ class AsyncDefNotAsyncioPipeline:
async def process_item(self, item, spider):
d1 = Deferred()
from twisted.internet import reactor
+
reactor.callLater(0, d1.callback, None)
await d1
d2 = Deferred()
reactor.callLater(0, d2.callback, None)
await maybe_deferred_to_future(d2)
- item['pipeline_passed'] = True
+ item["pipeline_passed"] = True
return item
class ItemSpider(Spider):
- name = 'itemspider'
+ name = "itemspider"
def start_requests(self):
- yield Request(self.mockserver.url('/status?n=200'))
+ yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
- return {'field': 42}
+ return {"field": 42}
class PipelineTestCase(unittest.TestCase):
@@ -84,12 +86,12 @@ class PipelineTestCase(unittest.TestCase):
def _on_item_scraped(self, item):
self.assertIsInstance(item, dict)
- self.assertTrue(item.get('pipeline_passed'))
+ self.assertTrue(item.get("pipeline_passed"))
self.items.append(item)
def _create_crawler(self, pipeline_class):
settings = {
- 'ITEM_PIPELINES': {pipeline_class: 1},
+ "ITEM_PIPELINES": {pipeline_class: 1},
}
crawler = get_crawler(ItemSpider, settings)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index ec55033d1..1584014b8 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -4,11 +4,10 @@ import unittest
import queuelib
from scrapy.http.request import Request
-from scrapy.pqueues import ScrapyPriorityQueue, DownloaderAwarePriorityQueue
+from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue
from scrapy.spiders import Spider
from scrapy.squeues import FifoMemoryQueue
from scrapy.utils.test import get_crawler
-
from tests.test_scheduler import MockDownloader, MockEngine
@@ -19,7 +18,9 @@ class PriorityQueueTest(unittest.TestCase):
def test_queue_push_pop_one(self):
temp_dir = tempfile.mkdtemp()
- queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+ queue = ScrapyPriorityQueue.from_crawler(
+ self.crawler, FifoMemoryQueue, temp_dir
+ )
self.assertIsNone(queue.pop())
self.assertEqual(len(queue), 0)
req1 = Request("https://example.org/1", priority=1)
@@ -35,9 +36,14 @@ class PriorityQueueTest(unittest.TestCase):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
temp_dir = tempfile.mkdtemp()
- queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+ queue = ScrapyPriorityQueue.from_crawler(
+ self.crawler, FifoMemoryQueue, temp_dir
+ )
queue.push(Request("https://example.org"))
- with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+ with self.assertRaises(
+ NotImplementedError,
+ msg="The underlying queue class does not implement 'peek'",
+ ):
queue.peek()
queue.close()
@@ -45,7 +51,9 @@ class PriorityQueueTest(unittest.TestCase):
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is undefined")
temp_dir = tempfile.mkdtemp()
- queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir)
+ queue = ScrapyPriorityQueue.from_crawler(
+ self.crawler, FifoMemoryQueue, temp_dir
+ )
self.assertEqual(len(queue), 0)
self.assertIsNone(queue.peek())
req1 = Request("https://example.org/1")
@@ -67,7 +75,9 @@ class PriorityQueueTest(unittest.TestCase):
def test_queue_push_pop_priorities(self):
temp_dir = tempfile.mkdtemp()
- queue = ScrapyPriorityQueue.from_crawler(self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3])
+ queue = ScrapyPriorityQueue.from_crawler(
+ self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
+ )
self.assertIsNone(queue.pop())
self.assertEqual(len(queue), 0)
req1 = Request("https://example.org/1", priority=1)
@@ -119,7 +129,10 @@ class DownloaderAwarePriorityQueueTest(unittest.TestCase):
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
raise unittest.SkipTest("queuelib.queue.FifoMemoryQueue.peek is defined")
self.queue.push(Request("https://example.org"))
- with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+ with self.assertRaises(
+ NotImplementedError,
+ msg="The underlying queue class does not implement 'peek'",
+ ):
self.queue.peek()
def test_peek(self):
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ea7701b5d..dc0a82086 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -3,44 +3,51 @@ import os
import re
import sys
from pathlib import Path
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
from urllib.parse import urlsplit, urlunsplit
+
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy.http import Request
from scrapy.utils.test import get_crawler
-
from tests.mockserver import MockServer
from tests.spiders import SimpleSpider, SingleRequestSpider
class MitmProxy:
- auth_user = 'scrapy'
- auth_pass = 'scrapy'
+ auth_user = "scrapy"
+ auth_pass = "scrapy"
def start(self):
- from scrapy.utils.test import get_testenv
script = """
import sys
from mitmproxy.tools.main import mitmdump
sys.argv[0] = "mitmdump"
sys.exit(mitmdump())
"""
- cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem'
- self.proc = Popen([sys.executable,
- '-c', script,
- '--listen-host', '127.0.0.1',
- '--listen-port', '0',
- '--proxyauth', f'{self.auth_user}:{self.auth_pass}',
- '--certs', str(cert_path),
- '--ssl-insecure',
- ],
- stdout=PIPE, env=get_testenv())
- line = self.proc.stdout.readline().decode('utf-8')
- host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
- address = f'http://{self.auth_user}:{self.auth_pass}@{host_port}'
+ cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem"
+ self.proc = Popen(
+ [
+ sys.executable,
+ "-c",
+ script,
+ "--listen-host",
+ "127.0.0.1",
+ "--listen-port",
+ "0",
+ "--proxyauth",
+ f"{self.auth_user}:{self.auth_pass}",
+ "--certs",
+ str(cert_path),
+ "--ssl-insecure",
+ ],
+ stdout=PIPE,
+ )
+ line = self.proc.stdout.readline().decode("utf-8")
+ host_port = re.search(r"listening at http://([^:]+:\d+)", line).group(1)
+ address = f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
return address
def stop(self):
@@ -50,17 +57,16 @@ sys.exit(mitmdump())
def _wrong_credentials(proxy_url):
bad_auth_proxy = list(urlsplit(proxy_url))
- bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+ bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")
return urlunsplit(bad_auth_proxy)
class ProxyConnectTestCase(TestCase):
-
def setUp(self):
try:
import mitmproxy # noqa: F401
except ImportError:
- self.skipTest('mitmproxy is not installed')
+ self.skipTest("mitmproxy is not installed")
self.mockserver = MockServer()
self.mockserver.__enter__()
@@ -68,8 +74,8 @@ class ProxyConnectTestCase(TestCase):
self._proxy = MitmProxy()
proxy_url = self._proxy.start()
- os.environ['https_proxy'] = proxy_url
- os.environ['http_proxy'] = proxy_url
+ os.environ["https_proxy"] = proxy_url
+ os.environ["http_proxy"] = proxy_url
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@@ -85,7 +91,7 @@ class ProxyConnectTestCase(TestCase):
@defer.inlineCallbacks
def test_https_tunnel_auth_error(self):
- os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+ os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
crawler = get_crawler(SimpleSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
@@ -100,13 +106,13 @@ class ProxyConnectTestCase(TestCase):
with LogCapture() as log:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, log)
- echo = json.loads(crawler.spider.meta['responses'][0].text)
- self.assertTrue('Proxy-Authorization' not in echo['headers'])
+ echo = json.loads(crawler.spider.meta["responses"][0].text)
+ self.assertTrue("Proxy-Authorization" not in echo["headers"])
def _assert_got_response_code(self, code, log):
print(log)
- self.assertEqual(str(log).count(f'Crawled ({code})'), 1)
+ self.assertEqual(str(log).count(f"Crawled ({code})"), 1)
def _assert_got_tunnel_error(self, log):
print(log)
- self.assertIn('TunnelError', str(log))
+ self.assertIn("TunnelError", str(log))
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 15e400327..d65d74206 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -1,22 +1,19 @@
+from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
-from testfixtures import LogCapture
-
from scrapy import Request, signals
from scrapy.http.response import Response
from scrapy.utils.test import get_crawler
-
from tests.mockserver import MockServer
from tests.spiders import SingleRequestSpider
-
-OVERRIDEN_URL = "https://example.org"
+OVERRIDDEN_URL = "https://example.org"
class ProcessResponseMiddleware:
def process_response(self, request, response, spider):
- return response.replace(request=Request(OVERRIDEN_URL))
+ return response.replace(request=Request(OVERRIDDEN_URL))
class RaiseExceptionRequestMiddleware:
@@ -30,7 +27,7 @@ class CatchExceptionOverrideRequestMiddleware:
return Response(
url="http://localhost/",
body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
- request=Request(OVERRIDEN_URL),
+ request=Request(OVERRIDDEN_URL),
)
@@ -52,7 +49,7 @@ class AlternativeCallbacksSpider(SingleRequestSpider):
class AlternativeCallbacksMiddleware:
def process_response(self, request, response, spider):
new_request = request.replace(
- url=OVERRIDEN_URL,
+ url=OVERRIDDEN_URL,
callback=spider.alt_callback,
cb_kwargs={"foo": "bar"},
)
@@ -60,7 +57,6 @@ class AlternativeCallbacksMiddleware:
class CrawlTestCase(TestCase):
-
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
@@ -90,11 +86,14 @@ class CrawlTestCase(TestCase):
@defer.inlineCallbacks
def test_downloader_middleware_raise_exception(self):
url = self.mockserver.url("/status?n=200")
- crawler = get_crawler(SingleRequestSpider, {
- "DOWNLOADER_MIDDLEWARES": {
- RaiseExceptionRequestMiddleware: 590,
+ crawler = get_crawler(
+ SingleRequestSpider,
+ {
+ "DOWNLOADER_MIDDLEWARES": {
+ RaiseExceptionRequestMiddleware: 590,
+ },
},
- })
+ )
yield crawler.crawl(seed=url, mockserver=self.mockserver)
failure = crawler.spider.meta["failure"]
self.assertEqual(failure.request.url, url)
@@ -116,24 +115,31 @@ class CrawlTestCase(TestCase):
signal_params["request"] = request
url = self.mockserver.url("/status?n=200")
- crawler = get_crawler(SingleRequestSpider, {
- "DOWNLOADER_MIDDLEWARES": {
- ProcessResponseMiddleware: 595,
- }
- })
+ crawler = get_crawler(
+ SingleRequestSpider,
+ {
+ "DOWNLOADER_MIDDLEWARES": {
+ ProcessResponseMiddleware: 595,
+ }
+ },
+ )
crawler.signals.connect(signal_handler, signal=signals.response_received)
with LogCapture() as log:
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
- self.assertEqual(response.request.url, OVERRIDEN_URL)
+ self.assertEqual(response.request.url, OVERRIDDEN_URL)
self.assertEqual(signal_params["response"].url, url)
- self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
+ self.assertEqual(signal_params["request"].url, OVERRIDDEN_URL)
log.check_present(
- ("scrapy.core.engine", "DEBUG", f"Crawled (200) <GET {OVERRIDEN_URL}> (referer: None)"),
+ (
+ "scrapy.core.engine",
+ "DEBUG",
+ f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)",
+ ),
)
@defer.inlineCallbacks
@@ -145,16 +151,19 @@ class CrawlTestCase(TestCase):
The spider callback should receive the overridden response.request
"""
url = self.mockserver.url("/status?n=200")
- crawler = get_crawler(SingleRequestSpider, {
- "DOWNLOADER_MIDDLEWARES": {
- RaiseExceptionRequestMiddleware: 590,
- CatchExceptionOverrideRequestMiddleware: 595,
+ crawler = get_crawler(
+ SingleRequestSpider,
+ {
+ "DOWNLOADER_MIDDLEWARES": {
+ RaiseExceptionRequestMiddleware: 590,
+ CatchExceptionOverrideRequestMiddleware: 595,
+ },
},
- })
+ )
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.body, b"Caught ZeroDivisionError")
- self.assertEqual(response.request.url, OVERRIDEN_URL)
+ self.assertEqual(response.request.url, OVERRIDDEN_URL)
@defer.inlineCallbacks
def test_downloader_middleware_do_not_override_in_process_exception(self):
@@ -165,12 +174,15 @@ class CrawlTestCase(TestCase):
The spider callback should receive the original response.request
"""
url = self.mockserver.url("/status?n=200")
- crawler = get_crawler(SingleRequestSpider, {
- "DOWNLOADER_MIDDLEWARES": {
- RaiseExceptionRequestMiddleware: 590,
- CatchExceptionDoNotOverrideRequestMiddleware: 595,
+ crawler = get_crawler(
+ SingleRequestSpider,
+ {
+ "DOWNLOADER_MIDDLEWARES": {
+ RaiseExceptionRequestMiddleware: 590,
+ CatchExceptionDoNotOverrideRequestMiddleware: 595,
+ },
},
- })
+ )
yield crawler.crawl(seed=url, mockserver=self.mockserver)
response = crawler.spider.meta["responses"][0]
self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -182,16 +194,23 @@ class CrawlTestCase(TestCase):
Downloader middleware which returns a response with a
specific 'request' attribute, with an alternative callback
"""
- crawler = get_crawler(AlternativeCallbacksSpider, {
- "DOWNLOADER_MIDDLEWARES": {
- AlternativeCallbacksMiddleware: 595,
- }
- })
+ crawler = get_crawler(
+ AlternativeCallbacksSpider,
+ {
+ "DOWNLOADER_MIDDLEWARES": {
+ AlternativeCallbacksMiddleware: 595,
+ }
+ },
+ )
with LogCapture() as log:
url = self.mockserver.url("/status?n=200")
yield crawler.crawl(seed=url, mockserver=self.mockserver)
log.check_present(
- ("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"),
+ (
+ "alternative_callbacks_spider",
+ "INFO",
+ "alt_callback was invoked with foo=bar",
+ ),
)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 63e3aac00..7299972f6 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,25 +1,28 @@
+from typing import List
+
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy.http import Request
from scrapy.utils.test import get_crawler
-from tests.spiders import MockServerSpider
from tests.mockserver import MockServer
+from tests.spiders import MockServerSpider
class InjectArgumentsDownloaderMiddleware:
"""
Make sure downloader middlewares are able to update the keyword arguments
"""
+
def process_request(self, request, spider):
- if request.callback.__name__ == 'parse_downloader_mw':
- request.cb_kwargs['from_process_request'] = True
+ if request.callback.__name__ == "parse_downloader_mw":
+ request.cb_kwargs["from_process_request"] = True
return None
def process_response(self, request, response, spider):
- if request.callback.__name__ == 'parse_downloader_mw':
- request.cb_kwargs['from_process_response'] = True
+ if request.callback.__name__ == "parse_downloader_mw":
+ request.cb_kwargs["from_process_response"] = True
return response
@@ -27,83 +30,98 @@ class InjectArgumentsSpiderMiddleware:
"""
Make sure spider middlewares are able to update the keyword arguments
"""
+
def process_start_requests(self, start_requests, spider):
for request in start_requests:
- if request.callback.__name__ == 'parse_spider_mw':
- request.cb_kwargs['from_process_start_requests'] = True
+ if request.callback.__name__ == "parse_spider_mw":
+ request.cb_kwargs["from_process_start_requests"] = True
yield request
def process_spider_input(self, response, spider):
request = response.request
- if request.callback.__name__ == 'parse_spider_mw':
- request.cb_kwargs['from_process_spider_input'] = True
+ if request.callback.__name__ == "parse_spider_mw":
+ request.cb_kwargs["from_process_spider_input"] = True
return None
def process_spider_output(self, response, result, spider):
for element in result:
- if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':
- element.cb_kwargs['from_process_spider_output'] = True
+ if (
+ isinstance(element, Request)
+ and element.callback.__name__ == "parse_spider_mw_2"
+ ):
+ element.cb_kwargs["from_process_spider_output"] = True
yield element
class KeywordArgumentsSpider(MockServerSpider):
- name = 'kwargs'
+ name = "kwargs"
custom_settings = {
- 'DOWNLOADER_MIDDLEWARES': {
+ "DOWNLOADER_MIDDLEWARES": {
InjectArgumentsDownloaderMiddleware: 750,
},
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
InjectArgumentsSpiderMiddleware: 750,
},
}
- checks = []
+ checks: List[bool] = []
def start_requests(self):
- data = {'key': 'value', 'number': 123, 'callback': 'some_callback'}
- yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
- yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
- yield Request(self.mockserver.url('/general_without'), self.parse_general)
- yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
- yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
- yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
- yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
- yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
- yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)
+ data = {"key": "value", "number": 123, "callback": "some_callback"}
+ yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data)
+ yield Request(
+ self.mockserver.url("/general_with"), self.parse_general, cb_kwargs=data
+ )
+ yield Request(self.mockserver.url("/general_without"), self.parse_general)
+ yield Request(self.mockserver.url("/no_kwargs"), self.parse_no_kwargs)
+ yield Request(
+ self.mockserver.url("/default"), self.parse_default, cb_kwargs=data
+ )
+ yield Request(
+ self.mockserver.url("/takes_less"), self.parse_takes_less, cb_kwargs=data
+ )
+ yield Request(
+ self.mockserver.url("/takes_more"), self.parse_takes_more, cb_kwargs=data
+ )
+ yield Request(self.mockserver.url("/downloader_mw"), self.parse_downloader_mw)
+ yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw)
def parse_first(self, response, key, number):
- self.checks.append(key == 'value')
+ self.checks.append(key == "value")
self.checks.append(number == 123)
- self.crawler.stats.inc_value('boolean_checks', 2)
+ self.crawler.stats.inc_value("boolean_checks", 2)
yield response.follow(
- self.mockserver.url('/two'),
+ self.mockserver.url("/two"),
self.parse_second,
- cb_kwargs={'new_key': 'new_value'})
+ cb_kwargs={"new_key": "new_value"},
+ )
def parse_second(self, response, new_key):
- self.checks.append(new_key == 'new_value')
- self.crawler.stats.inc_value('boolean_checks')
+ self.checks.append(new_key == "new_value")
+ self.crawler.stats.inc_value("boolean_checks")
def parse_general(self, response, **kwargs):
- if response.url.endswith('/general_with'):
- self.checks.append(kwargs['key'] == 'value')
- self.checks.append(kwargs['number'] == 123)
- self.checks.append(kwargs['callback'] == 'some_callback')
- self.crawler.stats.inc_value('boolean_checks', 3)
- elif response.url.endswith('/general_without'):
- self.checks.append(kwargs == {}) # pylint: disable=use-implicit-booleaness-not-comparison
- self.crawler.stats.inc_value('boolean_checks')
+ if response.url.endswith("/general_with"):
+ self.checks.append(kwargs["key"] == "value")
+ self.checks.append(kwargs["number"] == 123)
+ self.checks.append(kwargs["callback"] == "some_callback")
+ self.crawler.stats.inc_value("boolean_checks", 3)
+ elif response.url.endswith("/general_without"):
+ self.checks.append(
+ kwargs == {} # pylint: disable=use-implicit-booleaness-not-comparison
+ )
+ self.crawler.stats.inc_value("boolean_checks")
def parse_no_kwargs(self, response):
- self.checks.append(response.url.endswith('/no_kwargs'))
- self.crawler.stats.inc_value('boolean_checks')
+ self.checks.append(response.url.endswith("/no_kwargs"))
+ self.crawler.stats.inc_value("boolean_checks")
def parse_default(self, response, key, number=None, default=99):
- self.checks.append(response.url.endswith('/default'))
- self.checks.append(key == 'value')
+ self.checks.append(response.url.endswith("/default"))
+ self.checks.append(key == "value")
self.checks.append(number == 123)
self.checks.append(default == 99)
- self.crawler.stats.inc_value('boolean_checks', 4)
+ self.crawler.stats.inc_value("boolean_checks", 4)
def parse_takes_less(self, response, key, callback):
"""
@@ -117,24 +135,27 @@ class KeywordArgumentsSpider(MockServerSpider):
TypeError: parse_takes_more() missing 1 required positional argument: 'other'
"""
- def parse_downloader_mw(self, response, from_process_request, from_process_response):
+ def parse_downloader_mw(
+ self, response, from_process_request, from_process_response
+ ):
self.checks.append(bool(from_process_request))
self.checks.append(bool(from_process_response))
- self.crawler.stats.inc_value('boolean_checks', 2)
+ self.crawler.stats.inc_value("boolean_checks", 2)
- def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):
+ def parse_spider_mw(
+ self, response, from_process_spider_input, from_process_start_requests
+ ):
self.checks.append(bool(from_process_spider_input))
self.checks.append(bool(from_process_start_requests))
- self.crawler.stats.inc_value('boolean_checks', 2)
- return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)
+ self.crawler.stats.inc_value("boolean_checks", 2)
+ return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)
def parse_spider_mw_2(self, response, from_process_spider_output):
self.checks.append(bool(from_process_spider_output))
- self.crawler.stats.inc_value('boolean_checks', 1)
+ self.crawler.stats.inc_value("boolean_checks", 1)
class CallbackKeywordArgumentsTestCase(TestCase):
-
maxDiff = None
def setUp(self):
@@ -150,24 +171,26 @@ class CallbackKeywordArgumentsTestCase(TestCase):
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver)
self.assertTrue(all(crawler.spider.checks))
- self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+ self.assertEqual(
+ len(crawler.spider.checks), crawler.stats.get_value("boolean_checks")
+ )
# check exceptions for argument mismatch
exceptions = {}
for line in log.records:
- for key in ('takes_less', 'takes_more'):
+ for key in ("takes_less", "takes_more"):
if key in line.getMessage():
exceptions[key] = line
- self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+ self.assertEqual(exceptions["takes_less"].exc_info[0], TypeError)
self.assertTrue(
- str(exceptions['takes_less'].exc_info[1]).endswith(
+ str(exceptions["takes_less"].exc_info[1]).endswith(
"parse_takes_less() got an unexpected keyword argument 'number'"
),
- msg="Exception message: " + str(exceptions['takes_less'].exc_info[1]),
+ msg="Exception message: " + str(exceptions["takes_less"].exc_info[1]),
)
- self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+ self.assertEqual(exceptions["takes_more"].exc_info[0], TypeError)
self.assertTrue(
- str(exceptions['takes_more'].exc_info[1]).endswith(
+ str(exceptions["takes_more"].exc_info[1]).endswith(
"parse_takes_more() missing 1 required positional argument: 'other'"
),
- msg="Exception message: " + str(exceptions['takes_more'].exc_info[1]),
+ msg="Exception message: " + str(exceptions["takes_more"].exc_info[1]),
)
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index 5bdcb975b..7312eb036 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -1,10 +1,6 @@
-import sys
import unittest
-import warnings
-from contextlib import suppress
-from scrapy import Spider, Request
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy import Request, Spider
from scrapy.http import FormRequest, JsonRequest
from scrapy.utils.request import request_from_dict
@@ -14,7 +10,6 @@ class CustomRequest(Request):
class RequestSerializationTest(unittest.TestCase):
-
def setUp(self):
self.spider = TestSpider()
@@ -29,13 +24,13 @@ class RequestSerializationTest(unittest.TestCase):
errback=self.spider.handle_error,
method="POST",
body=b"some body",
- headers={'content-encoding': 'text/html; charset=latin-1'},
- cookies={'currency': 'руб'},
- encoding='latin-1',
+ headers={"content-encoding": "text/html; charset=latin-1"},
+ cookies={"currency": "руб"},
+ encoding="latin-1",
priority=20,
- meta={'a': 'b'},
- cb_kwargs={'k': 'v'},
- flags=['testFlag'],
+ meta={"a": "b"},
+ cb_kwargs={"k": "v"},
+ flags=["testFlag"],
)
self._assert_serializes_ok(r, spider=self.spider)
@@ -80,46 +75,57 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r3, spider=self.spider)
def test_callback_serialization(self):
- r = Request("http://www.example.com", callback=self.spider.parse_item,
- errback=self.spider.handle_error)
+ r = Request(
+ "http://www.example.com",
+ callback=self.spider.parse_item,
+ errback=self.spider.handle_error,
+ )
self._assert_serializes_ok(r, spider=self.spider)
def test_reference_callback_serialization(self):
- r = Request("http://www.example.com",
- callback=self.spider.parse_item_reference,
- errback=self.spider.handle_error_reference)
+ r = Request(
+ "http://www.example.com",
+ callback=self.spider.parse_item_reference,
+ errback=self.spider.handle_error_reference,
+ )
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
- self.assertEqual(request_dict['callback'], 'parse_item_reference')
- self.assertEqual(request_dict['errback'], 'handle_error_reference')
+ self.assertEqual(request_dict["callback"], "parse_item_reference")
+ self.assertEqual(request_dict["errback"], "handle_error_reference")
def test_private_reference_callback_serialization(self):
- r = Request("http://www.example.com",
- callback=self.spider._TestSpider__parse_item_reference,
- errback=self.spider._TestSpider__handle_error_reference)
+ r = Request(
+ "http://www.example.com",
+ callback=self.spider._TestSpider__parse_item_reference,
+ errback=self.spider._TestSpider__handle_error_reference,
+ )
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
- self.assertEqual(request_dict['callback'],
- '_TestSpider__parse_item_reference')
- self.assertEqual(request_dict['errback'],
- '_TestSpider__handle_error_reference')
+ self.assertEqual(request_dict["callback"], "_TestSpider__parse_item_reference")
+ self.assertEqual(request_dict["errback"], "_TestSpider__handle_error_reference")
def test_private_callback_serialization(self):
- r = Request("http://www.example.com",
- callback=self.spider._TestSpider__parse_item_private,
- errback=self.spider.handle_error)
+ r = Request(
+ "http://www.example.com",
+ callback=self.spider._TestSpider__parse_item_private,
+ errback=self.spider.handle_error,
+ )
self._assert_serializes_ok(r, spider=self.spider)
def test_mixin_private_callback_serialization(self):
- r = Request("http://www.example.com",
- callback=self.spider._TestSpiderMixin__mixin_callback,
- errback=self.spider.handle_error)
+ r = Request(
+ "http://www.example.com",
+ callback=self.spider._TestSpiderMixin__mixin_callback,
+ errback=self.spider.handle_error,
+ )
self._assert_serializes_ok(r, spider=self.spider)
def test_delegated_callback_serialization(self):
- r = Request("http://www.example.com",
- callback=self.spider.delegated_callback,
- errback=self.spider.handle_error)
+ r = Request(
+ "http://www.example.com",
+ callback=self.spider.delegated_callback,
+ errback=self.spider.handle_error,
+ )
self._assert_serializes_ok(r, spider=self.spider)
def test_unserializable_callback1(self):
@@ -134,14 +140,14 @@ class RequestSerializationTest(unittest.TestCase):
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
- name = 'my_spider'
+ name = "my_spider"
def parse(self, response):
pass
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
- setattr(spider, 'parse', None)
+ setattr(spider, "parse", None)
self.assertRaises(ValueError, r.to_dict, spider=spider)
def test_callback_not_available(self):
@@ -152,27 +158,6 @@ class RequestSerializationTest(unittest.TestCase):
self.assertRaises(ValueError, request_from_dict, d, spider=Spider("foo"))
-class DeprecatedMethodsRequestSerializationTest(RequestSerializationTest):
- def _assert_serializes_ok(self, request, spider=None):
- with warnings.catch_warnings(record=True) as caught:
- warnings.simplefilter("always")
- with suppress(KeyError):
- del sys.modules["scrapy.utils.reqser"] # delete module to reset the deprecation warning
-
- from scrapy.utils.reqser import request_from_dict as _from_dict, request_to_dict as _to_dict
-
- request_copy = _from_dict(_to_dict(request, spider), spider)
- self._assert_same_request(request, request_copy)
-
- self.assertEqual(len(caught), 1)
- self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
- self.assertEqual(
- "Module scrapy.utils.reqser is deprecated, please use request.to_dict method"
- " and/or scrapy.utils.request.request_from_dict instead",
- str(caught[0].message),
- )
-
-
class TestSpiderMixin:
def __mixin_callback(self, response):
pass
@@ -200,7 +185,7 @@ def private_handle_error(failure):
class TestSpider(Spider, TestSpiderMixin):
- name = 'test'
+ name = "test"
parse_item_reference = parse_item
handle_error_reference = handle_error
__parse_item_reference = private_parse_item
diff --git a/tests/test_request_left.py b/tests/test_request_left.py
index 4d4483881..54155f7ef 100644
--- a/tests/test_request_left.py
+++ b/tests/test_request_left.py
@@ -1,5 +1,6 @@
from twisted.internet import defer
from twisted.trial.unittest import TestCase
+
from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
@@ -7,12 +8,11 @@ from tests.mockserver import MockServer
class SignalCatcherSpider(Spider):
- name = 'signal_catcher'
+ name = "signal_catcher"
def __init__(self, crawler, url, *args, **kwargs):
super().__init__(*args, **kwargs)
- crawler.signals.connect(self.on_request_left,
- signal=request_left_downloader)
+ crawler.signals.connect(self.on_request_left, signal=request_left_downloader)
self.caught_times = 0
self.start_urls = [url]
@@ -26,7 +26,6 @@ class SignalCatcherSpider(Spider):
class TestCatching(TestCase):
-
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
@@ -42,8 +41,7 @@ class TestCatching(TestCase):
@defer.inlineCallbacks
def test_timeout(self):
- crawler = get_crawler(SignalCatcherSpider,
- {'DOWNLOAD_TIMEOUT': 0.1})
+ crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
self.assertEqual(crawler.spider.caught_times, 1)
@@ -56,5 +54,5 @@ class TestCatching(TestCase):
@defer.inlineCallbacks
def test_noconnect(self):
crawler = get_crawler(SignalCatcherSpider)
- yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
+ yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
self.assertEqual(crawler.spider.caught_times, 1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 42d4e481d..7d769cfda 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -5,15 +5,14 @@ import pytest
from scrapy.http import (
Headers,
HtmlResponse,
+ JsonResponse,
Response,
TextResponse,
XmlResponse,
)
from scrapy.responsetypes import responsetypes
-from .test_utils_response import (
- POST_XTRACTMIME_SCENARIOS,
- PRE_XTRACTMIME_SCENARIOS,
-)
+
+from .test_utils_response import POST_XTRACTMIME_SCENARIOS, PRE_XTRACTMIME_SCENARIOS
def _unmark(item):
@@ -50,15 +49,14 @@ def test_from_args(kwargs, response_class):
class ResponseTypesTest(unittest.TestCase):
-
def test_from_filename(self):
mappings = [
- ('data.bin', Response),
- ('file.txt', TextResponse),
- ('file.xml.gz', Response),
- ('file.xml', XmlResponse),
- ('file.html', HtmlResponse),
- ('file.unknownext', Response),
+ ("data.bin", Response),
+ ("file.txt", TextResponse),
+ ("file.xml.gz", Response),
+ ("file.xml", XmlResponse),
+ ("file.html", HtmlResponse),
+ ("file.unknownext", Response),
]
for source, cls in mappings:
retcls = responsetypes.from_filename(source)
@@ -67,13 +65,12 @@ class ResponseTypesTest(unittest.TestCase):
def test_from_content_disposition(self):
mappings = [
(b'attachment; filename="data.xml"', XmlResponse),
- (b'attachment; filename=data.xml', XmlResponse),
- ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
- ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
- ('attachment;filename=data高.doc'.encode('gbk'), Response),
- ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
- ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
-
+ (b"attachment; filename=data.xml", XmlResponse),
+ ("attachment;filename=data£.tar.gz".encode("utf-8"), Response),
+ ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
+ ("attachment;filename=data高.doc".encode("gbk"), Response),
+ ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
+ ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse),
]
for source, cls in mappings:
retcls = responsetypes.from_content_disposition(source)
@@ -81,14 +78,16 @@ class ResponseTypesTest(unittest.TestCase):
def test_from_content_type(self):
mappings = [
- ('text/html; charset=UTF-8', HtmlResponse),
- ('text/xml; charset=UTF-8', XmlResponse),
- ('application/xhtml+xml; charset=UTF-8', HtmlResponse),
- ('application/vnd.wap.xhtml+xml; charset=utf-8', HtmlResponse),
- ('application/xml; charset=UTF-8', XmlResponse),
- ('application/octet-stream', Response),
- ('application/x-json; encoding=UTF8;charset=UTF-8', TextResponse),
- ('application/json-amazonui-streaming;charset=UTF-8', TextResponse),
+ ("text/html; charset=UTF-8", HtmlResponse),
+ ("text/xml; charset=UTF-8", XmlResponse),
+ ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
+ ("application/vnd.wap.xhtml+xml; charset=utf-8", HtmlResponse),
+ ("application/xml; charset=UTF-8", XmlResponse),
+ ("application/octet-stream", Response),
+ ("application/json; encoding=UTF8;charset=UTF-8", JsonResponse),
+ ("application/x-json; encoding=UTF8;charset=UTF-8", JsonResponse),
+ ("application/json-amazonui-streaming;charset=UTF-8", JsonResponse),
+ (b"application/x-download; filename=\x80dummy.txt", Response),
]
for source, cls in mappings:
retcls = responsetypes.from_content_type(source)
@@ -96,11 +95,11 @@ class ResponseTypesTest(unittest.TestCase):
def test_from_body(self):
mappings = [
- (b'\x03\x02\xdf\xdd\x23', Response),
- (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
- (b'<html><head><title>Hello ', HtmlResponse),
+ (b"\x03\x02\xdf\xdd\x23", Response),
+ (b"Some plain text\ndata with tabs\t and null bytes\0", TextResponse),
+ (b"Hello ", HtmlResponse),
# https://codersblock.com/blog/the-smallest-valid-html5-page/
- (b'\n. ', HtmlResponse),
+ (b"\n. ", HtmlResponse),
(b'= version.parse("1.8.0")
+
class SelectorTestCase(unittest.TestCase):
-
def test_simple_selection(self):
"""Simple selector tests"""
body = b""
- response = TextResponse(url="http://example.com", body=body, encoding='utf-8')
+ response = TextResponse(url="http://example.com", body=body, encoding="utf-8")
sel = Selector(response)
- xl = sel.xpath('//input')
+ xl = sel.xpath("//input")
self.assertEqual(2, len(xl))
for x in xl:
assert isinstance(x, Selector)
self.assertEqual(
- sel.xpath('//input').getall(),
- [x.get() for x in sel.xpath('//input')]
+ sel.xpath("//input").getall(), [x.get() for x in sel.xpath("//input")]
)
self.assertEqual(
- [x.get() for x in sel.xpath("//input[@name='a']/@name")],
- ['a']
+ [x.get() for x in sel.xpath("//input[@name='a']/@name")], ["a"]
)
self.assertEqual(
- [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
- ['12.0']
+ [
+ x.get()
+ for x in sel.xpath(
+ "number(concat(//input[@name='a']/@value, //input[@name='b']/@value))"
+ )
+ ],
+ ["12.0"],
)
+ self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), ["xpathrules"])
self.assertEqual(
- sel.xpath("concat('xpath', 'rules')").getall(),
- ['xpathrules']
- )
- self.assertEqual(
- [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
- ['12']
+ [
+ x.get()
+ for x in sel.xpath(
+ "concat(//input[@name='a']/@value, //input[@name='b']/@value)"
+ )
+ ],
+ ["12"],
)
def test_root_base_url(self):
body = b''
url = "http://example.com"
- response = TextResponse(url=url, body=body, encoding='utf-8')
+ response = TextResponse(url=url, body=body, encoding="utf-8")
sel = Selector(response)
self.assertEqual(url, sel.root.base)
def test_flavor_detection(self):
text = b'
'
- sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
- self.assertEqual(sel.type, 'xml')
- self.assertEqual(sel.xpath("//div").getall(),
- [''])
+ sel = Selector(XmlResponse("http://example.com", body=text, encoding="utf-8"))
+ self.assertEqual(sel.type, "xml")
+ self.assertEqual(
+ sel.xpath("//div").getall(),
+ [''],
+ )
- sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
- self.assertEqual(sel.type, 'html')
- self.assertEqual(sel.xpath("//div").getall(),
- [''])
+ sel = Selector(HtmlResponse("http://example.com", body=text, encoding="utf-8"))
+ self.assertEqual(sel.type, "html")
+ self.assertEqual(
+ sel.xpath("//div").getall(), ['']
+ )
def test_http_header_encoding_precedence(self):
# '\xa3' = pound symbol in unicode
# '\xc2\xa3' = pound symbol in utf-8
# '\xa3' = pound symbol in latin-1 (iso-8859-1)
- meta = ''
- head = '' + meta + ''
+ meta = (
+ ''
+ )
+ head = "" + meta + ""
body_content = '\xa3'
- body = '' + body_content + ''
- html = '' + head + body + ''
- encoding = 'utf-8'
+ body = "" + body_content + ""
+ html = "" + head + body + ""
+ encoding = "utf-8"
html_utf8 = html.encode(encoding)
- headers = {'Content-Type': ['text/html; charset=utf-8']}
- response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
+ headers = {"Content-Type": ["text/html; charset=utf-8"]}
+ response = HtmlResponse(
+ url="http://example.com", headers=headers, body=html_utf8
+ )
x = Selector(response)
- self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
+ self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ["\xa3"])
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence
- r1 = TextResponse('http://www.example.com',
- body=b'")
+ self.assertEqual(
+ repr(self.attribute), ""
+ )
class BaseSettingsTest(unittest.TestCase):
-
def setUp(self):
self.settings = BaseSettings()
- def test_set_new_attribute(self):
- self.settings.set('TEST_OPTION', 'value', 0)
- self.assertIn('TEST_OPTION', self.settings.attributes)
+ def test_setdefault_not_existing_value(self):
+ settings = BaseSettings()
+ value = settings.setdefault("TEST_OPTION", "value")
+ self.assertEqual(settings["TEST_OPTION"], "value")
+ self.assertEqual(value, "value")
+ self.assertIsNotNone(value)
- attr = self.settings.attributes['TEST_OPTION']
+ def test_setdefault_existing_value(self):
+ settings = BaseSettings({"TEST_OPTION": "value"})
+ value = settings.setdefault("TEST_OPTION", None)
+ self.assertEqual(settings["TEST_OPTION"], "value")
+ self.assertEqual(value, "value")
+
+ def test_set_new_attribute(self):
+ self.settings.set("TEST_OPTION", "value", 0)
+ self.assertIn("TEST_OPTION", self.settings.attributes)
+
+ attr = self.settings.attributes["TEST_OPTION"]
self.assertIsInstance(attr, SettingsAttribute)
- self.assertEqual(attr.value, 'value')
+ self.assertEqual(attr.value, "value")
self.assertEqual(attr.priority, 0)
def test_set_settingsattribute(self):
myattr = SettingsAttribute(0, 30) # Note priority 30
- self.settings.set('TEST_ATTR', myattr, 10)
- self.assertEqual(self.settings.get('TEST_ATTR'), 0)
- self.assertEqual(self.settings.getpriority('TEST_ATTR'), 30)
+ self.settings.set("TEST_ATTR", myattr, 10)
+ self.assertEqual(self.settings.get("TEST_ATTR"), 0)
+ self.assertEqual(self.settings.getpriority("TEST_ATTR"), 30)
def test_set_instance_identity_on_update(self):
- attr = SettingsAttribute('value', 0)
- self.settings.attributes = {'TEST_OPTION': attr}
- self.settings.set('TEST_OPTION', 'othervalue', 10)
+ attr = SettingsAttribute("value", 0)
+ self.settings.attributes = {"TEST_OPTION": attr}
+ self.settings.set("TEST_OPTION", "othervalue", 10)
- self.assertIn('TEST_OPTION', self.settings.attributes)
- self.assertIs(attr, self.settings.attributes['TEST_OPTION'])
+ self.assertIn("TEST_OPTION", self.settings.attributes)
+ self.assertIs(attr, self.settings.attributes["TEST_OPTION"])
def test_set_calls_settings_attributes_methods_on_update(self):
- attr = SettingsAttribute('value', 10)
- with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set:
-
- self.settings.attributes = {'TEST_OPTION': attr}
+ attr = SettingsAttribute("value", 10)
+ with mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object(
+ attr, "set"
+ ) as mock_set:
+ self.settings.attributes = {"TEST_OPTION": attr}
for priority in (0, 10, 20):
- self.settings.set('TEST_OPTION', 'othervalue', priority)
- mock_set.assert_called_once_with('othervalue', priority)
+ self.settings.set("TEST_OPTION", "othervalue", priority)
+ mock_set.assert_called_once_with("othervalue", priority)
self.assertFalse(mock_setattr.called)
mock_set.reset_mock()
mock_setattr.reset_mock()
def test_setitem(self):
settings = BaseSettings()
- settings.set('key', 'a', 'default')
- settings['key'] = 'b'
- self.assertEqual(settings['key'], 'b')
- self.assertEqual(settings.getpriority('key'), 20)
- settings['key'] = 'c'
- self.assertEqual(settings['key'], 'c')
- settings['key2'] = 'x'
- self.assertIn('key2', settings)
- self.assertEqual(settings['key2'], 'x')
- self.assertEqual(settings.getpriority('key2'), 20)
+ settings.set("key", "a", "default")
+ settings["key"] = "b"
+ self.assertEqual(settings["key"], "b")
+ self.assertEqual(settings.getpriority("key"), 20)
+ settings["key"] = "c"
+ self.assertEqual(settings["key"], "c")
+ settings["key2"] = "x"
+ self.assertIn("key2", settings)
+ self.assertEqual(settings["key2"], "x")
+ self.assertEqual(settings.getpriority("key2"), 20)
def test_setdict_alias(self):
- with mock.patch.object(self.settings, 'set') as mock_set:
- self.settings.setdict({'TEST_1': 'value1', 'TEST_2': 'value2'}, 10)
+ with mock.patch.object(self.settings, "set") as mock_set:
+ self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10)
self.assertEqual(mock_set.call_count, 2)
- calls = [mock.call('TEST_1', 'value1', 10),
- mock.call('TEST_2', 'value2', 10)]
+ calls = [
+ mock.call("TEST_1", "value1", 10),
+ mock.call("TEST_2", "value2", 10),
+ ]
mock_set.assert_has_calls(calls, any_order=True)
def test_setmodule_only_load_uppercase_vars(self):
- class ModuleMock():
- UPPERCASE_VAR = 'value'
- MIXEDcase_VAR = 'othervalue'
- lowercase_var = 'anothervalue'
+ class ModuleMock:
+ UPPERCASE_VAR = "value"
+ MIXEDcase_VAR = "othervalue"
+ lowercase_var = "anothervalue"
self.settings.attributes = {}
self.settings.setmodule(ModuleMock(), 10)
- self.assertIn('UPPERCASE_VAR', self.settings.attributes)
- self.assertNotIn('MIXEDcase_VAR', self.settings.attributes)
- self.assertNotIn('lowercase_var', self.settings.attributes)
+ self.assertIn("UPPERCASE_VAR", self.settings.attributes)
+ self.assertNotIn("MIXEDcase_VAR", self.settings.attributes)
+ self.assertNotIn("lowercase_var", self.settings.attributes)
self.assertEqual(len(self.settings.attributes), 1)
def test_setmodule_alias(self):
- with mock.patch.object(self.settings, 'set') as mock_set:
+ with mock.patch.object(self.settings, "set") as mock_set:
self.settings.setmodule(default_settings, 10)
- mock_set.assert_any_call('TEST_DEFAULT', 'defvalue', 10)
- mock_set.assert_any_call('TEST_DICT', {'key': 'val'}, 10)
+ mock_set.assert_any_call("TEST_DEFAULT", "defvalue", 10)
+ mock_set.assert_any_call("TEST_DICT", {"key": "val"}, 10)
def test_setmodule_by_path(self):
self.settings.attributes = {}
@@ -143,11 +165,9 @@ class BaseSettingsTest(unittest.TestCase):
ctrl_attributes = self.settings.attributes.copy()
self.settings.attributes = {}
- self.settings.setmodule(
- 'tests.test_settings.default_settings', 10)
+ self.settings.setmodule("tests.test_settings.default_settings", 10)
- self.assertCountEqual(self.settings.attributes.keys(),
- ctrl_attributes.keys())
+ self.assertCountEqual(self.settings.attributes.keys(), ctrl_attributes.keys())
for key in ctrl_attributes.keys():
attr = self.settings.attributes[key]
@@ -156,181 +176,215 @@ class BaseSettingsTest(unittest.TestCase):
self.assertEqual(attr.priority, ctrl_attr.priority)
def test_update(self):
- settings = BaseSettings({'key_lowprio': 0}, priority=0)
- settings.set('key_highprio', 10, priority=50)
- custom_settings = BaseSettings({'key_lowprio': 1, 'key_highprio': 11},
- priority=30)
- custom_settings.set('newkey_one', None, priority=50)
- custom_dict = {'key_lowprio': 2, 'key_highprio': 12, 'newkey_two': None}
+ settings = BaseSettings({"key_lowprio": 0}, priority=0)
+ settings.set("key_highprio", 10, priority=50)
+ custom_settings = BaseSettings(
+ {"key_lowprio": 1, "key_highprio": 11}, priority=30
+ )
+ custom_settings.set("newkey_one", None, priority=50)
+ custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None}
settings.update(custom_dict, priority=20)
- self.assertEqual(settings['key_lowprio'], 2)
- self.assertEqual(settings.getpriority('key_lowprio'), 20)
- self.assertEqual(settings['key_highprio'], 10)
- self.assertIn('newkey_two', settings)
- self.assertEqual(settings.getpriority('newkey_two'), 20)
+ self.assertEqual(settings["key_lowprio"], 2)
+ self.assertEqual(settings.getpriority("key_lowprio"), 20)
+ self.assertEqual(settings["key_highprio"], 10)
+ self.assertIn("newkey_two", settings)
+ self.assertEqual(settings.getpriority("newkey_two"), 20)
settings.update(custom_settings)
- self.assertEqual(settings['key_lowprio'], 1)
- self.assertEqual(settings.getpriority('key_lowprio'), 30)
- self.assertEqual(settings['key_highprio'], 10)
- self.assertIn('newkey_one', settings)
- self.assertEqual(settings.getpriority('newkey_one'), 50)
+ self.assertEqual(settings["key_lowprio"], 1)
+ self.assertEqual(settings.getpriority("key_lowprio"), 30)
+ self.assertEqual(settings["key_highprio"], 10)
+ self.assertIn("newkey_one", settings)
+ self.assertEqual(settings.getpriority("newkey_one"), 50)
- settings.update({'key_lowprio': 3}, priority=20)
- self.assertEqual(settings['key_lowprio'], 1)
+ settings.update({"key_lowprio": 3}, priority=20)
+ self.assertEqual(settings["key_lowprio"], 1)
+
+ @pytest.mark.xfail(
+ raises=TypeError, reason="BaseSettings.update doesn't support kwargs input"
+ )
+ def test_update_kwargs(self):
+ settings = BaseSettings({"key": 0})
+ settings.update(key=1) # pylint: disable=unexpected-keyword-arg
+
+ @pytest.mark.xfail(
+ raises=AttributeError,
+ reason="BaseSettings.update doesn't support iterable input",
+ )
+ def test_update_iterable(self):
+ settings = BaseSettings({"key": 0})
+ settings.update([("key", 1)])
def test_update_jsonstring(self):
- settings = BaseSettings({'number': 0, 'dict': BaseSettings({'key': 'val'})})
+ settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})})
settings.update('{"number": 1, "newnumber": 2}')
- self.assertEqual(settings['number'], 1)
- self.assertEqual(settings['newnumber'], 2)
+ self.assertEqual(settings["number"], 1)
+ self.assertEqual(settings["newnumber"], 2)
settings.set("dict", '{"key": "newval", "newkey": "newval2"}')
- self.assertEqual(settings['dict']['key'], "newval")
- self.assertEqual(settings['dict']['newkey'], "newval2")
+ self.assertEqual(settings["dict"]["key"], "newval")
+ self.assertEqual(settings["dict"]["newkey"], "newval2")
def test_delete(self):
- settings = BaseSettings({'key': None})
- settings.set('key_highprio', None, priority=50)
- settings.delete('key')
- settings.delete('key_highprio')
- self.assertNotIn('key', settings)
- self.assertIn('key_highprio', settings)
- del settings['key_highprio']
- self.assertNotIn('key_highprio', settings)
+ settings = BaseSettings({"key": None})
+ settings.set("key_highprio", None, priority=50)
+ settings.delete("key")
+ settings.delete("key_highprio")
+ self.assertNotIn("key", settings)
+ self.assertIn("key_highprio", settings)
+ del settings["key_highprio"]
+ self.assertNotIn("key_highprio", settings)
+ with self.assertRaises(KeyError):
+ settings.delete("notkey")
+ with self.assertRaises(KeyError):
+ del settings["notkey"]
def test_get(self):
test_configuration = {
- 'TEST_ENABLED1': '1',
- 'TEST_ENABLED2': True,
- 'TEST_ENABLED3': 1,
- 'TEST_ENABLED4': 'True',
- 'TEST_ENABLED5': 'true',
- 'TEST_ENABLED_WRONG': 'on',
- 'TEST_DISABLED1': '0',
- 'TEST_DISABLED2': False,
- 'TEST_DISABLED3': 0,
- 'TEST_DISABLED4': 'False',
- 'TEST_DISABLED5': 'false',
- 'TEST_DISABLED_WRONG': 'off',
- 'TEST_INT1': 123,
- 'TEST_INT2': '123',
- 'TEST_FLOAT1': 123.45,
- 'TEST_FLOAT2': '123.45',
- 'TEST_LIST1': ['one', 'two'],
- 'TEST_LIST2': 'one,two',
- 'TEST_STR': 'value',
- 'TEST_DICT1': {'key1': 'val1', 'ke2': 3},
- 'TEST_DICT2': '{"key1": "val1", "ke2": 3}',
+ "TEST_ENABLED1": "1",
+ "TEST_ENABLED2": True,
+ "TEST_ENABLED3": 1,
+ "TEST_ENABLED4": "True",
+ "TEST_ENABLED5": "true",
+ "TEST_ENABLED_WRONG": "on",
+ "TEST_DISABLED1": "0",
+ "TEST_DISABLED2": False,
+ "TEST_DISABLED3": 0,
+ "TEST_DISABLED4": "False",
+ "TEST_DISABLED5": "false",
+ "TEST_DISABLED_WRONG": "off",
+ "TEST_INT1": 123,
+ "TEST_INT2": "123",
+ "TEST_FLOAT1": 123.45,
+ "TEST_FLOAT2": "123.45",
+ "TEST_LIST1": ["one", "two"],
+ "TEST_LIST2": "one,two",
+ "TEST_STR": "value",
+ "TEST_DICT1": {"key1": "val1", "ke2": 3},
+ "TEST_DICT2": '{"key1": "val1", "ke2": 3}',
}
settings = self.settings
- settings.attributes = {key: SettingsAttribute(value, 0) for key, value
- in test_configuration.items()}
+ settings.attributes = {
+ key: SettingsAttribute(value, 0)
+ for key, value in test_configuration.items()
+ }
- self.assertTrue(settings.getbool('TEST_ENABLED1'))
- self.assertTrue(settings.getbool('TEST_ENABLED2'))
- self.assertTrue(settings.getbool('TEST_ENABLED3'))
- self.assertTrue(settings.getbool('TEST_ENABLED4'))
- self.assertTrue(settings.getbool('TEST_ENABLED5'))
- self.assertFalse(settings.getbool('TEST_ENABLEDx'))
- self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
- self.assertFalse(settings.getbool('TEST_DISABLED1'))
- self.assertFalse(settings.getbool('TEST_DISABLED2'))
- self.assertFalse(settings.getbool('TEST_DISABLED3'))
- self.assertFalse(settings.getbool('TEST_DISABLED4'))
- self.assertFalse(settings.getbool('TEST_DISABLED5'))
- self.assertEqual(settings.getint('TEST_INT1'), 123)
- self.assertEqual(settings.getint('TEST_INT2'), 123)
- self.assertEqual(settings.getint('TEST_INTx'), 0)
- self.assertEqual(settings.getint('TEST_INTx', 45), 45)
- self.assertEqual(settings.getfloat('TEST_FLOAT1'), 123.45)
- self.assertEqual(settings.getfloat('TEST_FLOAT2'), 123.45)
- self.assertEqual(settings.getfloat('TEST_FLOATx'), 0.0)
- self.assertEqual(settings.getfloat('TEST_FLOATx', 55.0), 55.0)
- self.assertEqual(settings.getlist('TEST_LIST1'), ['one', 'two'])
- self.assertEqual(settings.getlist('TEST_LIST2'), ['one', 'two'])
- self.assertEqual(settings.getlist('TEST_LISTx'), [])
- self.assertEqual(settings.getlist('TEST_LISTx', ['default']), ['default'])
- self.assertEqual(settings['TEST_STR'], 'value')
- self.assertEqual(settings.get('TEST_STR'), 'value')
- self.assertEqual(settings['TEST_STRx'], None)
- self.assertEqual(settings.get('TEST_STRx'), None)
- self.assertEqual(settings.get('TEST_STRx', 'default'), 'default')
- self.assertEqual(settings.getdict('TEST_DICT1'), {'key1': 'val1', 'ke2': 3})
- self.assertEqual(settings.getdict('TEST_DICT2'), {'key1': 'val1', 'ke2': 3})
- self.assertEqual(settings.getdict('TEST_DICT3'), {})
- self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
- self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
- self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG')
- self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG')
+ self.assertTrue(settings.getbool("TEST_ENABLED1"))
+ self.assertTrue(settings.getbool("TEST_ENABLED2"))
+ self.assertTrue(settings.getbool("TEST_ENABLED3"))
+ self.assertTrue(settings.getbool("TEST_ENABLED4"))
+ self.assertTrue(settings.getbool("TEST_ENABLED5"))
+ self.assertFalse(settings.getbool("TEST_ENABLEDx"))
+ self.assertTrue(settings.getbool("TEST_ENABLEDx", True))
+ self.assertFalse(settings.getbool("TEST_DISABLED1"))
+ self.assertFalse(settings.getbool("TEST_DISABLED2"))
+ self.assertFalse(settings.getbool("TEST_DISABLED3"))
+ self.assertFalse(settings.getbool("TEST_DISABLED4"))
+ self.assertFalse(settings.getbool("TEST_DISABLED5"))
+ self.assertEqual(settings.getint("TEST_INT1"), 123)
+ self.assertEqual(settings.getint("TEST_INT2"), 123)
+ self.assertEqual(settings.getint("TEST_INTx"), 0)
+ self.assertEqual(settings.getint("TEST_INTx", 45), 45)
+ self.assertEqual(settings.getfloat("TEST_FLOAT1"), 123.45)
+ self.assertEqual(settings.getfloat("TEST_FLOAT2"), 123.45)
+ self.assertEqual(settings.getfloat("TEST_FLOATx"), 0.0)
+ self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0)
+ self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"])
+ self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"])
+ self.assertEqual(settings.getlist("TEST_LISTx"), [])
+ self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"])
+ self.assertEqual(settings["TEST_STR"], "value")
+ self.assertEqual(settings.get("TEST_STR"), "value")
+ self.assertEqual(settings["TEST_STRx"], None)
+ self.assertEqual(settings.get("TEST_STRx"), None)
+ self.assertEqual(settings.get("TEST_STRx", "default"), "default")
+ self.assertEqual(settings.getdict("TEST_DICT1"), {"key1": "val1", "ke2": 3})
+ self.assertEqual(settings.getdict("TEST_DICT2"), {"key1": "val1", "ke2": 3})
+ self.assertEqual(settings.getdict("TEST_DICT3"), {})
+ self.assertEqual(settings.getdict("TEST_DICT3", {"key1": 5}), {"key1": 5})
+ self.assertRaises(ValueError, settings.getdict, "TEST_LIST1")
+ self.assertRaises(ValueError, settings.getbool, "TEST_ENABLED_WRONG")
+ self.assertRaises(ValueError, settings.getbool, "TEST_DISABLED_WRONG")
def test_getpriority(self):
- settings = BaseSettings({'key': 'value'}, priority=99)
- self.assertEqual(settings.getpriority('key'), 99)
- self.assertEqual(settings.getpriority('nonexistentkey'), None)
+ settings = BaseSettings({"key": "value"}, priority=99)
+ self.assertEqual(settings.getpriority("key"), 99)
+ self.assertEqual(settings.getpriority("nonexistentkey"), None)
def test_getwithbase(self):
- s = BaseSettings({'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
- 'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
- 'HASNOBASE': BaseSettings({3: 3000}, 'default')})
- s['TEST'].set(2, 200, 'cmdline')
- self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
- self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
- self.assertEqual(s.getwithbase('NONEXISTENT'), {})
+ s = BaseSettings(
+ {
+ "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),
+ "TEST": BaseSettings({1: 10, 3: 30}, "default"),
+ "HASNOBASE": BaseSettings({3: 3000}, "default"),
+ }
+ )
+ s["TEST"].set(2, 200, "cmdline")
+ self.assertCountEqual(s.getwithbase("TEST"), {1: 1, 2: 200, 3: 30})
+ self.assertCountEqual(s.getwithbase("HASNOBASE"), s["HASNOBASE"])
+ self.assertEqual(s.getwithbase("NONEXISTENT"), {})
def test_maxpriority(self):
# Empty settings should return 'default'
self.assertEqual(self.settings.maxpriority(), 0)
- self.settings.set('A', 0, 10)
- self.settings.set('B', 0, 30)
+ self.settings.set("A", 0, 10)
+ self.settings.set("B", 0, 30)
self.assertEqual(self.settings.maxpriority(), 30)
def test_copy(self):
values = {
- 'TEST_BOOL': True,
- 'TEST_LIST': ['one', 'two'],
- 'TEST_LIST_OF_LISTS': [['first_one', 'first_two'],
- ['second_one', 'second_two']]
+ "TEST_BOOL": True,
+ "TEST_LIST": ["one", "two"],
+ "TEST_LIST_OF_LISTS": [
+ ["first_one", "first_two"],
+ ["second_one", "second_two"],
+ ],
}
self.settings.setdict(values)
copy = self.settings.copy()
- self.settings.set('TEST_BOOL', False)
- self.assertTrue(copy.get('TEST_BOOL'))
+ self.settings.set("TEST_BOOL", False)
+ self.assertTrue(copy.get("TEST_BOOL"))
- test_list = self.settings.get('TEST_LIST')
- test_list.append('three')
- self.assertListEqual(copy.get('TEST_LIST'), ['one', 'two'])
+ test_list = self.settings.get("TEST_LIST")
+ test_list.append("three")
+ self.assertListEqual(copy.get("TEST_LIST"), ["one", "two"])
- test_list_of_lists = self.settings.get('TEST_LIST_OF_LISTS')
- test_list_of_lists[0].append('first_three')
- self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0],
- ['first_one', 'first_two'])
+ test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS")
+ test_list_of_lists[0].append("first_three")
+ self.assertListEqual(
+ copy.get("TEST_LIST_OF_LISTS")[0], ["first_one", "first_two"]
+ )
def test_copy_to_dict(self):
- s = BaseSettings({'TEST_STRING': 'a string',
- 'TEST_LIST': [1, 2],
- 'TEST_BOOLEAN': False,
- 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
- 'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
- 'HASNOBASE': BaseSettings({3: 3000}, 'default')})
+ s = BaseSettings(
+ {
+ "TEST_STRING": "a string",
+ "TEST_LIST": [1, 2],
+ "TEST_BOOLEAN": False,
+ "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"),
+ "TEST": BaseSettings({1: 10, 3: 30}, "default"),
+ "HASNOBASE": BaseSettings({3: 3000}, "default"),
+ }
+ )
self.assertDictEqual(
s.copy_to_dict(),
{
- 'HASNOBASE': {3: 3000},
- 'TEST': {1: 10, 3: 30},
- 'TEST_BASE': {1: 1, 2: 2},
- 'TEST_LIST': [1, 2],
- 'TEST_BOOLEAN': False,
- 'TEST_STRING': 'a string',
- }
+ "HASNOBASE": {3: 3000},
+ "TEST": {1: 10, 3: 30},
+ "TEST_BASE": {1: 1, 2: 2},
+ "TEST_LIST": [1, 2],
+ "TEST_BOOLEAN": False,
+ "TEST_STRING": "a string",
+ },
)
def test_freeze(self):
self.settings.freeze()
with self.assertRaises(TypeError) as cm:
- self.settings.set('TEST_BOOL', False)
- self.assertEqual(str(cm.exception),
- "Trying to modify an immutable Settings object")
+ self.settings.set("TEST_BOOL", False)
+ self.assertEqual(
+ str(cm.exception), "Trying to modify an immutable Settings object"
+ )
def test_frozencopy(self):
frozencopy = self.settings.frozencopy()
@@ -339,83 +393,109 @@ class BaseSettingsTest(unittest.TestCase):
class SettingsTest(unittest.TestCase):
-
def setUp(self):
self.settings = Settings()
- @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {'default': 10})
- @mock.patch('scrapy.settings.default_settings', default_settings)
+ @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10})
+ @mock.patch("scrapy.settings.default_settings", default_settings)
def test_initial_defaults(self):
settings = Settings()
self.assertEqual(len(settings.attributes), 2)
- self.assertIn('TEST_DEFAULT', settings.attributes)
+ self.assertIn("TEST_DEFAULT", settings.attributes)
- attr = settings.attributes['TEST_DEFAULT']
+ attr = settings.attributes["TEST_DEFAULT"]
self.assertIsInstance(attr, SettingsAttribute)
- self.assertEqual(attr.value, 'defvalue')
+ self.assertEqual(attr.value, "defvalue")
self.assertEqual(attr.priority, 10)
- @mock.patch.dict('scrapy.settings.SETTINGS_PRIORITIES', {})
- @mock.patch('scrapy.settings.default_settings', {})
+ @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {})
+ @mock.patch("scrapy.settings.default_settings", {})
def test_initial_values(self):
- settings = Settings({'TEST_OPTION': 'value'}, 10)
+ settings = Settings({"TEST_OPTION": "value"}, 10)
self.assertEqual(len(settings.attributes), 1)
- self.assertIn('TEST_OPTION', settings.attributes)
+ self.assertIn("TEST_OPTION", settings.attributes)
- attr = settings.attributes['TEST_OPTION']
+ attr = settings.attributes["TEST_OPTION"]
self.assertIsInstance(attr, SettingsAttribute)
- self.assertEqual(attr.value, 'value')
+ self.assertEqual(attr.value, "value")
self.assertEqual(attr.priority, 10)
- @mock.patch('scrapy.settings.default_settings', default_settings)
+ @mock.patch("scrapy.settings.default_settings", default_settings)
def test_autopromote_dicts(self):
settings = Settings()
- mydict = settings.get('TEST_DICT')
+ mydict = settings.get("TEST_DICT")
self.assertIsInstance(mydict, BaseSettings)
- self.assertIn('key', mydict)
- self.assertEqual(mydict['key'], 'val')
- self.assertEqual(mydict.getpriority('key'), 0)
+ self.assertIn("key", mydict)
+ self.assertEqual(mydict["key"], "val")
+ self.assertEqual(mydict.getpriority("key"), 0)
- @mock.patch('scrapy.settings.default_settings', default_settings)
+ @mock.patch("scrapy.settings.default_settings", default_settings)
def test_getdict_autodegrade_basesettings(self):
settings = Settings()
- mydict = settings.getdict('TEST_DICT')
+ mydict = settings.getdict("TEST_DICT")
self.assertIsInstance(mydict, dict)
self.assertEqual(len(mydict), 1)
- self.assertIn('key', mydict)
- self.assertEqual(mydict['key'], 'val')
+ self.assertIn("key", mydict)
+ self.assertEqual(mydict["key"], "val")
def test_passing_objects_as_values(self):
from scrapy.core.downloader.handlers.file import FileDownloadHandler
- from scrapy.utils.misc import create_instance
+ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
- class TestPipeline():
+ class TestPipeline:
def process_item(self, i, s):
return i
- settings = Settings({
- 'ITEM_PIPELINES': {
- TestPipeline: 800,
- },
- 'DOWNLOAD_HANDLERS': {
- 'ftp': FileDownloadHandler,
- },
- })
+ settings = Settings(
+ {
+ "ITEM_PIPELINES": {
+ TestPipeline: 800,
+ },
+ "DOWNLOAD_HANDLERS": {
+ "ftp": FileDownloadHandler,
+ },
+ }
+ )
- self.assertIn('ITEM_PIPELINES', settings.attributes)
+ self.assertIn("ITEM_PIPELINES", settings.attributes)
- mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem()
+ mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem()
self.assertEqual(priority, 800)
self.assertEqual(mypipeline, TestPipeline)
self.assertIsInstance(mypipeline(), TestPipeline)
- self.assertEqual(mypipeline().process_item('item', None), 'item')
+ self.assertEqual(mypipeline().process_item("item", None), "item")
- myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp')
+ myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
self.assertEqual(myhandler, FileDownloadHandler)
- myhandler_instance = create_instance(myhandler, None, get_crawler())
+ myhandler_instance = build_from_crawler(myhandler, get_crawler())
self.assertIsInstance(myhandler_instance, FileDownloadHandler)
- self.assertTrue(hasattr(myhandler_instance, 'download_request'))
+ self.assertTrue(hasattr(myhandler_instance, "download_request"))
+
+ def test_pop_item_with_default_value(self):
+ settings = Settings()
+
+ with self.assertRaises(KeyError):
+ settings.pop("DUMMY_CONFIG")
+
+ dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value")
+ self.assertEqual(dummy_config_value, "dummy_value")
+
+ def test_pop_item_with_immutable_settings(self):
+ settings = Settings(
+ {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"}
+ )
+
+ self.assertEqual(settings.pop("DUMMY_CONFIG"), "dummy_value")
+
+ settings.freeze()
+
+ with self.assertRaises(TypeError) as error:
+ settings.pop("OTHER_DUMMY_CONFIG")
+
+ self.assertEqual(
+ str(error.exception), "Trying to modify an immutable Settings object"
+ )
if __name__ == "__main__":
diff --git a/tests/test_settings/default_settings.py b/tests/test_settings/default_settings.py
index 26a555275..2a2142b32 100644
--- a/tests/test_settings/default_settings.py
+++ b/tests/test_settings/default_settings.py
@@ -1,4 +1,3 @@
+TEST_DEFAULT = "defvalue"
-TEST_DEFAULT = 'defvalue'
-
-TEST_DICT = {'key': 'val'}
+TEST_DICT = {"key": "val"}
diff --git a/tests/test_signals.py b/tests/test_signals.py
index a43f00b27..0df104600 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -2,22 +2,22 @@ from pytest import mark
from twisted.internet import defer
from twisted.trial import unittest
-from scrapy import signals, Request, Spider
+from scrapy import Request, Spider, signals
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-
from tests.mockserver import MockServer
class ItemSpider(Spider):
- name = 'itemspider'
+ name = "itemspider"
def start_requests(self):
for index in range(10):
- yield Request(self.mockserver.url(f'/status?n=200&id={index}'),
- meta={'index': index})
+ yield Request(
+ self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index}
+ )
def parse(self, response):
- return {'index': response.meta['index']}
+ return {"index": response.meta["index"]}
class AsyncSignalTestCase(unittest.TestCase):
@@ -41,4 +41,4 @@ class AsyncSignalTestCase(unittest.TestCase):
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(self.items), 10)
for index in range(10):
- self.assertIn({'index': index}, self.items)
+ self.assertIn({"index": index}, self.items)
diff --git a/tests/test_spider.py b/tests/test_spider.py
index cb66066b0..00da3d485 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -2,31 +2,33 @@ import gzip
import inspect
import warnings
from io import BytesIO
+from typing import Any
from unittest import mock
from testfixtures import LogCapture
+from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
-
from w3lib.url import safe_url_string
+
from scrapy import signals
+from scrapy.crawler import Crawler
+from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse
+from scrapy.linkextractors import LinkExtractor
from scrapy.settings import Settings
-from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
-from scrapy.spiders.init import InitSpider
from scrapy.spiders import (
- CSVFeedSpider,
CrawlSpider,
+ CSVFeedSpider,
Rule,
SitemapSpider,
Spider,
XMLFeedSpider,
)
-from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders.init import InitSpider
from scrapy.utils.test import get_crawler
from tests import get_testdata
class SpiderTest(unittest.TestCase):
-
spider_class = Spider
def setUp(self):
@@ -37,39 +39,39 @@ class SpiderTest(unittest.TestCase):
def test_base_spider(self):
spider = self.spider_class("example.com")
- self.assertEqual(spider.name, 'example.com')
+ self.assertEqual(spider.name, "example.com")
self.assertEqual(spider.start_urls, [])
def test_start_requests(self):
- spider = self.spider_class('example.com')
+ spider = self.spider_class("example.com")
start_requests = spider.start_requests()
self.assertTrue(inspect.isgenerator(start_requests))
self.assertEqual(list(start_requests), [])
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
- spider = self.spider_class('example.com', foo='bar')
- self.assertEqual(spider.foo, 'bar')
+ spider = self.spider_class("example.com", foo="bar")
+ self.assertEqual(spider.foo, "bar")
def test_spider_without_name(self):
"""``__init__`` method arguments are assigned to spider attributes"""
self.assertRaises(ValueError, self.spider_class)
- self.assertRaises(ValueError, self.spider_class, somearg='foo')
+ self.assertRaises(ValueError, self.spider_class, somearg="foo")
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
- spider = self.spider_class.from_crawler(crawler, 'example.com')
- self.assertTrue(hasattr(spider, 'crawler'))
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+ self.assertTrue(hasattr(spider, "crawler"))
self.assertIs(spider.crawler, crawler)
- self.assertTrue(hasattr(spider, 'settings'))
+ self.assertTrue(hasattr(spider, "settings"))
self.assertIs(spider.settings, crawler.settings)
def test_from_crawler_init_call(self):
- with mock.patch.object(self.spider_class, '__init__',
- return_value=None) as mock_init:
- self.spider_class.from_crawler(get_crawler(), 'example.com',
- foo='bar')
- mock_init.assert_called_once_with('example.com', foo='bar')
+ with mock.patch.object(
+ self.spider_class, "__init__", return_value=None
+ ) as mock_init:
+ self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
+ mock_init.assert_called_once_with("example.com", foo="bar")
def test_closed_signal_call(self):
class TestSpider(self.spider_class):
@@ -79,48 +81,68 @@ class SpiderTest(unittest.TestCase):
self.closed_called = True
crawler = get_crawler()
- spider = TestSpider.from_crawler(crawler, 'example.com')
- crawler.signals.send_catch_log(signal=signals.spider_opened,
- spider=spider)
- crawler.signals.send_catch_log(signal=signals.spider_closed,
- spider=spider, reason=None)
+ spider = TestSpider.from_crawler(crawler, "example.com")
+ crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
+ crawler.signals.send_catch_log(
+ signal=signals.spider_closed, spider=spider, reason=None
+ )
self.assertTrue(spider.closed_called)
def test_update_settings(self):
- spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
- project_settings = {'TEST1': 'project', 'TEST3': 'project'}
+ spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+ project_settings = {"TEST1": "project", "TEST3": "project"}
self.spider_class.custom_settings = spider_settings
- settings = Settings(project_settings, priority='project')
+ settings = Settings(project_settings, priority="project")
self.spider_class.update_settings(settings)
- self.assertEqual(settings.get('TEST1'), 'spider')
- self.assertEqual(settings.get('TEST2'), 'spider')
- self.assertEqual(settings.get('TEST3'), 'project')
+ self.assertEqual(settings.get("TEST1"), "spider")
+ self.assertEqual(settings.get("TEST2"), "spider")
+ self.assertEqual(settings.get("TEST3"), "project")
+
+ @inlineCallbacks
+ def test_settings_in_from_crawler(self):
+ spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+ project_settings = {"TEST1": "project", "TEST3": "project"}
+
+ class TestSpider(self.spider_class):
+ name = "test"
+ custom_settings = spider_settings
+
+ @classmethod
+ def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ spider.settings.set("TEST1", "spider_instance", priority="spider")
+ return spider
+
+ crawler = Crawler(TestSpider, project_settings)
+ self.assertEqual(crawler.settings.get("TEST1"), "spider")
+ self.assertEqual(crawler.settings.get("TEST2"), "spider")
+ self.assertEqual(crawler.settings.get("TEST3"), "project")
+ yield crawler.crawl()
+ self.assertEqual(crawler.settings.get("TEST1"), "spider_instance")
def test_logger(self):
- spider = self.spider_class('example.com')
+ spider = self.spider_class("example.com")
with LogCapture() as lc:
- spider.logger.info('test log msg')
- lc.check(('example.com', 'INFO', 'test log msg'))
+ spider.logger.info("test log msg")
+ lc.check(("example.com", "INFO", "test log msg"))
record = lc.records[0]
- self.assertIn('spider', record.__dict__)
+ self.assertIn("spider", record.__dict__)
self.assertIs(record.spider, spider)
def test_log(self):
- spider = self.spider_class('example.com')
- with mock.patch('scrapy.spiders.Spider.logger') as mock_logger:
- spider.log('test log msg', 'INFO')
- mock_logger.log.assert_called_once_with('INFO', 'test log msg')
+ spider = self.spider_class("example.com")
+ with mock.patch("scrapy.spiders.Spider.logger") as mock_logger:
+ spider.log("test log msg", "INFO")
+ mock_logger.log.assert_called_once_with("INFO", "test log msg")
class InitSpiderTest(SpiderTest):
-
spider_class = InitSpider
class XMLFeedSpiderTest(SpiderTest):
-
spider_class = XMLFeedSpider
def test_register_namespace(self):
@@ -132,45 +154,52 @@ class XMLFeedSpiderTest(SpiderTest):
http://www.example.com/ 2009-08-16 Page title<title>
<body>
<p><a href="item/12.html">Item 12</a></p>
@@ -202,35 +230,37 @@ class CrawlSpiderTest(SpiderTest):
spider_class = CrawlSpider
def test_rule_without_link_extractor(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
- rules = (
- Rule(),
- )
+ allowed_domains = ["example.org"]
+ rules = (Rule(),)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.org/somepage/item/12.html',
- 'http://example.org/about.html',
- 'http://example.org/nofollow.html'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ],
+ )
def test_process_links(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
- rules = (
- Rule(LinkExtractor(), process_links="dummy_process_links"),
- )
+ allowed_domains = ["example.org"]
+ rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
return links
@@ -239,47 +269,52 @@ class CrawlSpiderTest(SpiderTest):
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.org/somepage/item/12.html',
- 'http://example.org/about.html',
- 'http://example.org/nofollow.html'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ],
+ )
def test_process_links_filter(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
class _CrawlSpider(self.spider_class):
import re
name = "test"
- allowed_domains = ['example.org']
- rules = (
- Rule(LinkExtractor(), process_links="filter_process_links"),
- )
- _test_regex = re.compile('nofollow')
+ allowed_domains = ["example.org"]
+ rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
+ _test_regex = re.compile("nofollow")
def filter_process_links(self, links):
- return [link for link in links
- if not self._test_regex.search(link.url)]
+ return [link for link in links if not self._test_regex.search(link.url)]
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 2)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.org/somepage/item/12.html',
- 'http://example.org/about.html'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ ],
+ )
def test_process_links_generator(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
- rules = (
- Rule(LinkExtractor(), process_links="dummy_process_links"),
- )
+ allowed_domains = ["example.org"]
+ rules = (Rule(LinkExtractor(), process_links="dummy_process_links"),)
def dummy_process_links(self, links):
for link in links:
@@ -289,21 +324,26 @@ class CrawlSpiderTest(SpiderTest):
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.org/somepage/item/12.html',
- 'http://example.org/about.html',
- 'http://example.org/nofollow.html'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ],
+ )
def test_process_request(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
def process_request_change_domain(request, response):
- return request.replace(url=request.url.replace('.org', '.com'))
+ return request.replace(url=request.url.replace(".org", ".com"))
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
+ allowed_domains = ["example.org"]
rules = (
Rule(LinkExtractor(), process_request=process_request_change_domain),
)
@@ -312,47 +352,59 @@ class CrawlSpiderTest(SpiderTest):
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.com/somepage/item/12.html',
- 'http://example.com/about.html',
- 'http://example.com/nofollow.html'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.com/somepage/item/12.html",
+ "http://example.com/about.html",
+ "http://example.com/nofollow.html",
+ ],
+ )
def test_process_request_with_response(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
def process_request_meta_response_class(request, response):
- request.meta['response_class'] = response.__class__.__name__
+ request.meta["response_class"] = response.__class__.__name__
return request
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
+ allowed_domains = ["example.org"]
rules = (
- Rule(LinkExtractor(), process_request=process_request_meta_response_class),
+ Rule(
+ LinkExtractor(), process_request=process_request_meta_response_class
+ ),
)
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.org/somepage/item/12.html',
- 'http://example.org/about.html',
- 'http://example.org/nofollow.html'])
- self.assertEqual([r.meta['response_class'] for r in output],
- ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ],
+ )
+ self.assertEqual(
+ [r.meta["response_class"] for r in output],
+ ["HtmlResponse", "HtmlResponse", "HtmlResponse"],
+ )
def test_process_request_instance_method(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
- rules = (
- Rule(LinkExtractor(), process_request='process_request_upper'),
- )
+ allowed_domains = ["example.org"]
+ rules = (Rule(LinkExtractor(), process_request="process_request_upper"),)
def process_request_upper(self, request, response):
return request.replace(url=request.url.upper())
@@ -361,65 +413,77 @@ class CrawlSpiderTest(SpiderTest):
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'),
- safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'),
- safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ safe_url_string("http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML"),
+ safe_url_string("http://EXAMPLE.ORG/ABOUT.HTML"),
+ safe_url_string("http://EXAMPLE.ORG/NOFOLLOW.HTML"),
+ ],
+ )
def test_process_request_instance_method_with_response(self):
-
- response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+ response = HtmlResponse(
+ "http://example.org/somepage/index.html", body=self.test_body
+ )
class _CrawlSpider(self.spider_class):
name = "test"
- allowed_domains = ['example.org']
+ allowed_domains = ["example.org"]
rules = (
- Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
+ Rule(
+ LinkExtractor(),
+ process_request="process_request_meta_response_class",
+ ),
)
def process_request_meta_response_class(self, request, response):
- request.meta['response_class'] = response.__class__.__name__
+ request.meta["response_class"] = response.__class__.__name__
return request
spider = _CrawlSpider()
output = list(spider._requests_to_follow(response))
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
- self.assertEqual([r.url for r in output],
- ['http://example.org/somepage/item/12.html',
- 'http://example.org/about.html',
- 'http://example.org/nofollow.html'])
- self.assertEqual([r.meta['response_class'] for r in output],
- ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+ self.assertEqual(
+ [r.url for r in output],
+ [
+ "http://example.org/somepage/item/12.html",
+ "http://example.org/about.html",
+ "http://example.org/nofollow.html",
+ ],
+ )
+ self.assertEqual(
+ [r.meta["response_class"] for r in output],
+ ["HtmlResponse", "HtmlResponse", "HtmlResponse"],
+ )
def test_follow_links_attribute_population(self):
crawler = get_crawler()
- spider = self.spider_class.from_crawler(crawler, 'example.com')
- self.assertTrue(hasattr(spider, '_follow_links'))
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+ self.assertTrue(hasattr(spider, "_follow_links"))
self.assertTrue(spider._follow_links)
- settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
+ settings_dict = {"CRAWLSPIDER_FOLLOW_LINKS": False}
crawler = get_crawler(settings_dict=settings_dict)
- spider = self.spider_class.from_crawler(crawler, 'example.com')
- self.assertTrue(hasattr(spider, '_follow_links'))
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+ self.assertTrue(hasattr(spider, "_follow_links"))
self.assertFalse(spider._follow_links)
def test_start_url(self):
spider = self.spider_class("example.com")
- spider.start_url = 'https://www.example.com'
+ spider.start_url = "https://www.example.com"
- with self.assertRaisesRegex(AttributeError,
- r'^Crawling could not start.*$'):
+ with self.assertRaisesRegex(AttributeError, r"^Crawling could not start.*$"):
list(spider.start_requests())
class SitemapSpiderTest(SpiderTest):
-
spider_class = SitemapSpider
BODY = b"SITEMAP"
f = BytesIO()
- g = gzip.GzipFile(fileobj=f, mode='w+b')
+ g = gzip.GzipFile(fileobj=f, mode="w+b")
g.write(BODY)
g.close()
GZBODY = f.getvalue()
@@ -439,8 +503,11 @@ class SitemapSpiderTest(SpiderTest):
self.assertSitemapBody(r, None)
def test_get_sitemap_body_gzip_headers(self):
- r = Response(url="http://www.example.com/sitemap", body=self.GZBODY,
- headers={"content-type": "application/gzip"})
+ r = Response(
+ url="http://www.example.com/sitemap",
+ body=self.GZBODY,
+ headers={"content-type": "application/gzip"},
+ )
self.assertSitemapBody(r, self.BODY)
def test_get_sitemap_body_xml_url(self):
@@ -465,11 +532,15 @@ Sitemap: /sitemap-relative-url.xml
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
spider = self.spider_class("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://example.com/sitemap.xml',
- 'http://example.com/sitemap-product-index.xml',
- 'http://example.com/sitemap-uppercase.xml',
- 'http://www.example.com/sitemap-relative-url.xml'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ [
+ "http://example.com/sitemap.xml",
+ "http://example.com/sitemap-product-index.xml",
+ "http://example.com/sitemap-uppercase.xml",
+ "http://www.example.com/sitemap-relative-url.xml",
+ ],
+ )
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -488,15 +559,21 @@ Sitemap: /sitemap-relative-url.xml
</urlset>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/english/'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ ["http://www.example.com/english/"],
+ )
spider.sitemap_alternate_links = True
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/english/',
- 'http://www.example.com/deutsch/',
- 'http://www.example.com/schweiz-deutsch/',
- 'http://www.example.com/italiano/'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ [
+ "http://www.example.com/english/",
+ "http://www.example.com/deutsch/",
+ "http://www.example.com/schweiz-deutsch/",
+ "http://www.example.com/italiano/",
+ ],
+ )
def test_sitemap_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -515,20 +592,24 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
from datetime import datetime
+
for entry in entries:
- date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
+ date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
if date_time.year > 2008:
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/english/',
- 'http://www.example.com/portuguese/'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ ["http://www.example.com/english/", "http://www.example.com/portuguese/"],
+ )
spider = FilteredSitemapSpider("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/english/'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ ["http://www.example.com/english/"],
+ )
def test_sitemap_filter_with_alternate_links(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -549,21 +630,27 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
for entry in entries:
- alternate_links = entry.get('alternate', tuple())
+ alternate_links = entry.get("alternate", tuple())
for link in alternate_links:
- if '/deutsch/' in link:
- entry['loc'] = link
+ if "/deutsch/" in link:
+ entry["loc"] = link
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/english/article_1/',
- 'http://www.example.com/english/article_2/'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ [
+ "http://www.example.com/english/article_1/",
+ "http://www.example.com/english/article_2/",
+ ],
+ )
spider = FilteredSitemapSpider("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/deutsch/article_1/'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ ["http://www.example.com/deutsch/article_1/"],
+ )
def test_sitemapindex_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -581,38 +668,45 @@ Sitemap: /sitemap-relative-url.xml
class FilteredSitemapSpider(self.spider_class):
def sitemap_filter(self, entries):
from datetime import datetime
+
for entry in entries:
- date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d')
+ date_time = datetime.strptime(
+ entry["lastmod"].split("T")[0], "%Y-%m-%d"
+ )
if date_time.year > 2004:
yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/sitemap1.xml',
- 'http://www.example.com/sitemap2.xml'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ [
+ "http://www.example.com/sitemap1.xml",
+ "http://www.example.com/sitemap2.xml",
+ ],
+ )
spider = FilteredSitemapSpider("example.com")
- self.assertEqual([req.url for req in spider._parse_sitemap(r)],
- ['http://www.example.com/sitemap2.xml'])
+ self.assertEqual(
+ [req.url for req in spider._parse_sitemap(r)],
+ ["http://www.example.com/sitemap2.xml"],
+ )
class DeprecationTest(unittest.TestCase):
-
def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
- assert isinstance(CrawlSpider(name='foo'), Spider)
+ assert isinstance(CrawlSpider(name="foo"), Spider)
class NoParseMethodSpiderTest(unittest.TestCase):
-
spider_class = Spider
def test_undefined_parse_method(self):
- spider = self.spider_class('example.com')
- text = b'Random text'
+ spider = self.spider_class("example.com")
+ text = b"Random text"
resp = TextResponse(url="http://www.example.com/random_url", body=text)
- exc_msg = 'Spider.parse callback is not defined'
+ exc_msg = "Spider.parse callback is not defined"
with self.assertRaisesRegex(NotImplementedError, exc_msg):
spider.parse(resp)
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 0b6b51a5b..04025d30d 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -1,21 +1,20 @@
-import sys
import shutil
+import sys
+import tempfile
import warnings
from pathlib import Path
-import tempfile
-from zope.interface.verify import verifyObject
from twisted.trial import unittest
-
+from zope.interface.verify import verifyObject
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
# alone
import scrapy
-from scrapy.interfaces import ISpiderLoader
-from scrapy.spiderloader import SpiderLoader
-from scrapy.settings import Settings
-from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
+from scrapy.interfaces import ISpiderLoader
+from scrapy.settings import Settings
+from scrapy.spiderloader import SpiderLoader
module_dir = Path(__file__).resolve().parent
@@ -28,19 +27,18 @@ def _copytree(source: Path, target: Path):
class SpiderLoaderTest(unittest.TestCase):
-
def setUp(self):
- orig_spiders_dir = module_dir / 'test_spiders'
+ orig_spiders_dir = module_dir / "test_spiders"
self.tmpdir = Path(tempfile.mkdtemp())
- self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
+ self.spiders_dir = self.tmpdir / "test_spiders_xxx"
_copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(str(self.tmpdir))
- settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+ settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
self.spider_loader = SpiderLoader.from_settings(settings)
def tearDown(self):
del self.spider_loader
- del sys.modules['test_spiders_xxx']
+ del sys.modules["test_spiders_xxx"]
sys.path.remove(str(self.tmpdir))
def test_interface(self):
@@ -48,78 +46,84 @@ class SpiderLoaderTest(unittest.TestCase):
def test_list(self):
self.assertEqual(
- set(self.spider_loader.list()),
- {'spider1', 'spider2', 'spider3', 'spider4'})
+ set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}
+ )
def test_load(self):
spider1 = self.spider_loader.load("spider1")
- self.assertEqual(spider1.__name__, 'Spider1')
+ self.assertEqual(spider1.__name__, "Spider1")
def test_find_by_request(self):
self.assertEqual(
- self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
- ['spider1'])
+ self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),
+ ["spider1"],
+ )
self.assertEqual(
- self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
- ['spider2'])
+ self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),
+ ["spider2"],
+ )
self.assertEqual(
- set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
- {'spider1', 'spider2'})
+ set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),
+ {"spider1", "spider2"},
+ )
self.assertEqual(
- self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
- [])
+ self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []
+ )
self.assertEqual(
- self.spider_loader.find_by_request(Request('http://spider3.com')),
- [])
+ self.spider_loader.find_by_request(Request("http://spider3.com")), []
+ )
self.assertEqual(
- self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
- ['spider3'])
+ self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),
+ ["spider3"],
+ )
def test_load_spider_module(self):
- module = 'tests.test_spiderloader.test_spiders.spider1'
- settings = Settings({'SPIDER_MODULES': [module]})
+ module = "tests.test_spiderloader.test_spiders.spider1"
+ settings = Settings({"SPIDER_MODULES": [module]})
self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spider_loader._spiders) == 1
def test_load_spider_module_multiple(self):
- prefix = 'tests.test_spiderloader.test_spiders.'
- module = ','.join(prefix + s for s in ('spider1', 'spider2'))
- settings = Settings({'SPIDER_MODULES': module})
+ prefix = "tests.test_spiderloader.test_spiders."
+ module = ",".join(prefix + s for s in ("spider1", "spider2"))
+ settings = Settings({"SPIDER_MODULES": module})
self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spider_loader._spiders) == 2
def test_load_base_spider(self):
- module = 'tests.test_spiderloader.test_spiders.spider0'
- settings = Settings({'SPIDER_MODULES': [module]})
+ module = "tests.test_spiderloader.test_spiders.spider0"
+ settings = Settings({"SPIDER_MODULES": [module]})
self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spider_loader._spiders) == 0
def test_crawler_runner_loading(self):
- module = 'tests.test_spiderloader.test_spiders.spider1'
- runner = CrawlerRunner({
- 'SPIDER_MODULES': [module],
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
- })
+ module = "tests.test_spiderloader.test_spiders.spider1"
+ runner = CrawlerRunner(
+ {
+ "SPIDER_MODULES": [module],
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
+ }
+ )
- self.assertRaisesRegex(KeyError, 'Spider not found',
- runner.create_crawler, 'spider2')
+ self.assertRaisesRegex(
+ KeyError, "Spider not found", runner.create_crawler, "spider2"
+ )
- crawler = runner.create_crawler('spider1')
+ crawler = runner.create_crawler("spider1")
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
- self.assertEqual(crawler.spidercls.name, 'spider1')
+ self.assertEqual(crawler.spidercls.name, "spider1")
def test_bad_spider_modules_exception(self):
-
- module = 'tests.test_spiderloader.test_spiders.doesnotexist'
- settings = Settings({'SPIDER_MODULES': [module]})
+ module = "tests.test_spiderloader.test_spiders.doesnotexist"
+ settings = Settings({"SPIDER_MODULES": [module]})
self.assertRaises(ImportError, SpiderLoader.from_settings, settings)
def test_bad_spider_modules_warning(self):
-
with warnings.catch_warnings(record=True) as w:
- module = 'tests.test_spiderloader.test_spiders.doesnotexist'
- settings = Settings({'SPIDER_MODULES': [module],
- 'SPIDER_LOADER_WARN_ONLY': True})
+ module = "tests.test_spiderloader.test_spiders.doesnotexist"
+ settings = Settings(
+ {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
+ )
spider_loader = SpiderLoader.from_settings(settings)
if str(w[0].message).startswith("_SixMetaPathImporter"):
# needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
@@ -133,24 +137,25 @@ class SpiderLoaderTest(unittest.TestCase):
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
-
def setUp(self):
- orig_spiders_dir = module_dir / 'test_spiders'
+ orig_spiders_dir = module_dir / "test_spiders"
self.tmpdir = Path(self.mktemp())
self.tmpdir.mkdir()
- self.spiders_dir = self.tmpdir / 'test_spiders_xxx'
+ self.spiders_dir = self.tmpdir / "test_spiders_xxx"
_copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(str(self.tmpdir))
- self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
+ self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
def tearDown(self):
- del sys.modules['test_spiders_xxx']
+ del sys.modules["test_spiders_xxx"]
sys.path.remove(str(self.tmpdir))
def test_dupename_warning(self):
# copy 1 spider module so as to have duplicate spider name
- shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py',
- self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py')
+ shutil.copyfile(
+ self.tmpdir / "test_spiders_xxx" / "spider3.py",
+ self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",
+ )
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
@@ -166,15 +171,19 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
self.assertNotIn("'spider4'", msg)
spiders = set(spider_loader.list())
- self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
+ self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
def test_multiple_dupename_warning(self):
# copy 2 spider modules so as to have duplicate spider name
# This should issue 2 warning, 1 for each duplicate spider name
- shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py',
- self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py')
- shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py',
- self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py')
+ shutil.copyfile(
+ self.tmpdir / "test_spiders_xxx" / "spider1.py",
+ self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",
+ )
+ shutil.copyfile(
+ self.tmpdir / "test_spiders_xxx" / "spider2.py",
+ self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",
+ )
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
@@ -192,4 +201,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
self.assertNotIn("'spider4'", msg)
spiders = set(spider_loader.list())
- self.assertEqual(spiders, {'spider1', 'spider2', 'spider3', 'spider4'})
+ self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})
diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py
index dbd1fb123..6c8ea4fe4 100644
--- a/tests/test_spiderloader/test_spiders/nested/spider4.py
+++ b/tests/test_spiderloader/test_spiders/nested/spider4.py
@@ -3,8 +3,8 @@ from scrapy.spiders import Spider
class Spider4(Spider):
name = "spider4"
- allowed_domains = ['spider4.com']
+ allowed_domains = ["spider4.com"]
@classmethod
def handles_request(cls, request):
- return request.url == 'http://spider4.com/onlythis'
+ return request.url == "http://spider4.com/onlythis"
diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py
index 84998ba35..d99878276 100644
--- a/tests/test_spiderloader/test_spiders/spider3.py
+++ b/tests/test_spiderloader/test_spiders/spider3.py
@@ -3,8 +3,8 @@ from scrapy.spiders import Spider
class Spider3(Spider):
name = "spider3"
- allowed_domains = ['spider3.com']
+ allowed_domains = ["spider3.com"]
@classmethod
def handles_request(cls, request):
- return request.url == 'http://spider3.com/onlythis'
+ return request.url == "http://spider3.com/onlythis"
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index edde6f682..d167adbb7 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -4,25 +4,24 @@ from unittest import mock
from testfixtures import LogCapture
from twisted.internet import defer
-from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
+from twisted.trial.unittest import TestCase
-from scrapy.spiders import Spider
-from scrapy.http import Request, Response
+from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import _InvalidOutput
+from scrapy.http import Request, Response
+from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
-from scrapy.core.spidermw import SpiderMiddlewareManager
class SpiderMiddlewareTestCase(TestCase):
-
def setUp(self):
- self.request = Request('http://example.com/index.html')
+ self.request = Request("http://example.com/index.html")
self.response = Response(self.request.url, request=self.request)
- self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}})
- self.spider = self.crawler._create_spider('foo')
+ self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
+ self.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
def _scrape_response(self):
@@ -30,7 +29,9 @@ class SpiderMiddlewareTestCase(TestCase):
Raise exception in case of failure.
"""
scrape_func = mock.MagicMock()
- dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider)
+ dfd = self.mwman.scrape_response(
+ scrape_func, self.response, self.request, self.spider
+ )
# catch deferred result and return the value
results = []
dfd.addBoth(results.append)
@@ -43,7 +44,6 @@ class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase):
"""Invalid return value for process_spider_input method"""
def test_invalid_process_spider_input(self):
-
class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
return 1
@@ -58,7 +58,6 @@ class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase):
"""Invalid return value for process_spider_output method"""
def test_invalid_process_spider_output(self):
-
class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider):
return 1
@@ -73,7 +72,6 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase):
"""Invalid return value for process_spider_exception method"""
def test_invalid_process_spider_exception(self):
-
class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception, spider):
return 1
@@ -93,7 +91,6 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
"""Re raise the exception by returning None"""
def test_process_spider_exception_return_none(self):
-
class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception, spider):
return None
@@ -110,11 +107,12 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
- """ Helpers for testing sync, async and mixed middlewares.
+ """Helpers for testing sync, async and mixed middlewares.
Should work for process_spider_output and, when it's supported, process_start_requests.
"""
+ ITEM_TYPE: type
RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
@staticmethod
@@ -124,23 +122,31 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
return {i: c for c, i in enumerate(mw_classes, start=start_index)}
def _scrape_func(self, *args, **kwargs):
- yield {'foo': 1}
- yield {'foo': 2}
- yield {'foo': 3}
+ yield {"foo": 1}
+ yield {"foo": 2}
+ yield {"foo": 3}
@defer.inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
- self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
- self.spider = self.crawler._create_spider('foo')
+ self.crawler = get_crawler(
+ Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
+ )
+ self.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
- result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+ result = yield self.mwman.scrape_response(
+ self._scrape_func, self.response, self.request, self.spider
+ )
return result
@defer.inlineCallbacks
- def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+ def _test_simple_base(
+ self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None
+ ):
with LogCapture() as log:
- result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
+ result = yield self._get_middleware_result(
+ *mw_classes, start_index=start_index
+ )
self.assertIsInstance(result, collections.abc.Iterable)
result_list = list(result)
self.assertEqual(len(result_list), self.RESULT_COUNT)
@@ -148,9 +154,13 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
self.assertEqual("downgraded to a non-async" in str(log), downgrade)
@defer.inlineCallbacks
- def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+ def _test_asyncgen_base(
+ self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None
+ ):
with LogCapture() as log:
- result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
+ result = yield self._get_middleware_result(
+ *mw_classes, start_index=start_index
+ )
self.assertIsInstance(result, collections.abc.AsyncIterator)
result_list = yield deferred_from_coro(collect_asyncgen(result))
self.assertEqual(len(result_list), self.RESULT_COUNT)
@@ -182,24 +192,25 @@ class ProcessSpiderOutputUniversalMiddleware:
class ProcessSpiderExceptionSimpleIterableMiddleware:
def process_spider_exception(self, response, exception, spider):
- yield {'foo': 1}
- yield {'foo': 2}
- yield {'foo': 3}
+ yield {"foo": 1}
+ yield {"foo": 2}
+ yield {"foo": 3}
class ProcessSpiderExceptionAsyncIterableMiddleware:
async def process_spider_exception(self, response, exception, spider):
- yield {'foo': 1}
+ yield {"foo": 1}
d = defer.Deferred()
from twisted.internet import reactor
+
reactor.callLater(0, d.callback, None)
await maybe_deferred_to_future(d)
- yield {'foo': 2}
- yield {'foo': 3}
+ yield {"foo": 2}
+ yield {"foo": 3}
class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
- """ process_spider_output tests for simple callbacks"""
+ """process_spider_output tests for simple callbacks"""
ITEM_TYPE = dict
MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
@@ -207,82 +218,70 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
def test_simple(self):
- """ Simple mw """
+ """Simple mw"""
return self._test_simple_base(self.MW_SIMPLE)
def test_asyncgen(self):
- """ Asyncgen mw; upgrade """
+ """Asyncgen mw; upgrade"""
return self._test_asyncgen_base(self.MW_ASYNCGEN)
def test_simple_asyncgen(self):
- """ Simple mw -> asyncgen mw; upgrade """
- return self._test_asyncgen_base(self.MW_ASYNCGEN,
- self.MW_SIMPLE)
+ """Simple mw -> asyncgen mw; upgrade"""
+ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
def test_asyncgen_simple(self):
- """ Asyncgen mw -> simple mw; upgrade then downgrade """
- return self._test_simple_base(self.MW_SIMPLE,
- self.MW_ASYNCGEN,
- downgrade=True)
+ """Asyncgen mw -> simple mw; upgrade then downgrade"""
+ return self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
def test_universal(self):
- """ Universal mw """
+ """Universal mw"""
return self._test_simple_base(self.MW_UNIVERSAL)
def test_universal_simple(self):
- """ Universal mw -> simple mw """
- return self._test_simple_base(self.MW_SIMPLE,
- self.MW_UNIVERSAL)
+ """Universal mw -> simple mw"""
+ return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
def test_simple_universal(self):
- """ Simple mw -> universal mw """
- return self._test_simple_base(self.MW_UNIVERSAL,
- self.MW_SIMPLE)
+ """Simple mw -> universal mw"""
+ return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
def test_universal_asyncgen(self):
- """ Universal mw -> asyncgen mw; upgrade """
- return self._test_asyncgen_base(self.MW_ASYNCGEN,
- self.MW_UNIVERSAL)
+ """Universal mw -> asyncgen mw; upgrade"""
+ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
def test_asyncgen_universal(self):
- """ Asyncgen mw -> universal mw; upgrade """
- return self._test_asyncgen_base(self.MW_UNIVERSAL,
- self.MW_ASYNCGEN)
+ """Asyncgen mw -> universal mw; upgrade"""
+ return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
- """ process_spider_output tests for async generator callbacks """
+ """process_spider_output tests for async generator callbacks"""
async def _scrape_func(self, *args, **kwargs):
for item in super()._scrape_func():
yield item
def test_simple(self):
- """ Simple mw; downgrade """
- return self._test_simple_base(self.MW_SIMPLE,
- downgrade=True)
+ """Simple mw; downgrade"""
+ return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
def test_simple_asyncgen(self):
- """ Simple mw -> asyncgen mw; downgrade then upgrade """
- return self._test_asyncgen_base(self.MW_ASYNCGEN,
- self.MW_SIMPLE,
- downgrade=True)
+ """Simple mw -> asyncgen mw; downgrade then upgrade"""
+ return self._test_asyncgen_base(
+ self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True
+ )
def test_universal(self):
- """ Universal mw """
+ """Universal mw"""
return self._test_asyncgen_base(self.MW_UNIVERSAL)
def test_universal_simple(self):
- """ Universal mw -> simple mw; downgrade """
- return self._test_simple_base(self.MW_SIMPLE,
- self.MW_UNIVERSAL,
- downgrade=True)
+ """Universal mw -> simple mw; downgrade"""
+ return self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
def test_simple_universal(self):
- """ Simple mw -> universal mw; downgrade """
- return self._test_simple_base(self.MW_UNIVERSAL,
- self.MW_SIMPLE,
- downgrade=True)
+ """Simple mw -> universal mw; downgrade"""
+ return self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
class ProcessSpiderOutputNonIterableMiddleware:
@@ -299,7 +298,6 @@ class ProcessSpiderOutputCoroutineMiddleware:
class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
-
@defer.inlineCallbacks
def test_non_iterable(self):
with self.assertRaisesRegex(
@@ -331,27 +329,29 @@ class ProcessStartRequestsSimpleMiddleware:
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
- """ process_start_requests tests for simple start_requests"""
+ """process_start_requests tests for simple start_requests"""
ITEM_TYPE = Request
MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
def _start_requests(self):
for i in range(3):
- yield Request(f'https://example.com/{i}', dont_filter=True)
+ yield Request(f"https://example.com/{i}", dont_filter=True)
@defer.inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
- self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
- self.spider = self.crawler._create_spider('foo')
+ self.crawler = get_crawler(
+ Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
+ )
+ self.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
start_requests = iter(self._start_requests())
results = yield self.mwman.process_start_requests(start_requests, self.spider)
return results
def test_simple(self):
- """ Simple mw """
+ """Simple mw"""
return self._test_simple_base(self.MW_SIMPLE)
@@ -383,41 +383,57 @@ class UniversalMiddlewareManagerTest(TestCase):
def test_simple_mw(self):
mw = ProcessSpiderOutputSimpleMiddleware
self.mwman._add_middleware(mw)
- self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+ self.assertEqual(
+ self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+ )
def test_async_mw(self):
mw = ProcessSpiderOutputAsyncGenMiddleware
self.mwman._add_middleware(mw)
- self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+ self.assertEqual(
+ self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+ )
def test_universal_mw(self):
mw = ProcessSpiderOutputUniversalMiddleware
self.mwman._add_middleware(mw)
- self.assertEqual(self.mwman.methods['process_spider_output'][0],
- (mw.process_spider_output, mw.process_spider_output_async))
+ self.assertEqual(
+ self.mwman.methods["process_spider_output"][0],
+ (mw.process_spider_output, mw.process_spider_output_async),
+ )
def test_universal_mw_no_sync(self):
with LogCapture() as log:
self.mwman._add_middleware(UniversalMiddlewareNoSync)
- self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async"
- " without process_spider_output", str(log))
- self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+ self.assertIn(
+ "UniversalMiddlewareNoSync has process_spider_output_async"
+ " without process_spider_output",
+ str(log),
+ )
+ self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
def test_universal_mw_both_sync(self):
mw = UniversalMiddlewareBothSync
with LogCapture() as log:
self.mwman._add_middleware(mw)
- self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async "
- "is not an async generator function", str(log))
- self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+ self.assertIn(
+ "UniversalMiddlewareBothSync.process_spider_output_async "
+ "is not an async generator function",
+ str(log),
+ )
+ self.assertEqual(
+ self.mwman.methods["process_spider_output"][0], mw.process_spider_output
+ )
def test_universal_mw_both_async(self):
with LogCapture() as log:
self.mwman._add_middleware(UniversalMiddlewareBothAsync)
- self.assertIn("UniversalMiddlewareBothAsync.process_spider_output "
- "is an async generator function while process_spider_output_async exists",
- str(log))
- self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+ self.assertIn(
+ "UniversalMiddlewareBothAsync.process_spider_output "
+ "is an async generator function while process_spider_output_async exists",
+ str(log),
+ )
+ self.assertEqual(self.mwman.methods["process_spider_output"][0], None)
class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
@@ -429,10 +445,12 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
- self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting})
- self.spider = self.crawler._create_spider('foo')
+ self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
+ self.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
- result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+ result = yield self.mwman.scrape_response(
+ self._scrape_func, self.response, self.request, self.spider
+ )
return result
def test_just_builtin(self):
@@ -442,7 +460,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
return self._test_simple_base(self.MW_SIMPLE, start_index=1000)
def test_builtin_async(self):
- """ Upgrade """
+ """Upgrade"""
return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
def test_builtin_universal(self):
@@ -452,7 +470,7 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
return self._test_simple_base(self.MW_SIMPLE)
def test_async_builtin(self):
- """ Upgrade """
+ """Upgrade"""
return self._test_asyncgen_base(self.MW_ASYNCGEN)
def test_universal_builtin(self):
@@ -468,7 +486,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
return self._test_asyncgen_base()
def test_builtin_simple(self):
- """ Downgrade """
+ """Downgrade"""
return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
def test_builtin_async(self):
@@ -478,7 +496,7 @@ class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
def test_simple_builtin(self):
- """ Downgrade """
+ """Downgrade"""
return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
def test_async_builtin(self):
@@ -501,33 +519,31 @@ class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
@defer.inlineCallbacks
def _test_asyncgen_nodowngrade(self, *mw_classes):
- with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"):
+ with self.assertRaisesRegex(
+ _InvalidOutput, "Async iterable returned from .+ cannot be downgraded"
+ ):
yield self._get_middleware_result(*mw_classes)
def test_exc_simple(self):
- """ Simple exc mw """
+ """Simple exc mw"""
return self._test_simple_base(self.MW_EXC_SIMPLE)
def test_exc_async(self):
- """ Async exc mw """
+ """Async exc mw"""
return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
def test_exc_simple_simple(self):
- """ Simple exc mw -> simple output mw """
- return self._test_simple_base(self.MW_SIMPLE,
- self.MW_EXC_SIMPLE)
+ """Simple exc mw -> simple output mw"""
+ return self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
def test_exc_async_async(self):
- """ Async exc mw -> async output mw """
- return self._test_asyncgen_base(self.MW_ASYNCGEN,
- self.MW_EXC_ASYNCGEN)
+ """Async exc mw -> async output mw"""
+ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_ASYNCGEN)
def test_exc_simple_async(self):
- """ Simple exc mw -> async output mw; upgrade """
- return self._test_asyncgen_base(self.MW_ASYNCGEN,
- self.MW_EXC_SIMPLE)
+ """Simple exc mw -> async output mw; upgrade"""
+ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
def test_exc_async_simple(self):
- """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """
- return self._test_asyncgen_nodowngrade(self.MW_SIMPLE,
- self.MW_EXC_ASYNCGEN)
+ """Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
+ return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 71cca2472..e359d9cfc 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -1,17 +1,16 @@
from unittest import TestCase
+from scrapy.http import Request, Response
from scrapy.spidermiddlewares.depth import DepthMiddleware
-from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.test import get_crawler
class TestDepthMiddleware(TestCase):
-
def setUp(self):
crawler = get_crawler(Spider)
- self.spider = crawler._create_spider('scrapytest.org')
+ self.spider = crawler._create_spider("scrapytest.org")
self.stats = StatsCollector(crawler)
self.stats.open_spider(self.spider)
@@ -19,24 +18,24 @@ class TestDepthMiddleware(TestCase):
self.mw = DepthMiddleware(1, self.stats, True)
def test_process_spider_output(self):
- req = Request('http://scrapytest.org')
- resp = Response('http://scrapytest.org')
+ req = Request("http://scrapytest.org")
+ resp = Response("http://scrapytest.org")
resp.request = req
- result = [Request('http://scrapytest.org')]
+ result = [Request("http://scrapytest.org")]
out = list(self.mw.process_spider_output(resp, result, self.spider))
self.assertEqual(out, result)
- rdc = self.stats.get_value('request_depth_count/1', spider=self.spider)
+ rdc = self.stats.get_value("request_depth_count/1", spider=self.spider)
self.assertEqual(rdc, 1)
- req.meta['depth'] = 1
+ req.meta["depth"] = 1
out2 = list(self.mw.process_spider_output(resp, result, self.spider))
self.assertEqual(out2, [])
- rdm = self.stats.get_value('request_depth_max', spider=self.spider)
+ rdm = self.stats.get_value("request_depth_max", spider=self.spider)
self.assertEqual(rdm, 1)
def tearDown(self):
- self.stats.close_spider(self.spider, '')
+ self.stats.close_spider(self.spider, "")
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index ee11ee492..044455415 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,22 +1,23 @@
import logging
+from typing import Set
from unittest import TestCase
from testfixtures import LogCapture
-from twisted.trial.unittest import TestCase as TrialTestCase
from twisted.internet import defer
+from twisted.trial.unittest import TestCase as TrialTestCase
-from scrapy.utils.test import get_crawler
-from scrapy.http import Response, Request
-from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError
+from scrapy.http import Request, Response
from scrapy.settings import Settings
+from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
from tests.spiders import MockServerSpider
class _HttpErrorSpider(MockServerSpider):
- name = 'httperror'
- bypass_status_codes = set()
+ name = "httperror"
+ bypass_status_codes: Set[int] = set()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -59,28 +60,35 @@ def _responses(request, status_codes):
class TestHttpErrorMiddleware(TestCase):
-
def setUp(self):
crawler = get_crawler(Spider)
- self.spider = Spider.from_crawler(crawler, name='foo')
+ self.spider = Spider.from_crawler(crawler, name="foo")
self.mw = HttpErrorMiddleware(Settings({}))
- self.req = Request('http://scrapytest.org')
+ self.req = Request("http://scrapytest.org")
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
- self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
+ self.assertRaises(
+ HttpError, self.mw.process_spider_input, self.res404, self.spider
+ )
def test_process_spider_exception(self):
self.assertEqual(
[],
- self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
- self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
+ self.mw.process_spider_exception(
+ self.res404, HttpError(self.res404), self.spider
+ ),
+ )
+ self.assertIsNone(
+ self.mw.process_spider_exception(self.res404, Exception(), self.spider)
+ )
def test_handle_httpstatus_list(self):
res = self.res404.copy()
- res.request = Request('http://scrapytest.org',
- meta={'handle_httpstatus_list': [404]})
+ res.request = Request(
+ "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
+ )
self.assertIsNone(self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
@@ -91,18 +99,22 @@ class TestHttpErrorMiddlewareSettings(TestCase):
"""Similar test, but with settings"""
def setUp(self):
- self.spider = Spider('foo')
- self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)}))
- self.req = Request('http://scrapytest.org')
+ self.spider = Spider("foo")
+ self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
+ self.req = Request("http://scrapytest.org")
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
- self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
+ self.assertRaises(
+ HttpError, self.mw.process_spider_input, self.res404, self.spider
+ )
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
- request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
+ request = Request(
+ "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
+ )
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()
@@ -114,15 +126,16 @@ class TestHttpErrorMiddlewareSettings(TestCase):
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
- self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
+ self.assertRaises(
+ HttpError, self.mw.process_spider_input, self.res402, self.spider
+ )
class TestHttpErrorMiddlewareHandleAll(TestCase):
-
def setUp(self):
- self.spider = Spider('foo')
- self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
- self.req = Request('http://scrapytest.org')
+ self.spider = Spider("foo")
+ self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
+ self.req = Request("http://scrapytest.org")
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
@@ -130,7 +143,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
- request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
+ request = Request(
+ "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
+ )
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()
@@ -142,8 +157,12 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
def test_httperror_allow_all_false(self):
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
- request_httpstatus_false = Request('http://scrapytest.org', meta={'handle_httpstatus_all': False})
- request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True})
+ request_httpstatus_false = Request(
+ "http://scrapytest.org", meta={"handle_httpstatus_all": False}
+ )
+ request_httpstatus_true = Request(
+ "http://scrapytest.org", meta={"handle_httpstatus_all": True}
+ )
res404 = self.res404.copy()
res404.request = request_httpstatus_false
res402 = self.res402.copy()
@@ -166,28 +185,28 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
assert not crawler.spider.skipped, crawler.spider.skipped
- self.assertEqual(crawler.spider.parsed, {'200'})
- self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+ self.assertEqual(crawler.spider.parsed, {"200"})
+ self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
get_value = crawler.stats.get_value
- self.assertEqual(get_value('httperror/response_ignored_count'), 3)
- self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)
- self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)
- self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)
+ self.assertEqual(get_value("httperror/response_ignored_count"), 3)
+ self.assertEqual(get_value("httperror/response_ignored_status_count/404"), 1)
+ self.assertEqual(get_value("httperror/response_ignored_status_count/402"), 1)
+ self.assertEqual(get_value("httperror/response_ignored_status_count/500"), 1)
@defer.inlineCallbacks
def test_logging(self):
crawler = get_crawler(_HttpErrorSpider)
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
- self.assertEqual(crawler.spider.parsed, {'200', '402'})
- self.assertEqual(crawler.spider.skipped, {'402'})
- self.assertEqual(crawler.spider.failed, {'404', '500'})
+ self.assertEqual(crawler.spider.parsed, {"200", "402"})
+ self.assertEqual(crawler.spider.skipped, {"402"})
+ self.assertEqual(crawler.spider.failed, {"404", "500"})
- self.assertIn('Ignoring response <404', str(log))
- self.assertIn('Ignoring response <500', str(log))
- self.assertNotIn('Ignoring response <200', str(log))
- self.assertNotIn('Ignoring response <402', str(log))
+ self.assertIn("Ignoring response <404", str(log))
+ self.assertIn("Ignoring response <500", str(log))
+ self.assertNotIn("Ignoring response <200", str(log))
+ self.assertNotIn("Ignoring response <402", str(log))
@defer.inlineCallbacks
def test_logging_level(self):
@@ -195,22 +214,22 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
crawler = get_crawler(_HttpErrorSpider)
with LogCapture(level=logging.INFO) as log:
yield crawler.crawl(mockserver=self.mockserver)
- self.assertEqual(crawler.spider.parsed, {'200'})
- self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+ self.assertEqual(crawler.spider.parsed, {"200"})
+ self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
- self.assertIn('Ignoring response <402', str(log))
- self.assertIn('Ignoring response <404', str(log))
- self.assertIn('Ignoring response <500', str(log))
- self.assertNotIn('Ignoring response <200', str(log))
+ self.assertIn("Ignoring response <402", str(log))
+ self.assertIn("Ignoring response <404", str(log))
+ self.assertIn("Ignoring response <500", str(log))
+ self.assertNotIn("Ignoring response <200", str(log))
# with level WARNING, we shouldn't capture anything from HttpError
crawler = get_crawler(_HttpErrorSpider)
with LogCapture(level=logging.WARNING) as log:
yield crawler.crawl(mockserver=self.mockserver)
- self.assertEqual(crawler.spider.parsed, {'200'})
- self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
+ self.assertEqual(crawler.spider.parsed, {"200"})
+ self.assertEqual(crawler.spider.failed, {"404", "402", "500"})
- self.assertNotIn('Ignoring response <402', str(log))
- self.assertNotIn('Ignoring response <404', str(log))
- self.assertNotIn('Ignoring response <500', str(log))
- self.assertNotIn('Ignoring response <200', str(log))
+ self.assertNotIn("Ignoring response <402", str(log))
+ self.assertNotIn("Ignoring response <404", str(log))
+ self.assertNotIn("Ignoring response <500", str(log))
+ self.assertNotIn("Ignoring response <200", str(log))
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 0f4b98a07..ea45b7698 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,15 +1,14 @@
+import warnings
from unittest import TestCase
from urllib.parse import urlparse
-import warnings
-from scrapy.http import Response, Request
+from scrapy.http import Request, Response
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning
from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning
from scrapy.utils.test import get_crawler
class TestOffsiteMiddleware(TestCase):
-
def setUp(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider(**self._get_spiderargs())
@@ -17,28 +16,31 @@ class TestOffsiteMiddleware(TestCase):
self.mw.spider_opened(self.spider)
def _get_spiderargs(self):
- return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org'])
+ return dict(
+ name="foo",
+ allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"],
+ )
def test_process_spider_output(self):
- res = Response('http://scrapytest.org')
+ res = Response("http://scrapytest.org")
onsite_reqs = [
- Request('http://scrapytest.org/1'),
- Request('http://scrapy.org/1'),
- Request('http://sub.scrapy.org/1'),
- Request('http://offsite.tld/letmepass', dont_filter=True),
- Request('http://scrapy.test.org/'),
- Request('http://scrapy.test.org:8000/'),
+ Request("http://scrapytest.org/1"),
+ Request("http://scrapy.org/1"),
+ Request("http://sub.scrapy.org/1"),
+ Request("http://offsite.tld/letmepass", dont_filter=True),
+ Request("http://scrapy.test.org/"),
+ Request("http://scrapy.test.org:8000/"),
]
offsite_reqs = [
- Request('http://scrapy2.org'),
- Request('http://offsite.tld/'),
- Request('http://offsite.tld/scrapytest.org'),
- Request('http://offsite.tld/rogue.scrapytest.org'),
- Request('http://rogue.scrapytest.org.haha.com'),
- Request('http://roguescrapytest.org'),
- Request('http://test.org/'),
- Request('http://notscrapy.test.org/'),
+ Request("http://scrapy2.org"),
+ Request("http://offsite.tld/"),
+ Request("http://offsite.tld/scrapytest.org"),
+ Request("http://offsite.tld/rogue.scrapytest.org"),
+ Request("http://rogue.scrapytest.org.haha.com"),
+ Request("http://roguescrapytest.org"),
+ Request("http://test.org/"),
+ Request("http://notscrapy.test.org/"),
]
reqs = onsite_reqs + offsite_reqs
@@ -47,40 +49,40 @@ class TestOffsiteMiddleware(TestCase):
class TestOffsiteMiddleware2(TestOffsiteMiddleware):
-
def _get_spiderargs(self):
- return dict(name='foo', allowed_domains=None)
+ return dict(name="foo", allowed_domains=None)
def test_process_spider_output(self):
- res = Response('http://scrapytest.org')
- reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')]
+ res = Response("http://scrapytest.org")
+ reqs = [Request("http://a.com/b.html"), Request("http://b.com/1")]
out = list(self.mw.process_spider_output(res, reqs, self.spider))
self.assertEqual(out, reqs)
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
-
def _get_spiderargs(self):
- return dict(name='foo')
+ return dict(name="foo")
class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
-
def _get_spiderargs(self):
- bad_hostname = urlparse('http:////scrapytest.org').hostname
- return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
+ bad_hostname = urlparse("http:////scrapytest.org").hostname
+ return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname])
def test_process_spider_output(self):
- res = Response('http://scrapytest.org')
- reqs = [Request('http://scrapytest.org/1')]
+ res = Response("http://scrapytest.org")
+ reqs = [Request("http://scrapytest.org/1")]
out = list(self.mw.process_spider_output(res, reqs, self.spider))
self.assertEqual(out, reqs)
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
-
def test_get_host_regex(self):
- self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']
+ self.spider.allowed_domains = [
+ "http://scrapytest.org",
+ "scrapy.org",
+ "scrapy.test.org",
+ ]
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
self.mw.get_host_regex(self.spider)
@@ -88,9 +90,12 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
class TestOffsiteMiddleware6(TestOffsiteMiddleware4):
-
def test_get_host_regex(self):
- self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org']
+ self.spider.allowed_domains = [
+ "scrapytest.org:8000",
+ "scrapy.org",
+ "scrapy.test.org",
+ ]
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
self.mw.get_host_regex(self.spider)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 8dd1def17..fad5dcaac 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -9,7 +9,9 @@ from tests.mockserver import MockServer
class LogExceptionMiddleware:
def process_spider_exception(self, response, exception, spider):
- spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+ spider.logger.info(
+ "Middleware: %s exception caught", exception.__class__.__name__
+ )
return None
@@ -17,34 +19,36 @@ class LogExceptionMiddleware:
# (0) recover from an exception on a spider callback
class RecoveryMiddleware:
def process_spider_exception(self, response, exception, spider):
- spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+ spider.logger.info(
+ "Middleware: %s exception caught", exception.__class__.__name__
+ )
return [
- {'from': 'process_spider_exception'},
- Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+ {"from": "process_spider_exception"},
+ Request(response.url, meta={"dont_fail": True}, dont_filter=True),
]
class RecoverySpider(Spider):
- name = 'RecoverySpider'
+ name = "RecoverySpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES_BASE': {},
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES_BASE": {},
+ "SPIDER_MIDDLEWARES": {
RecoveryMiddleware: 10,
},
}
def start_requests(self):
- yield Request(self.mockserver.url('/status?n=200'))
+ yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
- yield {'test': 1}
- self.logger.info('DONT_FAIL: %s', response.meta.get('dont_fail'))
- if not response.meta.get('dont_fail'):
+ yield {"test": 1}
+ self.logger.info("DONT_FAIL: %s", response.meta.get("dont_fail"))
+ if not response.meta.get("dont_fail"):
raise TabError()
class RecoveryAsyncGenSpider(RecoverySpider):
- name = 'RecoveryAsyncGenSpider'
+ name = "RecoveryAsyncGenSpider"
async def parse(self, response):
for r in super().parse(response):
@@ -55,14 +59,14 @@ class RecoveryAsyncGenSpider(RecoverySpider):
# (1) exceptions from a spider middleware's process_spider_input method
class FailProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
- spider.logger.info('Middleware: will raise IndexError')
+ spider.logger.info("Middleware: will raise IndexError")
raise IndexError()
class ProcessSpiderInputSpiderWithoutErrback(Spider):
- name = 'ProcessSpiderInputSpiderWithoutErrback'
+ name = "ProcessSpiderInputSpiderWithoutErrback"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
# spider
FailProcessSpiderInputMiddleware: 8,
LogExceptionMiddleware: 6,
@@ -71,55 +75,57 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
}
def start_requests(self):
- yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse)
+ yield Request(url=self.mockserver.url("/status?n=200"), callback=self.parse)
def parse(self, response):
- return {'from': 'callback'}
+ return {"from": "callback"}
class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
- name = 'ProcessSpiderInputSpiderWithErrback'
+ name = "ProcessSpiderInputSpiderWithErrback"
def start_requests(self):
- yield Request(self.mockserver.url('/status?n=200'), self.parse, errback=self.errback)
+ yield Request(
+ self.mockserver.url("/status?n=200"), self.parse, errback=self.errback
+ )
def errback(self, failure):
- self.logger.info('Got a Failure on the Request errback')
- return {'from': 'errback'}
+ self.logger.info("Got a Failure on the Request errback")
+ return {"from": "errback"}
# ================================================================================
# (2) exceptions from a spider callback (generator)
class GeneratorCallbackSpider(Spider):
- name = 'GeneratorCallbackSpider'
+ name = "GeneratorCallbackSpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
LogExceptionMiddleware: 10,
},
}
def start_requests(self):
- yield Request(self.mockserver.url('/status?n=200'))
+ yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
- yield {'test': 1}
- yield {'test': 2}
+ yield {"test": 1}
+ yield {"test": 2}
raise ImportError()
class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
async def parse(self, response):
- yield {'test': 1}
- yield {'test': 2}
+ yield {"test": 1}
+ yield {"test": 2}
raise ImportError()
# ================================================================================
# (2.1) exceptions from a spider callback (generator, middleware right after callback)
class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
- name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
+ name = "GeneratorCallbackSpiderMiddlewareRightAfterSpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
LogExceptionMiddleware: 100000,
},
}
@@ -128,26 +134,26 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
# ================================================================================
# (3) exceptions from a spider callback (not a generator)
class NotGeneratorCallbackSpider(Spider):
- name = 'NotGeneratorCallbackSpider'
+ name = "NotGeneratorCallbackSpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
LogExceptionMiddleware: 10,
},
}
def start_requests(self):
- yield Request(self.mockserver.url('/status?n=200'))
+ yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
- return [{'test': 1}, {'test': 1 / 0}]
+ return [{"test": 1}, {"test": 1 / 0}]
# ================================================================================
# (3.1) exceptions from a spider callback (not a generator, middleware right after callback)
class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider):
- name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
+ name = "NotGeneratorCallbackSpiderMiddlewareRightAfterSpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
LogExceptionMiddleware: 100000,
},
}
@@ -158,26 +164,26 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
class _GeneratorDoNothingMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
- r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+ r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
def process_spider_exception(self, response, exception, spider):
- method = f'{self.__class__.__name__}.process_spider_exception'
- spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
+ method = f"{self.__class__.__name__}.process_spider_exception"
+ spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
return None
class GeneratorFailMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
- r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+ r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
raise LookupError()
def process_spider_exception(self, response, exception, spider):
- method = f'{self.__class__.__name__}.process_spider_exception'
- spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
- yield {'processed': [method]}
+ method = f"{self.__class__.__name__}.process_spider_exception"
+ spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ yield {"processed": [method]}
class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
@@ -187,13 +193,13 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
class GeneratorRecoverMiddleware:
def process_spider_output(self, response, result, spider):
for r in result:
- r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+ r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
def process_spider_exception(self, response, exception, spider):
- method = f'{self.__class__.__name__}.process_spider_exception'
- spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
- yield {'processed': [method]}
+ method = f"{self.__class__.__name__}.process_spider_exception"
+ spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ yield {"processed": [method]}
class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
@@ -201,9 +207,9 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
class GeneratorOutputChainSpider(Spider):
- name = 'GeneratorOutputChainSpider'
+ name = "GeneratorOutputChainSpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
GeneratorFailMiddleware: 10,
GeneratorDoNothingAfterFailureMiddleware: 8,
GeneratorRecoverMiddleware: 5,
@@ -212,27 +218,28 @@ class GeneratorOutputChainSpider(Spider):
}
def start_requests(self):
- yield Request(self.mockserver.url('/status?n=200'))
+ yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
- yield {'processed': ['parse-first-item']}
- yield {'processed': ['parse-second-item']}
+ yield {"processed": ["parse-first-item"]}
+ yield {"processed": ["parse-second-item"]}
# ================================================================================
# (5) exceptions from a middleware process_spider_output method (not generator)
+
class _NotGeneratorDoNothingMiddleware:
def process_spider_output(self, response, result, spider):
out = []
for r in result:
- r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+ r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
def process_spider_exception(self, response, exception, spider):
- method = f'{self.__class__.__name__}.process_spider_exception'
- spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
+ method = f"{self.__class__.__name__}.process_spider_exception"
+ spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
return None
@@ -240,15 +247,15 @@ class NotGeneratorFailMiddleware:
def process_spider_output(self, response, result, spider):
out = []
for r in result:
- r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+ r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
raise ReferenceError()
return out
def process_spider_exception(self, response, exception, spider):
- method = f'{self.__class__.__name__}.process_spider_exception'
- spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
- return [{'processed': [method]}]
+ method = f"{self.__class__.__name__}.process_spider_exception"
+ spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ return [{"processed": [method]}]
class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware):
@@ -259,14 +266,14 @@ class NotGeneratorRecoverMiddleware:
def process_spider_output(self, response, result, spider):
out = []
for r in result:
- r['processed'].append(f'{self.__class__.__name__}.process_spider_output')
+ r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
def process_spider_exception(self, response, exception, spider):
- method = f'{self.__class__.__name__}.process_spider_exception'
- spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
- return [{'processed': [method]}]
+ method = f"{self.__class__.__name__}.process_spider_exception"
+ spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
+ return [{"processed": [method]}]
class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
@@ -274,9 +281,9 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew
class NotGeneratorOutputChainSpider(Spider):
- name = 'NotGeneratorOutputChainSpider'
+ name = "NotGeneratorOutputChainSpider"
custom_settings = {
- 'SPIDER_MIDDLEWARES': {
+ "SPIDER_MIDDLEWARES": {
NotGeneratorFailMiddleware: 10,
NotGeneratorDoNothingAfterFailureMiddleware: 8,
NotGeneratorRecoverMiddleware: 5,
@@ -285,10 +292,13 @@ class NotGeneratorOutputChainSpider(Spider):
}
def start_requests(self):
- return [Request(self.mockserver.url('/status?n=200'))]
+ return [Request(self.mockserver.url("/status?n=200"))]
def parse(self, response):
- return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
+ return [
+ {"processed": ["parse-first-item"]},
+ {"processed": ["parse-second-item"]},
+ ]
# ================================================================================
@@ -402,7 +412,9 @@ class TestSpiderMiddleware(TestCase):
(3.1) Special case of (3): Exceptions should be caught
even if the middleware is placed right after the spider
"""
- log31 = yield self.crawl_log(NotGeneratorCallbackSpiderMiddlewareRightAfterSpider)
+ log31 = yield self.crawl_log(
+ NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
+ )
self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31))
self.assertNotIn("item_scraped_count", str(log31))
@@ -418,28 +430,40 @@ class TestSpiderMiddleware(TestCase):
"""
log4 = yield self.crawl_log(GeneratorOutputChainSpider)
self.assertIn("'item_scraped_count': 2", str(log4))
- self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4))
+ self.assertIn(
+ "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught",
+ str(log4),
+ )
self.assertIn(
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught",
- str(log4))
+ str(log4),
+ )
self.assertNotIn(
"GeneratorFailMiddleware.process_spider_exception: LookupError caught",
- str(log4))
+ str(log4),
+ )
self.assertNotIn(
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught",
- str(log4))
- item_from_callback = {'processed': [
- 'parse-first-item',
- 'GeneratorFailMiddleware.process_spider_output',
- 'GeneratorDoNothingAfterFailureMiddleware.process_spider_output',
- 'GeneratorRecoverMiddleware.process_spider_output',
- 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
- item_recovered = {'processed': [
- 'GeneratorRecoverMiddleware.process_spider_exception',
- 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+ str(log4),
+ )
+ item_from_callback = {
+ "processed": [
+ "parse-first-item",
+ "GeneratorFailMiddleware.process_spider_output",
+ "GeneratorDoNothingAfterFailureMiddleware.process_spider_output",
+ "GeneratorRecoverMiddleware.process_spider_output",
+ "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
+ ]
+ }
+ item_recovered = {
+ "processed": [
+ "GeneratorRecoverMiddleware.process_spider_exception",
+ "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
+ ]
+ }
self.assertIn(str(item_from_callback), str(log4))
self.assertIn(str(item_recovered), str(log4))
- self.assertNotIn('parse-second-item', str(log4))
+ self.assertNotIn("parse-second-item", str(log4))
@defer.inlineCallbacks
def test_not_a_generator_output_chain(self):
@@ -453,17 +477,28 @@ class TestSpiderMiddleware(TestCase):
"""
log5 = yield self.crawl_log(NotGeneratorOutputChainSpider)
self.assertIn("'item_scraped_count': 1", str(log5))
- self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+ self.assertIn(
+ "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught",
+ str(log5),
+ )
self.assertIn(
"GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught",
- str(log5))
- self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5))
+ str(log5),
+ )
+ self.assertNotIn(
+ "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught",
+ str(log5),
+ )
self.assertNotIn(
"GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught",
- str(log5))
- item_recovered = {'processed': [
- 'NotGeneratorRecoverMiddleware.process_spider_exception',
- 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']}
+ str(log5),
+ )
+ item_recovered = {
+ "processed": [
+ "NotGeneratorRecoverMiddleware.process_spider_exception",
+ "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
+ ]
+ }
self.assertIn(str(item_recovered), str(log5))
- self.assertNotIn('parse-first-item', str(log5))
- self.assertNotIn('parse-second-item', str(log5))
+ self.assertNotIn("parse-first-item", str(log5))
+ self.assertNotIn("parse-second-item", str(log5))
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 63daf0b8a..afffa87fb 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,17 +1,12 @@
-from urllib.parse import urlparse
-from unittest import TestCase
import warnings
-from scrapy.http import Response, Request
+from typing import Any, Dict, List, Optional, Tuple
+from unittest import TestCase
+from urllib.parse import urlparse
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
+from scrapy.http import Request, Response
+from scrapy.settings import Settings
from scrapy.spidermiddlewares.referer import (
- DefaultReferrerPolicy,
- NoReferrerPolicy,
- NoReferrerWhenDowngradePolicy,
- OriginPolicy,
- OriginWhenCrossOriginPolicy,
POLICY_NO_REFERRER,
POLICY_NO_REFERRER_WHEN_DOWNGRADE,
POLICY_ORIGIN,
@@ -21,6 +16,11 @@ from scrapy.spidermiddlewares.referer import (
POLICY_STRICT_ORIGIN,
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
POLICY_UNSAFE_URL,
+ DefaultReferrerPolicy,
+ NoReferrerPolicy,
+ NoReferrerWhenDowngradePolicy,
+ OriginPolicy,
+ OriginWhenCrossOriginPolicy,
RefererMiddleware,
ReferrerPolicy,
SameOriginPolicy,
@@ -28,19 +28,19 @@ from scrapy.spidermiddlewares.referer import (
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
)
+from scrapy.spiders import Spider
class TestRefererMiddleware(TestCase):
-
- req_meta = {}
- resp_headers = {}
- settings = {}
- scenarii = [
- ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),
+ req_meta: Dict[str, Any] = {}
+ resp_headers: Dict[str, str] = {}
+ settings: Dict[str, Any] = {}
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
+ ("http://scrapytest.org", "http://scrapytest.org/", b"http://scrapytest.org"),
]
def setUp(self):
- self.spider = Spider('foo')
+ self.spider = Spider("foo")
settings = Settings(self.settings)
self.mw = RefererMiddleware(settings)
@@ -51,12 +51,11 @@ class TestRefererMiddleware(TestCase):
return Response(origin, headers=self.resp_headers)
def test(self):
-
for origin, target, referrer in self.scenarii:
response = self.get_response(origin)
request = self.get_request(target)
out = list(self.mw.process_spider_output(response, [request], self.spider))
- self.assertEqual(out[0].headers.get('Referer'), referrer)
+ self.assertEqual(out[0].headers.get("Referer"), referrer)
class MixinDefault:
@@ -65,271 +64,559 @@ class MixinDefault:
with some additional filtering of s3://
"""
- scenarii = [
- ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
- ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
- ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
- ('https://example.com/', 'http://scrapy.org/', None),
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
+ ("https://example.com/", "https://scrapy.org/", b"https://example.com/"),
+ ("http://example.com/", "http://scrapy.org/", b"http://example.com/"),
+ ("http://example.com/", "https://scrapy.org/", b"http://example.com/"),
+ ("https://example.com/", "http://scrapy.org/", None),
# no credentials leak
- ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
-
+ (
+ "http://user:password@example.com/",
+ "https://scrapy.org/",
+ b"http://example.com/",
+ ),
# no referrer leak for local schemes
- ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
- ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
-
+ ("file:///home/path/to/somefile.html", "https://scrapy.org/", None),
+ ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),
# no referrer leak for s3 origins
- ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
- ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
+ ("s3://mybucket/path/to/data.csv", "https://scrapy.org/", None),
+ ("s3://mybucket/path/to/data.csv", "http://scrapy.org/", None),
]
class MixinNoReferrer:
- scenarii = [
- ('https://example.com/page.html', 'https://example.com/', None),
- ('http://www.example.com/', 'https://scrapy.org/', None),
- ('http://www.example.com/', 'http://scrapy.org/', None),
- ('https://www.example.com/', 'http://scrapy.org/', None),
- ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
+ ("https://example.com/page.html", "https://example.com/", None),
+ ("http://www.example.com/", "https://scrapy.org/", None),
+ ("http://www.example.com/", "http://scrapy.org/", None),
+ ("https://www.example.com/", "http://scrapy.org/", None),
+ ("file:///home/path/to/somefile.html", "http://scrapy.org/", None),
]
class MixinNoReferrerWhenDowngrade:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# TLS to TLS: send non-empty referrer
- ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
- ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
- ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
- ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
- ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
-
+ (
+ "https://example.com/page.html",
+ "https://not.example.com/",
+ b"https://example.com/page.html",
+ ),
+ (
+ "https://example.com/page.html",
+ "https://scrapy.org/",
+ b"https://example.com/page.html",
+ ),
+ (
+ "https://example.com:443/page.html",
+ "https://scrapy.org/",
+ b"https://example.com/page.html",
+ ),
+ (
+ "https://example.com:444/page.html",
+ "https://scrapy.org/",
+ b"https://example.com:444/page.html",
+ ),
+ (
+ "ftps://example.com/urls.zip",
+ "https://scrapy.org/",
+ b"ftps://example.com/urls.zip",
+ ),
# TLS to non-TLS: do not send referrer
- ('https://example.com/page.html', 'http://not.example.com/', None),
- ('https://example.com/page.html', 'http://scrapy.org/', None),
- ('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
-
+ ("https://example.com/page.html", "http://not.example.com/", None),
+ ("https://example.com/page.html", "http://scrapy.org/", None),
+ ("ftps://example.com/urls.zip", "http://scrapy.org/", None),
# non-TLS to TLS or non-TLS: send referrer
- ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
- ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
- ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
- ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
- ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
- ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
- ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
- ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
-
+ (
+ "http://example.com/page.html",
+ "https://not.example.com/",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "https://scrapy.org/",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com:8080/page.html",
+ "https://scrapy.org/",
+ b"http://example.com:8080/page.html",
+ ),
+ (
+ "http://example.com:80/page.html",
+ "http://not.example.com/",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "http://scrapy.org/",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com:443/page.html",
+ "http://scrapy.org/",
+ b"http://example.com:443/page.html",
+ ),
+ (
+ "ftp://example.com/urls.zip",
+ "http://scrapy.org/",
+ b"ftp://example.com/urls.zip",
+ ),
+ (
+ "ftp://example.com/urls.zip",
+ "https://scrapy.org/",
+ b"ftp://example.com/urls.zip",
+ ),
# test for user/password stripping
- ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
+ (
+ "http://user:password@example.com/page.html",
+ "https://not.example.com/",
+ b"http://example.com/page.html",
+ ),
]
class MixinSameOrigin:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# Same origin (protocol, host, port): send referrer
- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
- ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
- ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
- ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
- ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
(
- 'http://example.com:8888/page.html',
- 'http://example.com:8888/not-page.html',
- b'http://example.com:8888/page.html',
+ "https://example.com/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
),
-
- # Different host: do NOT send referrer
- ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
- ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
- ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
-
- # Different port: do NOT send referrer
- ('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
- ('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
- ('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
-
- # Different protocols: do NOT send refferer
- ('https://example.com/page.html', 'http://example.com/not-page.html', None),
- ('https://example.com/page.html', 'http://not.example.com/', None),
- ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
- ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
- ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
-
- # test for user/password stripping
- ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
(
- 'https://user:password@example.com/page.html',
- 'https://example.com/not-page.html',
- b'https://example.com/page.html',
+ "http://example.com/page.html",
+ "http://example.com/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "https://example.com:443/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
+ ),
+ (
+ "http://example.com:80/page.html",
+ "http://example.com/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "http://example.com:80/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com:8888/page.html",
+ "http://example.com:8888/not-page.html",
+ b"http://example.com:8888/page.html",
+ ),
+ # Different host: do NOT send referrer
+ (
+ "https://example.com/page.html",
+ "https://not.example.com/otherpage.html",
+ None,
+ ),
+ ("http://example.com/page.html", "http://not.example.com/otherpage.html", None),
+ ("http://example.com/page.html", "http://www.example.com/otherpage.html", None),
+ # Different port: do NOT send referrer
+ (
+ "https://example.com:444/page.html",
+ "https://example.com/not-page.html",
+ None,
+ ),
+ ("http://example.com:81/page.html", "http://example.com/not-page.html", None),
+ ("http://example.com/page.html", "http://example.com:81/not-page.html", None),
+ # Different protocols: do NOT send referrer
+ ("https://example.com/page.html", "http://example.com/not-page.html", None),
+ ("https://example.com/page.html", "http://not.example.com/", None),
+ ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+ ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),
+ ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+ # test for user/password stripping
+ (
+ "https://user:password@example.com/page.html",
+ "http://example.com/not-page.html",
+ None,
+ ),
+ (
+ "https://user:password@example.com/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
),
]
class MixinOrigin:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
- ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
- ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
- ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
-
+ (
+ "https://example.com/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/",
+ ),
+ (
+ "https://example.com/page.html",
+ "https://scrapy.org",
+ b"https://example.com/",
+ ),
+ ("https://example.com/page.html", "http://scrapy.org", b"https://example.com/"),
+ ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),
# test for user/password stripping
- ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
+ (
+ "https://user:password@example.com/page.html",
+ "http://scrapy.org",
+ b"https://example.com/",
+ ),
]
class MixinStrictOrigin:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
- ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
- ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
-
+ (
+ "https://example.com/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/",
+ ),
+ (
+ "https://example.com/page.html",
+ "https://scrapy.org",
+ b"https://example.com/",
+ ),
+ ("http://example.com/page.html", "http://scrapy.org", b"http://example.com/"),
# downgrade: send nothing
- ('https://example.com/page.html', 'http://scrapy.org', None),
-
+ ("https://example.com/page.html", "http://scrapy.org", None),
# upgrade: send origin
- ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),
-
+ ("http://example.com/page.html", "https://scrapy.org", b"http://example.com/"),
# test for user/password stripping
- ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
- ('https://user:password@example.com/page.html', 'http://scrapy.org', None),
+ (
+ "https://user:password@example.com/page.html",
+ "https://scrapy.org",
+ b"https://example.com/",
+ ),
+ ("https://user:password@example.com/page.html", "http://scrapy.org", None),
]
class MixinOriginWhenCrossOrigin:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# Same origin (protocol, host, port): send referrer
- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
- ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
- ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
- ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
- ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
(
- 'http://example.com:8888/page.html',
- 'http://example.com:8888/not-page.html',
- b'http://example.com:8888/page.html',
+ "https://example.com/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "http://example.com/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "https://example.com:443/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
+ ),
+ (
+ "http://example.com:80/page.html",
+ "http://example.com/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "http://example.com:80/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com:8888/page.html",
+ "http://example.com:8888/not-page.html",
+ b"http://example.com:8888/page.html",
),
-
# Different host: send origin as referrer
- ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
- ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
- ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
+ (
+ "https://example2.com/page.html",
+ "https://scrapy.org/otherpage.html",
+ b"https://example2.com/",
+ ),
+ (
+ "https://example2.com/page.html",
+ "https://not.example2.com/otherpage.html",
+ b"https://example2.com/",
+ ),
+ (
+ "http://example2.com/page.html",
+ "http://not.example2.com/otherpage.html",
+ b"http://example2.com/",
+ ),
# exact match required
- ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
-
+ (
+ "http://example2.com/page.html",
+ "http://www.example2.com/otherpage.html",
+ b"http://example2.com/",
+ ),
# Different port: send origin as referrer
- ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
- ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
-
+ (
+ "https://example3.com:444/page.html",
+ "https://example3.com/not-page.html",
+ b"https://example3.com:444/",
+ ),
+ (
+ "http://example3.com:81/page.html",
+ "http://example3.com/not-page.html",
+ b"http://example3.com:81/",
+ ),
# Different protocols: send origin as referrer
- ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
- ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
- ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
-
+ (
+ "https://example4.com/page.html",
+ "http://example4.com/not-page.html",
+ b"https://example4.com/",
+ ),
+ (
+ "https://example4.com/page.html",
+ "http://not.example4.com/",
+ b"https://example4.com/",
+ ),
+ (
+ "ftps://example4.com/urls.zip",
+ "https://example4.com/not-page.html",
+ b"ftps://example4.com/",
+ ),
+ (
+ "ftp://example4.com/urls.zip",
+ "http://example4.com/not-page.html",
+ b"ftp://example4.com/",
+ ),
+ (
+ "ftps://example4.com/urls.zip",
+ "https://example4.com/not-page.html",
+ b"ftps://example4.com/",
+ ),
# test for user/password stripping
(
- 'https://user:password@example5.com/page.html',
- 'https://example5.com/not-page.html',
- b'https://example5.com/page.html',
+ "https://user:password@example5.com/page.html",
+ "https://example5.com/not-page.html",
+ b"https://example5.com/page.html",
),
# TLS to non-TLS downgrade: send origin
(
- 'https://user:password@example5.com/page.html',
- 'http://example5.com/not-page.html',
- b'https://example5.com/',
+ "https://user:password@example5.com/page.html",
+ "http://example5.com/not-page.html",
+ b"https://example5.com/",
),
]
class MixinStrictOriginWhenCrossOrigin:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# Same origin (protocol, host, port): send referrer
- ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
- ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
- ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
- ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
- ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
(
- 'http://example.com:8888/page.html',
- 'http://example.com:8888/not-page.html',
- b'http://example.com:8888/page.html',
+ "https://example.com/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "http://example.com/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "https://example.com:443/page.html",
+ "https://example.com/not-page.html",
+ b"https://example.com/page.html",
+ ),
+ (
+ "http://example.com:80/page.html",
+ "http://example.com/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com/page.html",
+ "http://example.com:80/not-page.html",
+ b"http://example.com/page.html",
+ ),
+ (
+ "http://example.com:8888/page.html",
+ "http://example.com:8888/not-page.html",
+ b"http://example.com:8888/page.html",
),
-
# Different host: send origin as referrer
- ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
- ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
- ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
+ (
+ "https://example2.com/page.html",
+ "https://scrapy.org/otherpage.html",
+ b"https://example2.com/",
+ ),
+ (
+ "https://example2.com/page.html",
+ "https://not.example2.com/otherpage.html",
+ b"https://example2.com/",
+ ),
+ (
+ "http://example2.com/page.html",
+ "http://not.example2.com/otherpage.html",
+ b"http://example2.com/",
+ ),
# exact match required
- ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
-
+ (
+ "http://example2.com/page.html",
+ "http://www.example2.com/otherpage.html",
+ b"http://example2.com/",
+ ),
# Different port: send origin as referrer
- ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
- ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
-
+ (
+ "https://example3.com:444/page.html",
+ "https://example3.com/not-page.html",
+ b"https://example3.com:444/",
+ ),
+ (
+ "http://example3.com:81/page.html",
+ "http://example3.com/not-page.html",
+ b"http://example3.com:81/",
+ ),
# downgrade
- ('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
- ('https://example4.com/page.html', 'http://not.example4.com/', None),
-
+ ("https://example4.com/page.html", "http://example4.com/not-page.html", None),
+ ("https://example4.com/page.html", "http://not.example4.com/", None),
# non-TLS to non-TLS
- ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
-
+ (
+ "ftp://example4.com/urls.zip",
+ "http://example4.com/not-page.html",
+ b"ftp://example4.com/",
+ ),
# upgrade
- ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
- ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
-
+ (
+ "http://example4.com/page.html",
+ "https://example4.com/not-page.html",
+ b"http://example4.com/",
+ ),
+ (
+ "http://example4.com/page.html",
+ "https://not.example4.com/",
+ b"http://example4.com/",
+ ),
# Different protocols: send origin as referrer
- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
- ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
-
+ (
+ "ftps://example4.com/urls.zip",
+ "https://example4.com/not-page.html",
+ b"ftps://example4.com/",
+ ),
+ (
+ "ftps://example4.com/urls.zip",
+ "https://example4.com/not-page.html",
+ b"ftps://example4.com/",
+ ),
# test for user/password stripping
(
- 'https://user:password@example5.com/page.html',
- 'https://example5.com/not-page.html',
- b'https://example5.com/page.html',
+ "https://user:password@example5.com/page.html",
+ "https://example5.com/not-page.html",
+ b"https://example5.com/page.html",
),
-
# TLS to non-TLS downgrade: send nothing
- ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
+ (
+ "https://user:password@example5.com/page.html",
+ "http://example5.com/not-page.html",
+ None,
+ ),
]
class MixinUnsafeUrl:
- scenarii = [
+ scenarii: List[Tuple[str, str, Optional[bytes]]] = [
# TLS to TLS: send referrer
- ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
- ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
- ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
- ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
- ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
- ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
-
- # TLS to non-TLS: send referrer (yes, it's unsafe)
- ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
- ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
- ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
-
- # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
- ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
- ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
- ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
- ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
- ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
- ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
- ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
- ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
-
- # test for user/password stripping
(
- 'http://user:password@example4.com/page.html',
- 'https://not.example4.com/',
- b'http://example4.com/page.html',
+ "https://example.com/sekrit.html",
+ "http://not.example.com/",
+ b"https://example.com/sekrit.html",
),
(
- 'https://user:password@example4.com/page.html',
- 'http://scrapy.org/',
- b'https://example4.com/page.html',
+ "https://example1.com/page.html",
+ "https://not.example1.com/",
+ b"https://example1.com/page.html",
+ ),
+ (
+ "https://example1.com/page.html",
+ "https://scrapy.org/",
+ b"https://example1.com/page.html",
+ ),
+ (
+ "https://example1.com:443/page.html",
+ "https://scrapy.org/",
+ b"https://example1.com/page.html",
+ ),
+ (
+ "https://example1.com:444/page.html",
+ "https://scrapy.org/",
+ b"https://example1.com:444/page.html",
+ ),
+ (
+ "ftps://example1.com/urls.zip",
+ "https://scrapy.org/",
+ b"ftps://example1.com/urls.zip",
+ ),
+ # TLS to non-TLS: send referrer (yes, it's unsafe)
+ (
+ "https://example2.com/page.html",
+ "http://not.example2.com/",
+ b"https://example2.com/page.html",
+ ),
+ (
+ "https://example2.com/page.html",
+ "http://scrapy.org/",
+ b"https://example2.com/page.html",
+ ),
+ (
+ "ftps://example2.com/urls.zip",
+ "http://scrapy.org/",
+ b"ftps://example2.com/urls.zip",
+ ),
+ # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
+ (
+ "http://example3.com/page.html",
+ "https://not.example3.com/",
+ b"http://example3.com/page.html",
+ ),
+ (
+ "http://example3.com/page.html",
+ "https://scrapy.org/",
+ b"http://example3.com/page.html",
+ ),
+ (
+ "http://example3.com:8080/page.html",
+ "https://scrapy.org/",
+ b"http://example3.com:8080/page.html",
+ ),
+ (
+ "http://example3.com:80/page.html",
+ "http://not.example3.com/",
+ b"http://example3.com/page.html",
+ ),
+ (
+ "http://example3.com/page.html",
+ "http://scrapy.org/",
+ b"http://example3.com/page.html",
+ ),
+ (
+ "http://example3.com:443/page.html",
+ "http://scrapy.org/",
+ b"http://example3.com:443/page.html",
+ ),
+ (
+ "ftp://example3.com/urls.zip",
+ "http://scrapy.org/",
+ b"ftp://example3.com/urls.zip",
+ ),
+ (
+ "ftp://example3.com/urls.zip",
+ "https://scrapy.org/",
+ b"ftp://example3.com/urls.zip",
+ ),
+ # test for user/password stripping
+ (
+ "http://user:password@example4.com/page.html",
+ "https://not.example4.com/",
+ b"http://example4.com/page.html",
+ ),
+ (
+ "https://user:password@example4.com/page.html",
+ "http://scrapy.org/",
+ b"https://example4.com/page.html",
),
]
@@ -340,35 +627,49 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
# --- Tests using settings to set policy using class path
class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerPolicy"}
-class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
+class TestSettingsNoReferrerWhenDowngrade(
+ MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
+ }
class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginPolicy"}
class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginPolicy"
+ }
-class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+class TestSettingsOriginWhenCrossOrigin(
+ MixinOriginWhenCrossOrigin, TestRefererMiddleware
+):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+ }
-class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
+class TestSettingsStrictOriginWhenCrossOrigin(
+ MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware
+):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy"
+ }
class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
class CustomPythonOrgPolicy(ReferrerPolicy):
@@ -376,137 +677,161 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
A dummy policy that returns referrer as http(s)://python.org
depending on the scheme of the target URL.
"""
+
def referrer(self, response, request):
scheme = urlparse(request).scheme
- if scheme == 'https':
- return b'https://python.org/'
- if scheme == 'http':
- return b'http://python.org/'
+ if scheme == "https":
+ return b"https://python.org/"
+ if scheme == "http":
+ return b"http://python.org/"
class TestSettingsCustomPolicy(TestRefererMiddleware):
- settings = {'REFERRER_POLICY': CustomPythonOrgPolicy}
+ settings = {"REFERRER_POLICY": CustomPythonOrgPolicy}
scenarii = [
- ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
- ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
- ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'),
- ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'),
- ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'),
- ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'),
-
+ ("https://example.com/", "https://scrapy.org/", b"https://python.org/"),
+ ("http://example.com/", "http://scrapy.org/", b"http://python.org/"),
+ ("http://example.com/", "https://scrapy.org/", b"https://python.org/"),
+ ("https://example.com/", "http://scrapy.org/", b"http://python.org/"),
+ (
+ "file:///home/path/to/somefile.html",
+ "https://scrapy.org/",
+ b"https://python.org/",
+ ),
+ (
+ "file:///home/path/to/somefile.html",
+ "http://scrapy.org/",
+ b"http://python.org/",
+ ),
]
# --- Tests using Request meta dict to set policy
class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
+ req_meta = {"referrer_policy": POLICY_SCRAPY_DEFAULT}
class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+ req_meta = {"referrer_policy": POLICY_NO_REFERRER}
-class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}
+class TestRequestMetaNoReferrerWhenDowngrade(
+ MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
+ req_meta = {"referrer_policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE}
class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}
+ req_meta = {"referrer_policy": POLICY_SAME_ORIGIN}
class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_ORIGIN}
+ req_meta = {"referrer_policy": POLICY_ORIGIN}
class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}
+ req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN}
-class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+class TestRequestMetaOriginWhenCrossOrigin(
+ MixinOriginWhenCrossOrigin, TestRefererMiddleware
+):
+ req_meta = {"referrer_policy": POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
-class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+class TestRequestMetaStrictOriginWhenCrossOrigin(
+ MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware
+):
+ req_meta = {"referrer_policy": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
- req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+ req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
-class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
- req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+class TestRequestMetaPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
+ req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
-class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
- req_meta = {'referrer_policy': POLICY_NO_REFERRER}
+class TestRequestMetaPrecedence002(MixinNoReferrer, TestRefererMiddleware):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
+ }
+ req_meta = {"referrer_policy": POLICY_NO_REFERRER}
-class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
- req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
+class TestRequestMetaPrecedence003(MixinUnsafeUrl, TestRefererMiddleware):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+ }
+ req_meta = {"referrer_policy": POLICY_UNSAFE_URL}
class TestRequestMetaSettingFallback(TestCase):
-
params = [
(
# When an unknown policy is referenced in Request.meta
# (here, a typo error),
# the policy defined in settings takes precedence
- {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'},
+ {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+ },
{},
- {'referrer_policy': 'ssscrapy-default'},
+ {"referrer_policy": "ssscrapy-default"},
OriginWhenCrossOriginPolicy,
- True
+ True,
),
(
# same as above but with string value for settings policy
- {'REFERRER_POLICY': 'origin-when-cross-origin'},
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
{},
- {'referrer_policy': 'ssscrapy-default'},
+ {"referrer_policy": "ssscrapy-default"},
OriginWhenCrossOriginPolicy,
- True
+ True,
),
(
# request meta references a wrong policy but it is set,
# so the Referrer-Policy header in response is not used,
# and the settings' policy is applied
- {'REFERRER_POLICY': 'origin-when-cross-origin'},
- {'Referrer-Policy': 'unsafe-url'},
- {'referrer_policy': 'ssscrapy-default'},
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {"Referrer-Policy": "unsafe-url"},
+ {"referrer_policy": "ssscrapy-default"},
OriginWhenCrossOriginPolicy,
- True
+ True,
),
(
# here, request meta does not set the policy
# so response headers take precedence
- {'REFERRER_POLICY': 'origin-when-cross-origin'},
- {'Referrer-Policy': 'unsafe-url'},
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {"Referrer-Policy": "unsafe-url"},
{},
UnsafeUrlPolicy,
- False
+ False,
),
(
# here, request meta does not set the policy,
# but response headers also use an unknown policy,
# so the settings' policy is used
- {'REFERRER_POLICY': 'origin-when-cross-origin'},
- {'Referrer-Policy': 'unknown'},
+ {"REFERRER_POLICY": "origin-when-cross-origin"},
+ {"Referrer-Policy": "unknown"},
{},
OriginWhenCrossOriginPolicy,
- True
- )
+ True,
+ ),
]
def test(self):
+ origin = "http://www.scrapy.org"
+ target = "http://www.example.com"
- origin = 'http://www.scrapy.org'
- target = 'http://www.example.com'
-
- for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
+ for (
+ settings,
+ response_headers,
+ request_meta,
+ policy_class,
+ check_warning,
+ ) in self.params[3:]:
mw = RefererMiddleware(Settings(settings))
response = Response(origin, headers=response_headers)
@@ -522,7 +847,6 @@ class TestRequestMetaSettingFallback(TestCase):
class TestSettingsPolicyByName(TestCase):
-
def test_valid_name(self):
for s, p in [
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
@@ -535,7 +859,7 @@ class TestSettingsPolicyByName(TestCase):
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
]:
- settings = Settings({'REFERRER_POLICY': s})
+ settings = Settings({"REFERRER_POLICY": s})
mw = RefererMiddleware(settings)
self.assertEqual(mw.default_policy, p)
@@ -551,98 +875,121 @@ class TestSettingsPolicyByName(TestCase):
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
]:
- settings = Settings({'REFERRER_POLICY': s.upper()})
+ settings = Settings({"REFERRER_POLICY": s.upper()})
mw = RefererMiddleware(settings)
self.assertEqual(mw.default_policy, p)
def test_invalid_name(self):
- settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
+ settings = Settings({"REFERRER_POLICY": "some-custom-unknown-policy"})
with self.assertRaises(RuntimeError):
RefererMiddleware(settings)
-class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
- resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
+class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}
+ resp_headers = {"Referrer-Policy": POLICY_UNSAFE_URL.upper()}
-class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
- resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
+class TestPolicyHeaderPrecedence002(MixinNoReferrer, TestRefererMiddleware):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy"
+ }
+ resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER.swapcase()}
-class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
- resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
+class TestPolicyHeaderPrecedence003(
+ MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+ }
+ resp_headers = {"Referrer-Policy": POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
-class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+class TestPolicyHeaderPrecedence004(
+ MixinNoReferrerWhenDowngrade, TestRefererMiddleware
+):
"""
The empty string means "no-referrer-when-downgrade"
"""
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
- resp_headers = {'Referrer-Policy': ''}
+
+ settings = {
+ "REFERRER_POLICY": "scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy"
+ }
+ resp_headers = {"Referrer-Policy": ""}
class TestReferrerOnRedirect(TestRefererMiddleware):
-
- settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
- scenarii = [
+ settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
+ scenarii: List[
+ Tuple[str, str, Tuple[Tuple[int, str], ...], Optional[bytes], Optional[bytes]]
+ ] = [ # type: ignore[assignment]
(
- 'http://scrapytest.org/1', # parent
- 'http://scrapytest.org/2', # target
+ "http://scrapytest.org/1", # parent
+ "http://scrapytest.org/2", # target
(
# redirections: code, URL
- (301, 'http://scrapytest.org/3'),
- (301, 'http://scrapytest.org/4'),
+ (301, "http://scrapytest.org/3"),
+ (301, "http://scrapytest.org/4"),
),
- b'http://scrapytest.org/1', # expected initial referer
- b'http://scrapytest.org/1', # expected referer for the redirection request
+ b"http://scrapytest.org/1", # expected initial referer
+ b"http://scrapytest.org/1", # expected referer for the redirection request
),
(
- 'https://scrapytest.org/1',
- 'https://scrapytest.org/2',
+ "https://scrapytest.org/1",
+ "https://scrapytest.org/2",
(
# redirecting to non-secure URL
- (301, 'http://scrapytest.org/3'),
+ (301, "http://scrapytest.org/3"),
),
- b'https://scrapytest.org/1',
- b'https://scrapytest.org/1',
+ b"https://scrapytest.org/1",
+ b"https://scrapytest.org/1",
),
(
- 'https://scrapytest.org/1',
- 'https://scrapytest.com/2',
+ "https://scrapytest.org/1",
+ "https://scrapytest.com/2",
(
# redirecting to non-secure URL: different origin
- (301, 'http://scrapytest.com/3'),
+ (301, "http://scrapytest.com/3"),
),
- b'https://scrapytest.org/1',
- b'https://scrapytest.org/1',
+ b"https://scrapytest.org/1",
+ b"https://scrapytest.org/1",
),
]
def setUp(self):
- self.spider = Spider('foo')
+ self.spider = Spider("foo")
settings = Settings(self.settings)
self.referrermw = RefererMiddleware(settings)
self.redirectmw = RedirectMiddleware(settings)
def test(self):
-
- for parent, target, redirections, init_referrer, final_referrer in self.scenarii:
+ for (
+ parent,
+ target,
+ redirections,
+ init_referrer,
+ final_referrer,
+ ) in self.scenarii:
response = self.get_response(parent)
request = self.get_request(target)
- out = list(self.referrermw.process_spider_output(response, [request], self.spider))
- self.assertEqual(out[0].headers.get('Referer'), init_referrer)
+ out = list(
+ self.referrermw.process_spider_output(response, [request], self.spider)
+ )
+ self.assertEqual(out[0].headers.get("Referer"), init_referrer)
for status, url in redirections:
- response = Response(request.url, headers={'Location': url}, status=status)
- request = self.redirectmw.process_response(request, response, self.spider)
+ response = Response(
+ request.url, headers={"Location": url}, status=status
+ )
+ request = self.redirectmw.process_response(
+ request, response, self.spider
+ )
self.referrermw.request_scheduled(request, self.spider)
assert isinstance(request, Request)
- self.assertEqual(request.headers.get('Referer'), final_referrer)
+ self.assertEqual(request.headers.get("Referer"), final_referrer)
class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
@@ -650,34 +997,31 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
No Referrer policy never sets the "Referer" header.
HTTP redirections should not change that.
"""
- settings = {'REFERRER_POLICY': 'no-referrer'}
+
+ settings = {"REFERRER_POLICY": "no-referrer"}
scenarii = [
(
- 'http://scrapytest.org/1', # parent
- 'http://scrapytest.org/2', # target
+ "http://scrapytest.org/1", # parent
+ "http://scrapytest.org/2", # target
(
# redirections: code, URL
- (301, 'http://scrapytest.org/3'),
- (301, 'http://scrapytest.org/4'),
+ (301, "http://scrapytest.org/3"),
+ (301, "http://scrapytest.org/4"),
),
None, # expected initial "Referer"
None, # expected "Referer" for the redirection request
),
(
- 'https://scrapytest.org/1',
- 'https://scrapytest.org/2',
- (
- (301, 'http://scrapytest.org/3'),
- ),
+ "https://scrapytest.org/1",
+ "https://scrapytest.org/2",
+ ((301, "http://scrapytest.org/3"),),
None,
None,
),
(
- 'https://scrapytest.org/1',
- 'https://example.com/2', # different origin
- (
- (301, 'http://scrapytest.com/3'),
- ),
+ "https://scrapytest.org/1",
+ "https://example.com/2", # different origin
+ ((301, "http://scrapytest.com/3"),),
None,
None,
),
@@ -692,37 +1036,38 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
HTTP redirections to a different domain or a lower secure level
should have the "Referer" removed.
"""
- settings = {'REFERRER_POLICY': 'same-origin'}
+
+ settings = {"REFERRER_POLICY": "same-origin"}
scenarii = [
(
- 'http://scrapytest.org/101', # origin
- 'http://scrapytest.org/102', # target
+ "http://scrapytest.org/101", # origin
+ "http://scrapytest.org/102", # target
(
# redirections: code, URL
- (301, 'http://scrapytest.org/103'),
- (301, 'http://scrapytest.org/104'),
+ (301, "http://scrapytest.org/103"),
+ (301, "http://scrapytest.org/104"),
),
- b'http://scrapytest.org/101', # expected initial "Referer"
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b"http://scrapytest.org/101", # expected initial "Referer"
+ b"http://scrapytest.org/101", # expected referer for the redirection request
),
(
- 'https://scrapytest.org/201',
- 'https://scrapytest.org/202',
+ "https://scrapytest.org/201",
+ "https://scrapytest.org/202",
(
# redirecting from secure to non-secure URL == different origin
- (301, 'http://scrapytest.org/203'),
+ (301, "http://scrapytest.org/203"),
),
- b'https://scrapytest.org/201',
+ b"https://scrapytest.org/201",
None,
),
(
- 'https://scrapytest.org/301',
- 'https://scrapytest.org/302',
+ "https://scrapytest.org/301",
+ "https://scrapytest.org/302",
(
# different domain == different origin
- (301, 'http://example.com/303'),
+ (301, "http://example.com/303"),
),
- b'https://scrapytest.org/301',
+ b"https://scrapytest.org/301",
None,
),
]
@@ -737,63 +1082,65 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
Redirections from secure to non-secure URLs should have the
"Referrer" header removed if necessary.
"""
- settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
+
+ settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN}
scenarii = [
(
- 'http://scrapytest.org/101',
- 'http://scrapytest.org/102',
+ "http://scrapytest.org/101",
+ "http://scrapytest.org/102",
(
- (301, 'http://scrapytest.org/103'),
- (301, 'http://scrapytest.org/104'),
+ (301, "http://scrapytest.org/103"),
+ (301, "http://scrapytest.org/104"),
),
- b'http://scrapytest.org/', # send origin
- b'http://scrapytest.org/', # redirects to same origin: send origin
+ b"http://scrapytest.org/", # send origin
+ b"http://scrapytest.org/", # redirects to same origin: send origin
),
(
- 'https://scrapytest.org/201',
- 'https://scrapytest.org/202',
+ "https://scrapytest.org/201",
+ "https://scrapytest.org/202",
(
# redirecting to non-secure URL: no referrer
- (301, 'http://scrapytest.org/203'),
+ (301, "http://scrapytest.org/203"),
),
- b'https://scrapytest.org/',
+ b"https://scrapytest.org/",
None,
),
(
- 'https://scrapytest.org/301',
- 'https://scrapytest.org/302',
+ "https://scrapytest.org/301",
+ "https://scrapytest.org/302",
(
# redirecting to non-secure URL (different domain): no referrer
- (301, 'http://example.com/303'),
+ (301, "http://example.com/303"),
),
- b'https://scrapytest.org/',
+ b"https://scrapytest.org/",
None,
),
(
- 'http://scrapy.org/401',
- 'http://example.com/402',
- (
- (301, 'http://scrapytest.org/403'),
- ),
- b'http://scrapy.org/',
- b'http://scrapy.org/',
+ "http://scrapy.org/401",
+ "http://example.com/402",
+ ((301, "http://scrapytest.org/403"),),
+ b"http://scrapy.org/",
+ b"http://scrapy.org/",
),
(
- 'https://scrapy.org/501',
- 'https://example.com/502',
+ "https://scrapy.org/501",
+ "https://example.com/502",
(
# HTTPS all along, so origin referrer is kept as-is
- (301, 'https://google.com/503'),
- (301, 'https://facebook.com/504'),
+ (301, "https://google.com/503"),
+ (301, "https://facebook.com/504"),
),
- b'https://scrapy.org/',
- b'https://scrapy.org/',
+ b"https://scrapy.org/",
+ b"https://scrapy.org/",
),
(
- 'https://scrapytest.org/601',
- 'http://scrapytest.org/602', # TLS to non-TLS: no referrer
+ "https://scrapytest.org/601",
+ "http://scrapytest.org/602", # TLS to non-TLS: no referrer
(
- (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer
+ (
+ 301,
+ "https://scrapytest.org/603",
+ ), # TLS URL again: (still) no referrer
),
None,
None,
@@ -810,67 +1157,64 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
Redirections to a different origin should strip the "Referer"
to the parent origin.
"""
- settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
+
+ settings = {"REFERRER_POLICY": POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
scenarii = [
(
- 'http://scrapytest.org/101', # origin
- 'http://scrapytest.org/102', # target + redirection
+ "http://scrapytest.org/101", # origin
+ "http://scrapytest.org/102", # target + redirection
(
# redirections: code, URL
- (301, 'http://scrapytest.org/103'),
- (301, 'http://scrapytest.org/104'),
+ (301, "http://scrapytest.org/103"),
+ (301, "http://scrapytest.org/104"),
),
- b'http://scrapytest.org/101', # expected initial referer
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b"http://scrapytest.org/101", # expected initial referer
+ b"http://scrapytest.org/101", # expected referer for the redirection request
),
(
- 'https://scrapytest.org/201',
- 'https://scrapytest.org/202',
+ "https://scrapytest.org/201",
+ "https://scrapytest.org/202",
(
# redirecting to non-secure URL: send origin
- (301, 'http://scrapytest.org/203'),
+ (301, "http://scrapytest.org/203"),
),
- b'https://scrapytest.org/201',
- b'https://scrapytest.org/',
+ b"https://scrapytest.org/201",
+ b"https://scrapytest.org/",
),
(
- 'https://scrapytest.org/301',
- 'https://scrapytest.org/302',
+ "https://scrapytest.org/301",
+ "https://scrapytest.org/302",
(
# redirecting to non-secure URL (different domain): send origin
- (301, 'http://example.com/303'),
+ (301, "http://example.com/303"),
),
- b'https://scrapytest.org/301',
- b'https://scrapytest.org/',
+ b"https://scrapytest.org/301",
+ b"https://scrapytest.org/",
),
(
- 'http://scrapy.org/401',
- 'http://example.com/402',
- (
- (301, 'http://scrapytest.org/403'),
- ),
- b'http://scrapy.org/',
- b'http://scrapy.org/',
+ "http://scrapy.org/401",
+ "http://example.com/402",
+ ((301, "http://scrapytest.org/403"),),
+ b"http://scrapy.org/",
+ b"http://scrapy.org/",
),
(
- 'https://scrapy.org/501',
- 'https://example.com/502',
+ "https://scrapy.org/501",
+ "https://example.com/502",
(
# all different domains: send origin
- (301, 'https://google.com/503'),
- (301, 'https://facebook.com/504'),
+ (301, "https://google.com/503"),
+ (301, "https://facebook.com/504"),
),
- b'https://scrapy.org/',
- b'https://scrapy.org/',
+ b"https://scrapy.org/",
+ b"https://scrapy.org/",
),
(
- 'https://scrapytest.org/301',
- 'http://scrapytest.org/302', # TLS to non-TLS: send origin
- (
- (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also)
- ),
- b'https://scrapytest.org/',
- b'https://scrapytest.org/',
+ "https://scrapytest.org/301",
+ "http://scrapytest.org/302", # TLS to non-TLS: send origin
+ ((301, "https://scrapytest.org/303"),), # TLS URL again: send origin (also)
+ b"https://scrapytest.org/",
+ b"https://scrapytest.org/",
),
]
@@ -886,64 +1230,66 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
Redirections to a different origin should strip the "Referer" to the parent origin,
and from https:// to http:// will remove the "Referer" header.
"""
- settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
+
+ settings = {"REFERRER_POLICY": POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
scenarii = [
(
- 'http://scrapytest.org/101', # origin
- 'http://scrapytest.org/102', # target + redirection
+ "http://scrapytest.org/101", # origin
+ "http://scrapytest.org/102", # target + redirection
(
# redirections: code, URL
- (301, 'http://scrapytest.org/103'),
- (301, 'http://scrapytest.org/104'),
+ (301, "http://scrapytest.org/103"),
+ (301, "http://scrapytest.org/104"),
),
- b'http://scrapytest.org/101', # expected initial referer
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b"http://scrapytest.org/101", # expected initial referer
+ b"http://scrapytest.org/101", # expected referer for the redirection request
),
(
- 'https://scrapytest.org/201',
- 'https://scrapytest.org/202',
+ "https://scrapytest.org/201",
+ "https://scrapytest.org/202",
(
# redirecting to non-secure URL: do not send the "Referer" header
- (301, 'http://scrapytest.org/203'),
+ (301, "http://scrapytest.org/203"),
),
- b'https://scrapytest.org/201',
+ b"https://scrapytest.org/201",
None,
),
(
- 'https://scrapytest.org/301',
- 'https://scrapytest.org/302',
+ "https://scrapytest.org/301",
+ "https://scrapytest.org/302",
(
# redirecting to non-secure URL (different domain): send origin
- (301, 'http://example.com/303'),
+ (301, "http://example.com/303"),
),
- b'https://scrapytest.org/301',
+ b"https://scrapytest.org/301",
None,
),
(
- 'http://scrapy.org/401',
- 'http://example.com/402',
- (
- (301, 'http://scrapytest.org/403'),
- ),
- b'http://scrapy.org/',
- b'http://scrapy.org/',
+ "http://scrapy.org/401",
+ "http://example.com/402",
+ ((301, "http://scrapytest.org/403"),),
+ b"http://scrapy.org/",
+ b"http://scrapy.org/",
),
(
- 'https://scrapy.org/501',
- 'https://example.com/502',
+ "https://scrapy.org/501",
+ "https://example.com/502",
(
# all different domains: send origin
- (301, 'https://google.com/503'),
- (301, 'https://facebook.com/504'),
+ (301, "https://google.com/503"),
+ (301, "https://facebook.com/504"),
),
- b'https://scrapy.org/',
- b'https://scrapy.org/',
+ b"https://scrapy.org/",
+ b"https://scrapy.org/",
),
(
- 'https://scrapytest.org/601',
- 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer"
+ "https://scrapytest.org/601",
+ "http://scrapytest.org/602", # TLS to non-TLS: do not send "Referer"
(
- (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing
+ (
+ 301,
+ "https://scrapytest.org/603",
+ ), # TLS URL again: (still) send nothing
),
None,
None,
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 171f4ddfd..9111e4c82 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -2,31 +2,32 @@ from unittest import TestCase
from testfixtures import LogCapture
+from scrapy.http import Request, Response
+from scrapy.settings import Settings
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
-from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
-from scrapy.settings import Settings
class TestUrlLengthMiddleware(TestCase):
-
def setUp(self):
self.maxlength = 25
- settings = Settings({'URLLENGTH_LIMIT': self.maxlength})
+ settings = Settings({"URLLENGTH_LIMIT": self.maxlength})
crawler = get_crawler(Spider)
- self.spider = crawler._create_spider('foo')
+ self.spider = crawler._create_spider("foo")
self.stats = crawler.stats
self.mw = UrlLengthMiddleware.from_settings(settings)
- self.response = Response('http://scrapytest.org')
- self.short_url_req = Request('http://scrapytest.org/')
- self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
+ self.response = Response("http://scrapytest.org")
+ self.short_url_req = Request("http://scrapytest.org/")
+ self.long_url_req = Request("http://scrapytest.org/this_is_a_long_url")
self.reqs = [self.short_url_req, self.long_url_req]
def process_spider_output(self):
- return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
+ return list(
+ self.mw.process_spider_output(self.response, self.reqs, self.spider)
+ )
def test_middleware_works(self):
self.assertEqual(self.process_spider_output(), [self.short_url_req])
@@ -35,7 +36,9 @@ class TestUrlLengthMiddleware(TestCase):
with LogCapture() as log:
self.process_spider_output()
- ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider)
+ ric = self.stats.get_value(
+ "urllength/request_ignored_count", spider=self.spider
+ )
self.assertEqual(ric, 1)
- self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log))
+ self.assertIn(f"Ignoring link (url length > {self.maxlength})", str(log))
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index ab215576e..f97125b76 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,33 +1,33 @@
-from datetime import datetime
-from pathlib import Path
import shutil
+from datetime import datetime, timezone
+from pathlib import Path
+
from twisted.trial import unittest
+from scrapy.exceptions import NotConfigured
from scrapy.extensions.spiderstate import SpiderState
from scrapy.spiders import Spider
-from scrapy.exceptions import NotConfigured
from scrapy.utils.test import get_crawler
class SpiderStateTest(unittest.TestCase):
-
def test_store_load(self):
jobdir = self.mktemp()
Path(jobdir).mkdir()
try:
- spider = Spider(name='default')
- dt = datetime.now()
+ spider = Spider(name="default")
+ dt = datetime.now(tz=timezone.utc)
ss = SpiderState(jobdir)
ss.spider_opened(spider)
- spider.state['one'] = 1
- spider.state['dt'] = dt
+ spider.state["one"] = 1
+ spider.state["dt"] = dt
ss.spider_closed(spider)
- spider2 = Spider(name='default')
+ spider2 = Spider(name="default")
ss2 = SpiderState(jobdir)
ss2.spider_opened(spider2)
- self.assertEqual(spider.state, {'one': 1, 'dt': dt})
+ self.assertEqual(spider.state, {"one": 1, "dt": dt})
ss2.spider_closed(spider2)
finally:
shutil.rmtree(jobdir)
@@ -35,7 +35,7 @@ class SpiderStateTest(unittest.TestCase):
def test_state_attribute(self):
# state attribute must be present if jobdir is not set, to provide a
# consistent interface
- spider = Spider(name='default')
+ spider = Spider(name="default")
ss = SpiderState()
ss.spider_opened(spider)
self.assertEqual(spider.state, {})
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index acc821b83..4ce7cc9a4 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -2,16 +2,17 @@ import pickle
import sys
from queuelib.tests import test_queue as t
+
+from scrapy.http import Request
+from scrapy.item import Field, Item
+from scrapy.loader import ItemLoader
+from scrapy.selector import Selector
from scrapy.squeues import (
_MarshalFifoSerializationDiskQueue,
_MarshalLifoSerializationDiskQueue,
_PickleFifoSerializationDiskQueue,
_PickleLifoSerializationDiskQueue,
)
-from scrapy.item import Item, Field
-from scrapy.http import Request
-from scrapy.loader import ItemLoader
-from scrapy.selector import Selector
class TestItem(Item):
@@ -31,20 +32,19 @@ def nonserializable_object_test(self):
q = self.queue()
self.assertRaises(ValueError, q.push, lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
- sel = Selector(text='<html><body><p>some text</p></body></html>')
+ sel = Selector(text="<html><body><p>some text</p></body></html>")
self.assertRaises(ValueError, q.push, sel)
class FifoDiskQueueTestMixin:
-
def test_serialize(self):
q = self.queue()
- q.push('a')
+ q.push("a")
q.push(123)
- q.push({'a': 'dict'})
- self.assertEqual(q.pop(), 'a')
+ q.push({"a": "dict"})
+ self.assertEqual(q.pop(), "a")
self.assertEqual(q.pop(), 123)
- self.assertEqual(q.pop(), {'a': 'dict'})
+ self.assertEqual(q.pop(), {"a": "dict"})
test_nonserializable_object = nonserializable_object_test
@@ -73,7 +73,6 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
-
chunksize = 100000
def queue(self):
@@ -81,7 +80,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
def test_serialize_item(self):
q = self.queue()
- i = TestItem(name='foo')
+ i = TestItem(name="foo")
q.push(i)
i2 = q.pop()
assert isinstance(i2, TestItem)
@@ -94,17 +93,17 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
- self.assertEqual(loader2.name_out('x'), 'xx')
+ self.assertEqual(loader2.name_out("x"), "xx")
def test_serialize_request_recursive(self):
q = self.queue()
- r = Request('http://www.example.com')
- r.meta['request'] = r
+ r = Request("http://www.example.com")
+ r.meta["request"] = r
q.push(r)
r2 = q.pop()
assert isinstance(r2, Request)
self.assertEqual(r.url, r2.url)
- assert r2.meta['request'] is r2
+ assert r2.meta["request"] is r2
def test_non_pickable_object(self):
q = self.queue()
@@ -115,7 +114,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
self.assertIsInstance(exc.__context__, pickle.PicklingError)
else:
self.assertIsInstance(exc.__context__, AttributeError)
- sel = Selector(text='<html><body><p>some text</p></body></html>')
+ sel = Selector(text="<html><body><p>some text</p></body></html>")
try:
q.push(sel)
except ValueError as exc:
@@ -139,33 +138,30 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
class LifoDiskQueueTestMixin:
-
def test_serialize(self):
q = self.queue()
- q.push('a')
+ q.push("a")
q.push(123)
- q.push({'a': 'dict'})
- self.assertEqual(q.pop(), {'a': 'dict'})
+ q.push({"a": "dict"})
+ self.assertEqual(q.pop(), {"a": "dict"})
self.assertEqual(q.pop(), 123)
- self.assertEqual(q.pop(), 'a')
+ self.assertEqual(q.pop(), "a")
test_nonserializable_object = nonserializable_object_test
class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
-
def queue(self):
return _MarshalLifoSerializationDiskQueue(self.qpath)
class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
-
def queue(self):
return _PickleLifoSerializationDiskQueue(self.qpath)
def test_serialize_item(self):
q = self.queue()
- i = TestItem(name='foo')
+ i = TestItem(name="foo")
q.push(i)
i2 = q.pop()
assert isinstance(i2, TestItem)
@@ -178,14 +174,14 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
loader2 = q.pop()
assert isinstance(loader2, TestLoader)
assert loader2.default_item_class is TestItem
- self.assertEqual(loader2.name_out('x'), 'xx')
+ self.assertEqual(loader2.name_out("x"), "xx")
def test_serialize_request_recursive(self):
q = self.queue()
- r = Request('http://www.example.com')
- r.meta['request'] = r
+ r = Request("http://www.example.com")
+ r.meta["request"] = r
q.push(r)
r2 = q.pop()
assert isinstance(r2, Request)
self.assertEqual(r.url, r2.url)
- assert r2.meta['request'] is r2
+ assert r2.meta["request"] is r2
diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py
index 22dafb2d2..b444c32b7 100644
--- a/tests/test_squeues_request.py
+++ b/tests/test_squeues_request.py
@@ -4,16 +4,16 @@ import unittest
import queuelib
-from scrapy.squeues import (
- PickleFifoDiskQueue,
- PickleLifoDiskQueue,
- MarshalFifoDiskQueue,
- MarshalLifoDiskQueue,
- FifoMemoryQueue,
- LifoMemoryQueue,
-)
from scrapy.http import Request
from scrapy.spiders import Spider
+from scrapy.squeues import (
+ FifoMemoryQueue,
+ LifoMemoryQueue,
+ MarshalFifoDiskQueue,
+ MarshalLifoDiskQueue,
+ PickleFifoDiskQueue,
+ PickleLifoDiskQueue,
+)
from scrapy.utils.test import get_crawler
"""
@@ -69,7 +69,10 @@ class RequestQueueTestMixin:
req = Request("http://www.example.com")
q.push(req)
self.assertEqual(len(q), 1)
- with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+ with self.assertRaises(
+ NotImplementedError,
+ msg="The underlying queue class does not implement 'peek'",
+ ):
q.peek()
self.assertEqual(q.pop().url, req.url)
self.assertEqual(len(q), 0)
@@ -117,7 +120,10 @@ class FifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
- with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+ with self.assertRaises(
+ NotImplementedError,
+ msg="The underlying queue class does not implement 'peek'",
+ ):
q.peek()
self.assertEqual(len(q), 3)
self.assertEqual(q.pop().url, req1.url)
@@ -170,7 +176,10 @@ class LifoQueueMixin(RequestQueueTestMixin):
q.push(req1)
q.push(req2)
q.push(req3)
- with self.assertRaises(NotImplementedError, msg="The underlying queue class does not implement 'peek'"):
+ with self.assertRaises(
+ NotImplementedError,
+ msg="The underlying queue class does not implement 'peek'",
+ ):
q.peek()
self.assertEqual(len(q), 3)
self.assertEqual(q.pop().url, req3.url)
@@ -195,12 +204,16 @@ class PickleLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
class MarshalFifoDiskQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
def queue(self):
- return MarshalFifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/fifo")
+ return MarshalFifoDiskQueue.from_crawler(
+ crawler=self.crawler, key="marshal/fifo"
+ )
class MarshalLifoDiskQueueRequestTest(LifoQueueMixin, BaseQueueTestCase):
def queue(self):
- return MarshalLifoDiskQueue.from_crawler(crawler=self.crawler, key="marshal/lifo")
+ return MarshalLifoDiskQueue.from_crawler(
+ crawler=self.crawler, key="marshal/lifo"
+ )
class FifoMemoryQueueRequestTest(FifoQueueMixin, BaseQueueTestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2bbbb9e2c..3d4c7e88e 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,42 +1,41 @@
-from datetime import datetime
import unittest
+from datetime import datetime
from unittest import mock
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector, DummyStatsCollector
+from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.test import get_crawler
class CoreStatsExtensionTest(unittest.TestCase):
-
def setUp(self):
self.crawler = get_crawler(Spider)
- self.spider = self.crawler._create_spider('foo')
+ self.spider = self.crawler._create_spider("foo")
- @mock.patch('scrapy.extensions.corestats.datetime')
+ @mock.patch("scrapy.extensions.corestats.datetime")
def test_core_stats_default_stats_collector(self, mock_datetime):
fixed_datetime = datetime(2019, 12, 1, 11, 38)
- mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
+ mock_datetime.now = mock.Mock(return_value=fixed_datetime)
self.crawler.stats = StatsCollector(self.crawler)
ext = CoreStats.from_crawler(self.crawler)
ext.spider_opened(self.spider)
ext.item_scraped({}, self.spider)
ext.response_received(self.spider)
ext.item_dropped({}, self.spider, ZeroDivisionError())
- ext.spider_closed(self.spider, 'finished')
+ ext.spider_closed(self.spider, "finished")
self.assertEqual(
ext.stats._stats,
{
- 'start_time': fixed_datetime,
- 'finish_time': fixed_datetime,
- 'item_scraped_count': 1,
- 'response_received_count': 1,
- 'item_dropped_count': 1,
- 'item_dropped_reasons_count/ZeroDivisionError': 1,
- 'finish_reason': 'finished',
- 'elapsed_time_seconds': 0.0,
- }
+ "start_time": fixed_datetime,
+ "finish_time": fixed_datetime,
+ "item_scraped_count": 1,
+ "response_received_count": 1,
+ "item_dropped_count": 1,
+ "item_dropped_reasons_count/ZeroDivisionError": 1,
+ "finish_reason": "finished",
+ "elapsed_time_seconds": 0.0,
+ },
)
def test_core_stats_dummy_stats_collector(self):
@@ -46,53 +45,52 @@ class CoreStatsExtensionTest(unittest.TestCase):
ext.item_scraped({}, self.spider)
ext.response_received(self.spider)
ext.item_dropped({}, self.spider, ZeroDivisionError())
- ext.spider_closed(self.spider, 'finished')
+ ext.spider_closed(self.spider, "finished")
self.assertEqual(ext.stats._stats, {})
class StatsCollectorTest(unittest.TestCase):
-
def setUp(self):
self.crawler = get_crawler(Spider)
- self.spider = self.crawler._create_spider('foo')
+ self.spider = self.crawler._create_spider("foo")
def test_collector(self):
stats = StatsCollector(self.crawler)
self.assertEqual(stats.get_stats(), {})
- self.assertEqual(stats.get_value('anything'), None)
- self.assertEqual(stats.get_value('anything', 'default'), 'default')
- stats.set_value('test', 'value')
- self.assertEqual(stats.get_stats(), {'test': 'value'})
- stats.set_value('test2', 23)
- self.assertEqual(stats.get_stats(), {'test': 'value', 'test2': 23})
- self.assertEqual(stats.get_value('test2'), 23)
- stats.inc_value('test2')
- self.assertEqual(stats.get_value('test2'), 24)
- stats.inc_value('test2', 6)
- self.assertEqual(stats.get_value('test2'), 30)
- stats.max_value('test2', 6)
- self.assertEqual(stats.get_value('test2'), 30)
- stats.max_value('test2', 40)
- self.assertEqual(stats.get_value('test2'), 40)
- stats.max_value('test3', 1)
- self.assertEqual(stats.get_value('test3'), 1)
- stats.min_value('test2', 60)
- self.assertEqual(stats.get_value('test2'), 40)
- stats.min_value('test2', 35)
- self.assertEqual(stats.get_value('test2'), 35)
- stats.min_value('test4', 7)
- self.assertEqual(stats.get_value('test4'), 7)
+ self.assertEqual(stats.get_value("anything"), None)
+ self.assertEqual(stats.get_value("anything", "default"), "default")
+ stats.set_value("test", "value")
+ self.assertEqual(stats.get_stats(), {"test": "value"})
+ stats.set_value("test2", 23)
+ self.assertEqual(stats.get_stats(), {"test": "value", "test2": 23})
+ self.assertEqual(stats.get_value("test2"), 23)
+ stats.inc_value("test2")
+ self.assertEqual(stats.get_value("test2"), 24)
+ stats.inc_value("test2", 6)
+ self.assertEqual(stats.get_value("test2"), 30)
+ stats.max_value("test2", 6)
+ self.assertEqual(stats.get_value("test2"), 30)
+ stats.max_value("test2", 40)
+ self.assertEqual(stats.get_value("test2"), 40)
+ stats.max_value("test3", 1)
+ self.assertEqual(stats.get_value("test3"), 1)
+ stats.min_value("test2", 60)
+ self.assertEqual(stats.get_value("test2"), 40)
+ stats.min_value("test2", 35)
+ self.assertEqual(stats.get_value("test2"), 35)
+ stats.min_value("test4", 7)
+ self.assertEqual(stats.get_value("test4"), 7)
def test_dummy_collector(self):
stats = DummyStatsCollector(self.crawler)
self.assertEqual(stats.get_stats(), {})
- self.assertEqual(stats.get_value('anything'), None)
- self.assertEqual(stats.get_value('anything', 'default'), 'default')
- stats.set_value('test', 'value')
- stats.inc_value('v1')
- stats.max_value('v2', 100)
- stats.min_value('v3', 100)
- stats.open_spider('a')
- stats.set_value('test', 'value', spider=self.spider)
+ self.assertEqual(stats.get_value("anything"), None)
+ self.assertEqual(stats.get_value("anything", "default"), "default")
+ stats.set_value("test", "value")
+ stats.inc_value("v1")
+ stats.max_value("v2", 100)
+ stats.min_value("v3", 100)
+ stats.open_spider("a")
+ stats.set_value("test", "value", spider=self.spider)
self.assertEqual(stats.get_stats(), {})
- self.assertEqual(stats.get_stats('a'), {})
+ self.assertEqual(stats.get_stats("a"), {})
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index fdc5df166..d272101b8 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -4,7 +4,6 @@ import scrapy
class ToplevelTestCase(TestCase):
-
def test_version(self):
self.assertIs(type(scrapy.__version__), str)
@@ -12,19 +11,23 @@ class ToplevelTestCase(TestCase):
self.assertIs(type(scrapy.version_info), tuple)
def test_request_shortcut(self):
- from scrapy.http import Request, FormRequest
+ from scrapy.http import FormRequest, Request
+
self.assertIs(scrapy.Request, Request)
self.assertIs(scrapy.FormRequest, FormRequest)
def test_spider_shortcut(self):
from scrapy.spiders import Spider
+
self.assertIs(scrapy.Spider, Spider)
def test_selector_shortcut(self):
from scrapy.selector import Selector
+
self.assertIs(scrapy.Selector, Selector)
def test_item_shortcut(self):
- from scrapy.item import Item, Field
+ from scrapy.item import Field, Item
+
self.assertIs(scrapy.Item, Item)
self.assertIs(scrapy.Field, Field)
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index bea0cf3e5..c695968d7 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,12 +1,11 @@
-from urllib.parse import urlparse
import unittest
+from urllib.parse import urlparse
class UrlparseTestCase(unittest.TestCase):
-
def test_s3_url(self):
- p = urlparse('s3://bucket/key/name?param=value')
- self.assertEqual(p.scheme, 's3')
- self.assertEqual(p.hostname, 'bucket')
- self.assertEqual(p.path, '/key/name')
- self.assertEqual(p.query, 'param=value')
+ p = urlparse("s3://bucket/key/name?param=value")
+ self.assertEqual(p.scheme, "s3")
+ self.assertEqual(p.hostname, "bucket")
+ self.assertEqual(p.path, "/key/name")
+ self.assertEqual(p.query, "param=value")
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 42780ace7..65e352053 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -1,22 +1,37 @@
+import asyncio
import warnings
from unittest import TestCase
from pytest import mark
-from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor
+from scrapy.utils.defer import deferred_f_from_coro_f
+from scrapy.utils.reactor import (
+ install_reactor,
+ is_asyncio_reactor_installed,
+ set_asyncio_event_loop,
+)
-@mark.usefixtures('reactor_pytest')
+@mark.usefixtures("reactor_pytest")
class AsyncioTest(TestCase):
-
def test_is_asyncio_reactor_installed(self):
# the result should depend only on the pytest --reactor argument
- self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio')
+ self.assertEqual(
+ is_asyncio_reactor_installed(), self.reactor_pytest == "asyncio"
+ )
def test_install_asyncio_reactor(self):
from twisted.internet import reactor as original_reactor
+
with warnings.catch_warnings(record=True) as w:
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
self.assertEqual(len(w), 0)
from twisted.internet import reactor
+
assert original_reactor == reactor
+
+ @mark.only_asyncio()
+ @deferred_f_from_coro_f
+ async def test_set_asyncio_event_loop(self):
+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+ assert set_asyncio_event_loop(None) is asyncio.get_running_loop()
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index a92880626..dc3f01d57 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,206 +1,244 @@
import unittest
import warnings
-from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
+import pytest
+
+from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
arglist_to_dict,
build_component_list,
feed_complete_default_values_from_settings,
- feed_process_params_from_cli
+ feed_process_params_from_cli,
)
class BuildComponentListTest(unittest.TestCase):
-
def test_build_dict(self):
- d = {'one': 1, 'two': None, 'three': 8, 'four': 4}
- self.assertEqual(build_component_list(d, convert=lambda x: x),
- ['one', 'four', 'three'])
+ d = {"one": 1, "two": None, "three": 8, "four": 4}
+ self.assertEqual(
+ build_component_list(d, convert=lambda x: x), ["one", "four", "three"]
+ )
def test_backward_compatible_build_dict(self):
- base = {'one': 1, 'two': 2, 'three': 3, 'five': 5, 'six': None}
- custom = {'two': None, 'three': 8, 'four': 4}
- self.assertEqual(build_component_list(base, custom,
- convert=lambda x: x),
- ['one', 'four', 'five', 'three'])
+ base = {"one": 1, "two": 2, "three": 3, "five": 5, "six": None}
+ custom = {"two": None, "three": 8, "four": 4}
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ self.assertEqual(
+ build_component_list(base, custom, convert=lambda x: x),
+ ["one", "four", "five", "three"],
+ )
def test_return_list(self):
- custom = ['a', 'b', 'c']
- self.assertEqual(build_component_list(None, custom,
- convert=lambda x: x),
- custom)
+ custom = ["a", "b", "c"]
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ self.assertEqual(
+ build_component_list(None, custom, convert=lambda x: x), custom
+ )
def test_map_dict(self):
- custom = {'one': 1, 'two': 2, 'three': 3}
- self.assertEqual(build_component_list({}, custom,
- convert=lambda x: x.upper()),
- ['ONE', 'TWO', 'THREE'])
+ custom = {"one": 1, "two": 2, "three": 3}
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ self.assertEqual(
+ build_component_list({}, custom, convert=lambda x: x.upper()),
+ ["ONE", "TWO", "THREE"],
+ )
def test_map_list(self):
- custom = ['a', 'b', 'c']
- self.assertEqual(build_component_list(None, custom,
- lambda x: x.upper()),
- ['A', 'B', 'C'])
+ custom = ["a", "b", "c"]
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ self.assertEqual(
+ build_component_list(None, custom, lambda x: x.upper()), ["A", "B", "C"]
+ )
def test_duplicate_components_in_dict(self):
- duplicate_dict = {'one': 1, 'two': 2, 'ONE': 4}
- self.assertRaises(ValueError, build_component_list, {}, duplicate_dict,
- convert=lambda x: x.lower())
+ duplicate_dict = {"one": 1, "two": 2, "ONE": 4}
+ with self.assertRaises(ValueError):
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ build_component_list({}, duplicate_dict, convert=lambda x: x.lower())
def test_duplicate_components_in_list(self):
- duplicate_list = ['a', 'b', 'a']
+ duplicate_list = ["a", "b", "a"]
with self.assertRaises(ValueError) as cm:
- build_component_list(None, duplicate_list, convert=lambda x: x)
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ build_component_list(None, duplicate_list, convert=lambda x: x)
self.assertIn(str(duplicate_list), str(cm.exception))
def test_duplicate_components_in_basesettings(self):
# Higher priority takes precedence
- duplicate_bs = BaseSettings({'one': 1, 'two': 2}, priority=0)
- duplicate_bs.set('ONE', 4, priority=10)
- self.assertEqual(build_component_list(duplicate_bs,
- convert=lambda x: x.lower()),
- ['two', 'one'])
- duplicate_bs.set('one', duplicate_bs['one'], priority=20)
- self.assertEqual(build_component_list(duplicate_bs,
- convert=lambda x: x.lower()),
- ['one', 'two'])
+ duplicate_bs = BaseSettings({"one": 1, "two": 2}, priority=0)
+ duplicate_bs.set("ONE", 4, priority=10)
+ self.assertEqual(
+ build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+ ["two", "one"],
+ )
+ duplicate_bs.set("one", duplicate_bs["one"], priority=20)
+ self.assertEqual(
+ build_component_list(duplicate_bs, convert=lambda x: x.lower()),
+ ["one", "two"],
+ )
# Same priority raises ValueError
- duplicate_bs.set('ONE', duplicate_bs['ONE'], priority=20)
- self.assertRaises(ValueError, build_component_list, duplicate_bs,
- convert=lambda x: x.lower())
+ duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
+ with self.assertRaises(ValueError):
+ build_component_list(duplicate_bs, convert=lambda x: x.lower())
def test_valid_numbers(self):
# work well with None and numeric values
- d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0}
- self.assertEqual(build_component_list(d, convert=lambda x: x),
- ['d', 'a', 'c'])
- d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222}
- self.assertEqual(build_component_list(d, convert=lambda x: x),
- ['b', 'c', 'a'])
+ d = {"a": 10, "b": None, "c": 15, "d": 5.0}
+ self.assertEqual(build_component_list(d, convert=lambda x: x), ["d", "a", "c"])
+ d = {
+ "a": 33333333333333333333,
+ "b": 11111111111111111111,
+ "c": 22222222222222222222,
+ }
+ self.assertEqual(build_component_list(d, convert=lambda x: x), ["b", "c", "a"])
# raise exception for invalid values
- d = {'one': '5'}
- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
- d = {'one': '1.0'}
- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
- d = {'one': [1, 2, 3]}
- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
- d = {'one': {'a': 'a', 'b': 2}}
- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
- d = {'one': 'lorem ipsum'}
- self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
+ d = {"one": "5"}
+ with self.assertRaises(ValueError):
+ with pytest.warns(ScrapyDeprecationWarning, match="The 'custom' attribute"):
+ build_component_list({}, d, convert=lambda x: x)
class UtilsConfTestCase(unittest.TestCase):
-
def test_arglist_to_dict(self):
self.assertEqual(
- arglist_to_dict(['arg1=val1', 'arg2=val2']),
- {'arg1': 'val1', 'arg2': 'val2'})
+ arglist_to_dict(["arg1=val1", "arg2=val2"]),
+ {"arg1": "val1", "arg2": "val2"},
+ )
class FeedExportConfigTestCase(unittest.TestCase):
-
def test_feed_export_config_invalid_format(self):
settings = Settings()
- self.assertRaises(UsageError, feed_process_params_from_cli, settings, ['items.dat'], 'noformat')
+ self.assertRaises(
+ UsageError,
+ feed_process_params_from_cli,
+ settings,
+ ["items.dat"],
+ "noformat",
+ )
def test_feed_export_config_mismatch(self):
settings = Settings()
self.assertRaises(
UsageError,
- feed_process_params_from_cli, settings, ['items1.dat', 'items2.dat'], 'noformat'
+ feed_process_params_from_cli,
+ settings,
+ ["items1.dat", "items2.dat"],
+ "noformat",
)
def test_feed_export_config_backward_compatible(self):
with warnings.catch_warnings(record=True) as cw:
settings = Settings()
self.assertEqual(
- {'items.dat': {'format': 'csv'}},
- feed_process_params_from_cli(settings, ['items.dat'], 'csv')
+ {"items.dat": {"format": "csv"}},
+ feed_process_params_from_cli(settings, ["items.dat"], "csv"),
)
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
def test_feed_export_config_explicit_formats(self):
settings = Settings()
self.assertEqual(
- {'items_1.dat': {'format': 'json'}, 'items_2.dat': {'format': 'xml'}, 'items_3.dat': {'format': 'csv'}},
- feed_process_params_from_cli(settings, ['items_1.dat:json', 'items_2.dat:xml', 'items_3.dat:csv'])
+ {
+ "items_1.dat": {"format": "json"},
+ "items_2.dat": {"format": "xml"},
+ "items_3.dat": {"format": "csv"},
+ },
+ feed_process_params_from_cli(
+ settings, ["items_1.dat:json", "items_2.dat:xml", "items_3.dat:csv"]
+ ),
)
def test_feed_export_config_implicit_formats(self):
settings = Settings()
self.assertEqual(
- {'items_1.json': {'format': 'json'}, 'items_2.xml': {'format': 'xml'}, 'items_3.csv': {'format': 'csv'}},
- feed_process_params_from_cli(settings, ['items_1.json', 'items_2.xml', 'items_3.csv'])
+ {
+ "items_1.json": {"format": "json"},
+ "items_2.xml": {"format": "xml"},
+ "items_3.csv": {"format": "csv"},
+ },
+ feed_process_params_from_cli(
+ settings, ["items_1.json", "items_2.xml", "items_3.csv"]
+ ),
)
def test_feed_export_config_stdout(self):
settings = Settings()
self.assertEqual(
- {'stdout:': {'format': 'pickle'}},
- feed_process_params_from_cli(settings, ['-:pickle'])
+ {"stdout:": {"format": "pickle"}},
+ feed_process_params_from_cli(settings, ["-:pickle"]),
)
def test_feed_export_config_overwrite(self):
settings = Settings()
self.assertEqual(
- {'output.json': {'format': 'json', 'overwrite': True}},
- feed_process_params_from_cli(settings, [], None, ['output.json'])
+ {"output.json": {"format": "json", "overwrite": True}},
+ feed_process_params_from_cli(settings, [], None, ["output.json"]),
)
def test_output_and_overwrite_output(self):
with self.assertRaises(UsageError):
feed_process_params_from_cli(
Settings(),
- ['output1.json'],
+ ["output1.json"],
None,
- ['output2.json'],
+ ["output2.json"],
)
def test_feed_complete_default_values_from_settings_empty(self):
feed = {}
- settings = Settings({
- "FEED_EXPORT_ENCODING": "custom encoding",
- "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
- "FEED_EXPORT_INDENT": 42,
- "FEED_STORE_EMPTY": True,
- "FEED_URI_PARAMS": (1, 2, 3, 4),
- "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
- })
+ settings = Settings(
+ {
+ "FEED_EXPORT_ENCODING": "custom encoding",
+ "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
+ "FEED_EXPORT_INDENT": 42,
+ "FEED_STORE_EMPTY": True,
+ "FEED_URI_PARAMS": (1, 2, 3, 4),
+ "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+ }
+ )
new_feed = feed_complete_default_values_from_settings(feed, settings)
- self.assertEqual(new_feed, {
- "encoding": "custom encoding",
- "fields": ["f1", "f2", "f3"],
- "indent": 42,
- "store_empty": True,
- "uri_params": (1, 2, 3, 4),
- "batch_item_count": 2,
- "item_export_kwargs": {},
- })
+ self.assertEqual(
+ new_feed,
+ {
+ "encoding": "custom encoding",
+ "fields": ["f1", "f2", "f3"],
+ "indent": 42,
+ "store_empty": True,
+ "uri_params": (1, 2, 3, 4),
+ "batch_item_count": 2,
+ "item_export_kwargs": {},
+ },
+ )
def test_feed_complete_default_values_from_settings_non_empty(self):
feed = {
"encoding": "other encoding",
"fields": None,
}
- settings = Settings({
- "FEED_EXPORT_ENCODING": "custom encoding",
- "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
- "FEED_EXPORT_INDENT": 42,
- "FEED_STORE_EMPTY": True,
- "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
- })
+ settings = Settings(
+ {
+ "FEED_EXPORT_ENCODING": "custom encoding",
+ "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
+ "FEED_EXPORT_INDENT": 42,
+ "FEED_STORE_EMPTY": True,
+ "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+ }
+ )
new_feed = feed_complete_default_values_from_settings(feed, settings)
- self.assertEqual(new_feed, {
- "encoding": "other encoding",
- "fields": None,
- "indent": 42,
- "store_empty": True,
- "uri_params": None,
- "batch_item_count": 2,
- "item_export_kwargs": {},
- })
+ self.assertEqual(
+ new_feed,
+ {
+ "encoding": "other encoding",
+ "fields": None,
+ "indent": 42,
+ "store_empty": True,
+ "uri_params": None,
+ "batch_item_count": 2,
+ "item_export_kwargs": {},
+ },
+ )
if __name__ == "__main__":
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 380c41367..dabd6054d 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -1,14 +1,17 @@
import unittest
from scrapy.utils.console import get_shell_embed_func
+
try:
import bpython
+
bpy = True
del bpython
except ImportError:
bpy = False
try:
import IPython
+
ipy = True
del IPython
except ImportError:
@@ -16,29 +19,25 @@ except ImportError:
class UtilsConsoleTestCase(unittest.TestCase):
-
def test_get_shell_embed_func(self):
-
- shell = get_shell_embed_func(['invalid'])
+ shell = get_shell_embed_func(["invalid"])
self.assertEqual(shell, None)
- shell = get_shell_embed_func(['invalid', 'python'])
+ shell = get_shell_embed_func(["invalid", "python"])
self.assertTrue(callable(shell))
- self.assertEqual(shell.__name__, '_embed_standard_shell')
+ self.assertEqual(shell.__name__, "_embed_standard_shell")
- @unittest.skipIf(not bpy, 'bpython not available in testenv')
+ @unittest.skipIf(not bpy, "bpython not available in testenv")
def test_get_shell_embed_func2(self):
-
- shell = get_shell_embed_func(['bpython'])
+ shell = get_shell_embed_func(["bpython"])
self.assertTrue(callable(shell))
- self.assertEqual(shell.__name__, '_embed_bpython_shell')
+ self.assertEqual(shell.__name__, "_embed_bpython_shell")
- @unittest.skipIf(not ipy, 'IPython not available in testenv')
+ @unittest.skipIf(not ipy, "IPython not available in testenv")
def test_get_shell_embed_func3(self):
-
# default shell should be 'ipython'
shell = get_shell_embed_func()
- self.assertEqual(shell.__name__, '_embed_ipython_shell')
+ self.assertEqual(shell.__name__, "_embed_ipython_shell")
if __name__ == "__main__":
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index f5d684d3f..1816db29b 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -34,10 +34,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
"method": "GET",
"url": "https://api.test.com/",
"headers": [
- (
- "Authorization",
- basic_auth_header("some_username", "some_password")
- )
+ ("Authorization", basic_auth_header("some_username", "some_password"))
],
}
self._test_command(curl_command, expected_result)
@@ -77,11 +74,11 @@ class CurlToRequestKwargsTest(unittest.TestCase):
("Connection", "keep-alive"),
],
"cookies": {
- '_gauges_unique_year': '1',
- '_gauges_unique_hour': '1',
- '_gauges_unique_day': '1',
- '_gauges_unique': '1',
- '_gauges_unique_month': '1'
+ "_gauges_unique_year": "1",
+ "_gauges_unique_hour": "1",
+ "_gauges_unique_day": "1",
+ "_gauges_unique": "1",
+ "_gauges_unique_month": "1",
},
}
self._test_command(curl_command, expected_result)
@@ -107,14 +104,14 @@ class CurlToRequestKwargsTest(unittest.TestCase):
"method": "POST",
"url": "http://httpbin.org/post",
"body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl"
- "e.org&size=small&topping=cheese&topping=onion&delivery=12"
- "%3A15&comments=",
+ "e.org&size=small&topping=cheese&topping=onion&delivery=12"
+ "%3A15&comments=",
"cookies": {
- '_gauges_unique_year': '1',
- '_gauges_unique_hour': '1',
- '_gauges_unique_day': '1',
- '_gauges_unique': '1',
- '_gauges_unique_month': '1'
+ "_gauges_unique_year": "1",
+ "_gauges_unique_hour": "1",
+ "_gauges_unique_day": "1",
+ "_gauges_unique": "1",
+ "_gauges_unique_month": "1",
},
"headers": [
("Origin", "http://httpbin.org"),
@@ -152,16 +149,26 @@ class CurlToRequestKwargsTest(unittest.TestCase):
"body": (
"excerptLength=200&enableDidYouMean=true&sortCriteria=ffirstz3"
"2xnamez32x201740686%20ascending&queryFunctions=%5B%5D&ranking"
- "Functions=%5B%5D")
+ "Functions=%5B%5D"
+ ),
+ }
+ self._test_command(curl_command, expected_result)
+
+ def test_post_data_raw_with_string_prefix(self):
+ curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'"
+ expected_result = {
+ "method": "POST",
+ "url": "https://www.example.org/",
+ "body": '{"$filters":"Filter!"}',
}
self._test_command(curl_command, expected_result)
def test_explicit_get_with_data(self):
- curl_command = 'curl httpbin.org/anything -X GET --data asdf'
+ curl_command = "curl httpbin.org/anything -X GET --data asdf"
expected_result = {
"method": "GET",
"url": "http://httpbin.org/anything",
- "body": "asdf"
+ "body": "asdf",
}
self._test_command(curl_command, expected_result)
@@ -182,16 +189,14 @@ class CurlToRequestKwargsTest(unittest.TestCase):
("Authorization", basic_auth_header("username", "password")),
],
"body": '{"hostname": "agent02.example.com", "agent_config_state"'
- ': "Enabled", "resources": ["Java","Linux"], "environments'
- '": ["Dev"]}',
+ ': "Enabled", "resources": ["Java","Linux"], "environments'
+ '": ["Dev"]}',
}
self._test_command(curl_command, expected_result)
def test_delete(self):
curl_command = 'curl -X "DELETE" https://www.url.com/page'
- expected_result = {
- "method": "DELETE", "url": "https://www.url.com/page"
- }
+ expected_result = {"method": "DELETE", "url": "https://www.url.com/page"}
self._test_command(curl_command, expected_result)
def test_get_silent(self):
@@ -209,8 +214,8 @@ class CurlToRequestKwargsTest(unittest.TestCase):
def test_ignore_unknown_options(self):
# case 1: ignore_unknown_options=True:
with warnings.catch_warnings(): # avoid warning when executing tests
- warnings.simplefilter('ignore')
- curl_command = 'curl --bar --baz http://www.example.com'
+ warnings.simplefilter("ignore")
+ curl_command = "curl --bar --baz http://www.example.com"
expected_result = {"method": "GET", "url": "http://www.example.com"}
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
@@ -219,13 +224,12 @@ class CurlToRequestKwargsTest(unittest.TestCase):
ValueError,
"Unrecognized options:.*--bar.*--baz",
lambda: curl_to_request_kwargs(
- "curl --bar --baz http://www.example.com",
- ignore_unknown_options=False
+ "curl --bar --baz http://www.example.com", ignore_unknown_options=False
),
)
def test_must_start_with_curl_error(self):
self.assertRaises(
ValueError,
- lambda: curl_to_request_kwargs("carl -X POST http://example.org")
+ lambda: curl_to_request_kwargs("carl -X POST http://example.org"),
)
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index e4bccf30e..9e5f88f48 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,28 +1,35 @@
import copy
import unittest
+import warnings
from collections.abc import Mapping, MutableMapping
+from typing import Iterator
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
-from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude
+from scrapy.utils.datatypes import (
+ CaseInsensitiveDict,
+ CaselessDict,
+ LocalCache,
+ LocalWeakReferencedCache,
+ SequenceExclude,
+)
from scrapy.utils.python import garbage_collect
-
-__doctests__ = ['scrapy.utils.datatypes']
+__doctests__ = ["scrapy.utils.datatypes"]
-class CaselessDictTest(unittest.TestCase):
-
+class CaseInsensitiveDictMixin:
def test_init_dict(self):
- seq = {'red': 1, 'black': 3}
- d = CaselessDict(seq)
- self.assertEqual(d['red'], 1)
- self.assertEqual(d['black'], 3)
+ seq = {"red": 1, "black": 3}
+ d = self.dict_class(seq)
+ self.assertEqual(d["red"], 1)
+ self.assertEqual(d["black"], 3)
def test_init_pair_sequence(self):
- seq = (('red', 1), ('black', 3))
- d = CaselessDict(seq)
- self.assertEqual(d['red'], 1)
- self.assertEqual(d['black'], 3)
+ seq = (("red", 1), ("black", 3))
+ d = self.dict_class(seq)
+ self.assertEqual(d["red"], 1)
+ self.assertEqual(d["black"], 3)
def test_init_mapping(self):
class MyMapping(Mapping):
@@ -39,9 +46,9 @@ class CaselessDictTest(unittest.TestCase):
return len(self._d)
seq = MyMapping(red=1, black=3)
- d = CaselessDict(seq)
- self.assertEqual(d['red'], 1)
- self.assertEqual(d['black'], 3)
+ d = self.dict_class(seq)
+ self.assertEqual(d["red"], 1)
+ self.assertEqual(d["black"], 3)
def test_init_mutable_mapping(self):
class MyMutableMapping(MutableMapping):
@@ -64,122 +71,165 @@ class CaselessDictTest(unittest.TestCase):
return len(self._d)
seq = MyMutableMapping(red=1, black=3)
- d = CaselessDict(seq)
- self.assertEqual(d['red'], 1)
- self.assertEqual(d['black'], 3)
+ d = self.dict_class(seq)
+ self.assertEqual(d["red"], 1)
+ self.assertEqual(d["black"], 3)
def test_caseless(self):
- d = CaselessDict()
- d['key_Lower'] = 1
- self.assertEqual(d['KEy_loWer'], 1)
- self.assertEqual(d.get('KEy_loWer'), 1)
+ d = self.dict_class()
+ d["key_Lower"] = 1
+ self.assertEqual(d["KEy_loWer"], 1)
+ self.assertEqual(d.get("KEy_loWer"), 1)
- d['KEY_LOWER'] = 3
- self.assertEqual(d['key_Lower'], 3)
- self.assertEqual(d.get('key_Lower'), 3)
+ d["KEY_LOWER"] = 3
+ self.assertEqual(d["key_Lower"], 3)
+ self.assertEqual(d.get("key_Lower"), 3)
def test_delete(self):
- d = CaselessDict({'key_lower': 1})
- del d['key_LOWER']
- self.assertRaises(KeyError, d.__getitem__, 'key_LOWER')
- self.assertRaises(KeyError, d.__getitem__, 'key_lower')
+ d = self.dict_class({"key_lower": 1})
+ del d["key_LOWER"]
+ self.assertRaises(KeyError, d.__getitem__, "key_LOWER")
+ self.assertRaises(KeyError, d.__getitem__, "key_lower")
def test_getdefault(self):
d = CaselessDict()
- self.assertEqual(d.get('c', 5), 5)
- d['c'] = 10
- self.assertEqual(d.get('c', 5), 10)
+ self.assertEqual(d.get("c", 5), 5)
+ d["c"] = 10
+ self.assertEqual(d.get("c", 5), 10)
def test_setdefault(self):
- d = CaselessDict({'a': 1, 'b': 2})
+ d = CaselessDict({"a": 1, "b": 2})
- r = d.setdefault('A', 5)
+ r = d.setdefault("A", 5)
self.assertEqual(r, 1)
- self.assertEqual(d['A'], 1)
+ self.assertEqual(d["A"], 1)
- r = d.setdefault('c', 5)
+ r = d.setdefault("c", 5)
self.assertEqual(r, 5)
- self.assertEqual(d['C'], 5)
+ self.assertEqual(d["C"], 5)
def test_fromkeys(self):
- keys = ('a', 'b')
+ keys = ("a", "b")
- d = CaselessDict.fromkeys(keys)
- self.assertEqual(d['A'], None)
- self.assertEqual(d['B'], None)
+ d = self.dict_class.fromkeys(keys)
+ self.assertEqual(d["A"], None)
+ self.assertEqual(d["B"], None)
- d = CaselessDict.fromkeys(keys, 1)
- self.assertEqual(d['A'], 1)
- self.assertEqual(d['B'], 1)
+ d = self.dict_class.fromkeys(keys, 1)
+ self.assertEqual(d["A"], 1)
+ self.assertEqual(d["B"], 1)
- instance = CaselessDict()
+ instance = self.dict_class()
d = instance.fromkeys(keys)
- self.assertEqual(d['A'], None)
- self.assertEqual(d['B'], None)
+ self.assertEqual(d["A"], None)
+ self.assertEqual(d["B"], None)
d = instance.fromkeys(keys, 1)
- self.assertEqual(d['A'], 1)
- self.assertEqual(d['B'], 1)
+ self.assertEqual(d["A"], 1)
+ self.assertEqual(d["B"], 1)
def test_contains(self):
- d = CaselessDict()
- d['a'] = 1
- assert 'a' in d
+ d = self.dict_class()
+ d["a"] = 1
+ assert "A" in d
def test_pop(self):
- d = CaselessDict()
- d['a'] = 1
- self.assertEqual(d.pop('A'), 1)
- self.assertRaises(KeyError, d.pop, 'A')
+ d = self.dict_class()
+ d["a"] = 1
+ self.assertEqual(d.pop("A"), 1)
+ self.assertRaises(KeyError, d.pop, "A")
def test_normkey(self):
- class MyDict(CaselessDict):
- def normkey(self, key):
+ class MyDict(self.dict_class):
+ def _normkey(self, key):
return key.title()
+ normkey = _normkey # deprecated CaselessDict class
+
d = MyDict()
- d['key-one'] = 2
- self.assertEqual(list(d.keys()), ['Key-One'])
+ d["key-one"] = 2
+ self.assertEqual(list(d.keys()), ["Key-One"])
def test_normvalue(self):
- class MyDict(CaselessDict):
- def normvalue(self, value):
+ class MyDict(self.dict_class):
+ def _normvalue(self, value):
if value is not None:
return value + 1
- d = MyDict({'key': 1})
- self.assertEqual(d['key'], 2)
- self.assertEqual(d.get('key'), 2)
+ normvalue = _normvalue # deprecated CaselessDict class
+
+ d = MyDict({"key": 1})
+ self.assertEqual(d["key"], 2)
+ self.assertEqual(d.get("key"), 2)
d = MyDict()
- d['key'] = 1
- self.assertEqual(d['key'], 2)
- self.assertEqual(d.get('key'), 2)
+ d["key"] = 1
+ self.assertEqual(d["key"], 2)
+ self.assertEqual(d.get("key"), 2)
d = MyDict()
- d.setdefault('key', 1)
- self.assertEqual(d['key'], 2)
- self.assertEqual(d.get('key'), 2)
+ d.setdefault("key", 1)
+ self.assertEqual(d["key"], 2)
+ self.assertEqual(d.get("key"), 2)
d = MyDict()
- d.update({'key': 1})
- self.assertEqual(d['key'], 2)
- self.assertEqual(d.get('key'), 2)
+ d.update({"key": 1})
+ self.assertEqual(d["key"], 2)
+ self.assertEqual(d.get("key"), 2)
- d = MyDict.fromkeys(('key',), 1)
- self.assertEqual(d['key'], 2)
- self.assertEqual(d.get('key'), 2)
+ d = MyDict.fromkeys(("key",), 1)
+ self.assertEqual(d["key"], 2)
+ self.assertEqual(d.get("key"), 2)
def test_copy(self):
- h1 = CaselessDict({'header1': 'value'})
+ h1 = self.dict_class({"header1": "value"})
h2 = copy.copy(h1)
+ assert isinstance(h2, self.dict_class)
self.assertEqual(h1, h2)
- self.assertEqual(h1.get('header1'), h2.get('header1'))
- assert isinstance(h2, CaselessDict)
+ self.assertEqual(h1.get("header1"), h2.get("header1"))
+ self.assertEqual(h1.get("header1"), h2.get("HEADER1"))
+ h3 = h1.copy()
+ assert isinstance(h3, self.dict_class)
+ self.assertEqual(h1, h3)
+ self.assertEqual(h1.get("header1"), h3.get("header1"))
+ self.assertEqual(h1.get("header1"), h3.get("HEADER1"))
+
+
+class CaseInsensitiveDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+ dict_class = CaseInsensitiveDict
+
+ def test_repr(self):
+ d1 = self.dict_class({"foo": "bar"})
+ self.assertEqual(repr(d1), "<CaseInsensitiveDict: {'foo': 'bar'}>")
+ d2 = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+ self.assertEqual(
+ repr(d2), "<CaseInsensitiveDict: {'AsDf': 'QwErTy', 'FoO': 'bAr'}>"
+ )
+
+ def test_iter(self):
+ d = self.dict_class({"AsDf": "QwErTy", "FoO": "bAr"})
+ iterkeys = iter(d)
+ self.assertIsInstance(iterkeys, Iterator)
+ self.assertEqual(list(iterkeys), ["AsDf", "FoO"])
+
+
+class CaselessDictTest(CaseInsensitiveDictMixin, unittest.TestCase):
+ dict_class = CaselessDict
+
+ def test_deprecation_message(self):
+ with warnings.catch_warnings(record=True) as caught:
+ self.dict_class({"foo": "bar"})
+
+ self.assertEqual(len(caught), 1)
+ self.assertTrue(issubclass(caught[0].category, ScrapyDeprecationWarning))
+ self.assertEqual(
+ "scrapy.utils.datatypes.CaselessDict is deprecated,"
+ " please use scrapy.utils.datatypes.CaseInsensitiveDict instead",
+ str(caught[0].message),
+ )
class SequenceExcludeTest(unittest.TestCase):
-
def test_list(self):
seq = [1, 2, 3]
d = SequenceExclude(seq)
@@ -226,25 +276,24 @@ class SequenceExcludeTest(unittest.TestCase):
# supplied sequence is a set, so checking for list (non)inclusion fails
self.assertRaises(TypeError, (0, 1, 2) in d)
- self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c'])
+ self.assertRaises(TypeError, d.__contains__, ["a", "b", "c"])
for v in [-3, "test", 1.1]:
self.assertNotIn(v, d)
class LocalCacheTest(unittest.TestCase):
-
def test_cache_with_limit(self):
cache = LocalCache(limit=2)
- cache['a'] = 1
- cache['b'] = 2
- cache['c'] = 3
+ cache["a"] = 1
+ cache["b"] = 2
+ cache["c"] = 3
self.assertEqual(len(cache), 2)
- self.assertNotIn('a', cache)
- self.assertIn('b', cache)
- self.assertIn('c', cache)
- self.assertEqual(cache['b'], 2)
- self.assertEqual(cache['c'], 3)
+ self.assertNotIn("a", cache)
+ self.assertIn("b", cache)
+ self.assertIn("c", cache)
+ self.assertEqual(cache["b"], 2)
+ self.assertEqual(cache["c"], 3)
def test_cache_without_limit(self):
maximum = 10**4
@@ -258,12 +307,11 @@ class LocalCacheTest(unittest.TestCase):
class LocalWeakReferencedCacheTest(unittest.TestCase):
-
def test_cache_with_limit(self):
cache = LocalWeakReferencedCache(limit=2)
- r1 = Request('https://example.org')
- r2 = Request('https://example.com')
- r3 = Request('https://example.net')
+ r1 = Request("https://example.org")
+ r2 = Request("https://example.com")
+ r3 = Request("https://example.net")
cache[r1] = 1
cache[r2] = 2
cache[r3] = 3
@@ -299,7 +347,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
cache = LocalWeakReferencedCache()
refs = []
for x in range(max):
- refs.append(Request(f'https://example.org/{x}'))
+ refs.append(Request(f"https://example.org/{x}"))
cache[refs[-1]] = x
self.assertEqual(len(cache), max)
for i, r in enumerate(refs):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 97c1c60d1..bb0ebc2a4 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,11 +1,11 @@
import random
from pytest import mark
-from twisted.trial import unittest
-from twisted.internet import reactor, defer
+from twisted.internet import defer, reactor
from twisted.python.failure import Failure
+from twisted.trial import unittest
-from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
aiter_errback,
deferred_f_from_coro_f,
@@ -68,41 +68,43 @@ def eb1(failure, arg1, arg2):
class DeferUtilsTest(unittest.TestCase):
-
@defer.inlineCallbacks
def test_process_chain(self):
- x = yield process_chain([cb1, cb2, cb3], 'res', 'v1', 'v2')
+ x = yield process_chain([cb1, cb2, cb3], "res", "v1", "v2")
self.assertEqual(x, "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)")
gotexc = False
try:
- yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
+ yield process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")
except TypeError:
gotexc = True
self.assertTrue(gotexc)
@defer.inlineCallbacks
def test_process_chain_both(self):
- x = yield process_chain_both([cb_fail, cb2, cb3], [None, eb1, None], 'res', 'v1', 'v2')
+ x = yield process_chain_both(
+ [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2"
+ )
self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)")
fail = Failure(ZeroDivisionError())
- x = yield process_chain_both([eb1, cb2, cb3], [eb1, None, None], fail, 'v1', 'v2')
+ x = yield process_chain_both(
+ [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2"
+ )
self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)")
@defer.inlineCallbacks
def test_process_parallel(self):
- x = yield process_parallel([cb1, cb2, cb3], 'res', 'v1', 'v2')
- self.assertEqual(x, ['(cb1 res v1 v2)', '(cb2 res v1 v2)', '(cb3 res v1 v2)'])
+ x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
+ self.assertEqual(x, ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"])
def test_process_parallel_failure(self):
- d = process_parallel([cb1, cb_fail, cb3], 'res', 'v1', 'v2')
+ d = process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
self.failUnlessFailure(d, TypeError)
return d
class IterErrbackTest(unittest.TestCase):
-
def test_iter_errback_good(self):
def itergood():
for x in range(10):
@@ -128,7 +130,6 @@ class IterErrbackTest(unittest.TestCase):
class AiterErrbackTest(unittest.TestCase):
-
@deferred_f_from_coro_f
async def test_aiter_errback_good(self):
async def itergood():
@@ -171,7 +172,7 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
class AsyncCooperatorTest(unittest.TestCase):
- """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
+ """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
parallel_async is called with the results of a callback (so an iterable of items, requests and None,
with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable
@@ -182,6 +183,7 @@ class AsyncCooperatorTest(unittest.TestCase):
We also want to simulate the real usage, with arbitrary delays between getting the values
from the iterable. We also want to simulate sync and async results from the callable.
"""
+
CONCURRENT_ITEMS = 50
@staticmethod
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 50c63dfab..eedb6f6af 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,7 +1,7 @@
import inspect
import unittest
-from unittest import mock
import warnings
+from unittest import mock
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
@@ -20,22 +20,23 @@ class NewName(SomeBaseClass):
class WarnWhenSubclassedTest(unittest.TestCase):
-
def _mywarnings(self, w, category=MyWarning):
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):
with warnings.catch_warnings(record=True) as w:
- create_deprecated_class('Deprecated', NewName)
+ create_deprecated_class("Deprecated", NewName)
w = self._mywarnings(w)
self.assertEqual(w, [])
def test_subclassing_warning_message(self):
- Deprecated = create_deprecated_class('Deprecated', NewName,
- warn_category=MyWarning)
+ Deprecated = create_deprecated_class(
+ "Deprecated", NewName, warn_category=MyWarning
+ )
with warnings.catch_warnings(record=True) as w:
+
class UserClass(Deprecated):
pass
@@ -46,17 +47,21 @@ class WarnWhenSubclassedTest(unittest.TestCase):
"tests.test_utils_deprecate.UserClass inherits from "
"deprecated class tests.test_utils_deprecate.Deprecated, "
"please inherit from tests.test_utils_deprecate.NewName."
- " (warning only on first subclass, there may be others)"
+ " (warning only on first subclass, there may be others)",
)
self.assertEqual(w[0].lineno, inspect.getsourcelines(UserClass)[1])
def test_custom_class_paths(self):
- Deprecated = create_deprecated_class('Deprecated', NewName,
- new_class_path='foo.NewClass',
- old_class_path='bar.OldClass',
- warn_category=MyWarning)
+ Deprecated = create_deprecated_class(
+ "Deprecated",
+ NewName,
+ new_class_path="foo.NewClass",
+ old_class_path="bar.OldClass",
+ warn_category=MyWarning,
+ )
with warnings.catch_warnings(record=True) as w:
+
class UserClass(Deprecated):
pass
@@ -64,17 +69,18 @@ class WarnWhenSubclassedTest(unittest.TestCase):
w = self._mywarnings(w)
self.assertEqual(len(w), 2)
- self.assertIn('foo.NewClass', str(w[0].message))
- self.assertIn('bar.OldClass', str(w[0].message))
- self.assertIn('foo.NewClass', str(w[1].message))
- self.assertIn('bar.OldClass', str(w[1].message))
+ self.assertIn("foo.NewClass", str(w[0].message))
+ self.assertIn("bar.OldClass", str(w[0].message))
+ self.assertIn("foo.NewClass", str(w[1].message))
+ self.assertIn("bar.OldClass", str(w[1].message))
- def test_subclassing_warns_only_on_direct_childs(self):
- Deprecated = create_deprecated_class('Deprecated', NewName,
- warn_once=False,
- warn_category=MyWarning)
+ def test_subclassing_warns_only_on_direct_children(self):
+ Deprecated = create_deprecated_class(
+ "Deprecated", NewName, warn_once=False, warn_category=MyWarning
+ )
with warnings.catch_warnings(record=True) as w:
+
class UserClass(Deprecated):
pass
@@ -83,13 +89,15 @@ class WarnWhenSubclassedTest(unittest.TestCase):
w = self._mywarnings(w)
self.assertEqual(len(w), 1)
- self.assertIn('UserClass', str(w[0].message))
+ self.assertIn("UserClass", str(w[0].message))
def test_subclassing_warns_once_by_default(self):
- Deprecated = create_deprecated_class('Deprecated', NewName,
- warn_category=MyWarning)
+ Deprecated = create_deprecated_class(
+ "Deprecated", NewName, warn_category=MyWarning
+ )
with warnings.catch_warnings(record=True) as w:
+
class UserClass(Deprecated):
pass
@@ -101,15 +109,16 @@ class WarnWhenSubclassedTest(unittest.TestCase):
w = self._mywarnings(w)
self.assertEqual(len(w), 1)
- self.assertIn('UserClass', str(w[0].message))
+ self.assertIn("UserClass", str(w[0].message))
def test_warning_on_instance(self):
- Deprecated = create_deprecated_class('Deprecated', NewName,
- warn_category=MyWarning)
+ Deprecated = create_deprecated_class(
+ "Deprecated", NewName, warn_category=MyWarning
+ )
# ignore subclassing warnings
with warnings.catch_warnings():
- warnings.simplefilter('ignore', MyWarning)
+ warnings.simplefilter("ignore", MyWarning)
class UserClass(Deprecated):
pass
@@ -123,13 +132,13 @@ class WarnWhenSubclassedTest(unittest.TestCase):
self.assertEqual(
str(w[0].message),
"tests.test_utils_deprecate.Deprecated is deprecated, "
- "instantiate tests.test_utils_deprecate.NewName instead."
+ "instantiate tests.test_utils_deprecate.NewName instead.",
)
self.assertEqual(w[0].lineno, lineno)
def test_warning_auto_message(self):
with warnings.catch_warnings(record=True) as w:
- Deprecated = create_deprecated_class('Deprecated', NewName)
+ Deprecated = create_deprecated_class("Deprecated", NewName)
class UserClass2(Deprecated):
pass
@@ -140,8 +149,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
def test_issubclass(self):
with warnings.catch_warnings():
- warnings.simplefilter('ignore', ScrapyDeprecationWarning)
- DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
class UpdatedUserClass1(NewName):
pass
@@ -176,8 +185,8 @@ class WarnWhenSubclassedTest(unittest.TestCase):
def test_isinstance(self):
with warnings.catch_warnings():
- warnings.simplefilter('ignore', ScrapyDeprecationWarning)
- DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
class UpdatedUserClass2(NewName):
pass
@@ -210,24 +219,28 @@ class WarnWhenSubclassedTest(unittest.TestCase):
def test_clsdict(self):
with warnings.catch_warnings():
- warnings.simplefilter('ignore', ScrapyDeprecationWarning)
- Deprecated = create_deprecated_class('Deprecated', NewName, {'foo': 'bar'})
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
- self.assertEqual(Deprecated.foo, 'bar')
+ self.assertEqual(Deprecated.foo, "bar")
def test_deprecate_a_class_with_custom_metaclass(self):
- Meta1 = type('Meta1', (type,), {})
- New = Meta1('New', (), {})
- create_deprecated_class('Deprecated', New)
+ Meta1 = type("Meta1", (type,), {})
+ New = Meta1("New", (), {})
+ create_deprecated_class("Deprecated", New)
def test_deprecate_subclass_of_deprecated_class(self):
with warnings.catch_warnings(record=True) as w:
- warnings.simplefilter('always')
- Deprecated = create_deprecated_class('Deprecated', NewName,
- warn_category=MyWarning)
- AlsoDeprecated = create_deprecated_class('AlsoDeprecated', Deprecated,
- new_class_path='foo.Bar',
- warn_category=MyWarning)
+ warnings.simplefilter("always")
+ Deprecated = create_deprecated_class(
+ "Deprecated", NewName, warn_category=MyWarning
+ )
+ AlsoDeprecated = create_deprecated_class(
+ "AlsoDeprecated",
+ Deprecated,
+ new_class_path="foo.Bar",
+ warn_category=MyWarning,
+ )
w = self._mywarnings(w)
self.assertEqual(len(w), 0, str(map(str, w)))
@@ -240,15 +253,15 @@ class WarnWhenSubclassedTest(unittest.TestCase):
w = self._mywarnings(w)
self.assertEqual(len(w), 2)
- self.assertIn('AlsoDeprecated', str(w[0].message))
- self.assertIn('foo.Bar', str(w[0].message))
- self.assertIn('AlsoDeprecated', str(w[1].message))
- self.assertIn('foo.Bar', str(w[1].message))
+ self.assertIn("AlsoDeprecated", str(w[0].message))
+ self.assertIn("foo.Bar", str(w[0].message))
+ self.assertIn("AlsoDeprecated", str(w[1].message))
+ self.assertIn("foo.Bar", str(w[1].message))
def test_inspect_stack(self):
- with mock.patch('inspect.stack', side_effect=IndexError):
+ with mock.patch("inspect.stack", side_effect=IndexError):
with warnings.catch_warnings(record=True) as w:
- DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
+ DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
class SubClass(DeprecatedName):
pass
@@ -256,27 +269,34 @@ class WarnWhenSubclassedTest(unittest.TestCase):
self.assertIn("Error detecting parent module", str(w[0].message))
-@mock.patch('scrapy.utils.deprecate.DEPRECATION_RULES',
- [('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
- ('scrapy.contrib.', 'scrapy.extensions.')])
+@mock.patch(
+ "scrapy.utils.deprecate.DEPRECATION_RULES",
+ [
+ ("scrapy.contrib.pipeline.", "scrapy.pipelines."),
+ ("scrapy.contrib.", "scrapy.extensions."),
+ ],
+)
class UpdateClassPathTest(unittest.TestCase):
-
def test_old_path_gets_fixed(self):
with warnings.catch_warnings(record=True) as w:
- output = update_classpath('scrapy.contrib.debug.Debug')
- self.assertEqual(output, 'scrapy.extensions.debug.Debug')
+ output = update_classpath("scrapy.contrib.debug.Debug")
+ self.assertEqual(output, "scrapy.extensions.debug.Debug")
self.assertEqual(len(w), 1)
self.assertIn("scrapy.contrib.debug.Debug", str(w[0].message))
self.assertIn("scrapy.extensions.debug.Debug", str(w[0].message))
def test_sorted_replacement(self):
with warnings.catch_warnings():
- warnings.simplefilter('ignore', ScrapyDeprecationWarning)
- output = update_classpath('scrapy.contrib.pipeline.Pipeline')
- self.assertEqual(output, 'scrapy.pipelines.Pipeline')
+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+ output = update_classpath("scrapy.contrib.pipeline.Pipeline")
+ self.assertEqual(output, "scrapy.pipelines.Pipeline")
def test_unmatched_path_stays_the_same(self):
with warnings.catch_warnings(record=True) as w:
- output = update_classpath('scrapy.unmatched.Path')
- self.assertEqual(output, 'scrapy.unmatched.Path')
+ output = update_classpath("scrapy.unmatched.Path")
+ self.assertEqual(output, "scrapy.unmatched.Path")
self.assertEqual(len(w), 0)
+
+ def test_returns_nonstring(self):
+ for notastring in [None, True, [1, 2, 3], object()]:
+ self.assertEqual(update_classpath(notastring), notastring)
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 1d1fb50e6..d1bf64828 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -1,12 +1,11 @@
from io import StringIO
-
-from unittest import mock, TestCase
+from unittest import TestCase, mock
from scrapy.utils.display import pformat, pprint
class TestDisplay(TestCase):
- object = {'a': 1}
+ object = {"a": 1}
colorized_strings = {
(
(
@@ -23,7 +22,7 @@ class TestDisplay(TestCase):
}
plain_string = "{'a': 1}"
- @mock.patch('sys.platform', 'linux')
+ @mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat(self, isatty):
isatty.return_value = True
@@ -37,44 +36,47 @@ class TestDisplay(TestCase):
def test_pformat_not_tty(self):
self.assertEqual(pformat(self.object), self.plain_string)
- @mock.patch('sys.platform', 'win32')
- @mock.patch('platform.version')
+ @mock.patch("sys.platform", "win32")
+ @mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_old_windows(self, isatty, version):
isatty.return_value = True
- version.return_value = '10.0.14392'
+ version.return_value = "10.0.14392"
self.assertIn(pformat(self.object), self.colorized_strings)
- @mock.patch('sys.platform', 'win32')
- @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
- @mock.patch('platform.version')
+ @mock.patch("sys.platform", "win32")
+ @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
+ @mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
- def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing):
+ def test_pformat_windows_no_terminal_processing(
+ self, isatty, version, terminal_processing
+ ):
isatty.return_value = True
- version.return_value = '10.0.14393'
+ version.return_value = "10.0.14393"
terminal_processing.return_value = False
self.assertEqual(pformat(self.object), self.plain_string)
- @mock.patch('sys.platform', 'win32')
- @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
- @mock.patch('platform.version')
+ @mock.patch("sys.platform", "win32")
+ @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
+ @mock.patch("platform.version")
@mock.patch("sys.stdout.isatty")
def test_pformat_windows(self, isatty, version, terminal_processing):
isatty.return_value = True
- version.return_value = '10.0.14393'
+ version.return_value = "10.0.14393"
terminal_processing.return_value = True
self.assertIn(pformat(self.object), self.colorized_strings)
- @mock.patch('sys.platform', 'linux')
+ @mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat_no_pygments(self, isatty):
isatty.return_value = True
import builtins
+
real_import = builtins.__import__
def mock_import(name, globals, locals, fromlist, level):
- if 'pygments' in name:
+ if "pygments" in name:
raise ImportError
return real_import(name, globals, locals, fromlist, level)
@@ -83,6 +85,6 @@ class TestDisplay(TestCase):
builtins.__import__ = real_import
def test_pprint(self):
- with mock.patch('sys.stdout', new=StringIO()) as mock_out:
+ with mock.patch("sys.stdout", new=StringIO()) as mock_out:
pprint(self.object)
self.assertEqual(mock_out.getvalue(), "{'a': 1}\n")
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index ca98bff21..7b7a25db8 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -3,18 +3,19 @@ from pathlib import Path
from w3lib.encoding import html_to_unicode
-from scrapy.utils.gz import gunzip, gzip_magic_number
from scrapy.http import Response
+from scrapy.utils.gz import gunzip, gzip_magic_number
from tests import tests_datadir
-
-SAMPLEDIR = Path(tests_datadir, 'compressed')
+SAMPLEDIR = Path(tests_datadir, "compressed")
class GunzipTest(unittest.TestCase):
-
def test_gunzip_basic(self):
- r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes())
+ r1 = Response(
+ "http://www.example.com",
+ body=(SAMPLEDIR / "feed-sample1.xml.gz").read_bytes(),
+ )
self.assertTrue(gzip_magic_number(r1))
r2 = Response("http://www.example.com", body=gunzip(r1.body))
@@ -22,18 +23,23 @@ class GunzipTest(unittest.TestCase):
self.assertEqual(len(r2.body), 9950)
def test_gunzip_truncated(self):
- text = gunzip((SAMPLEDIR / 'truncated-crc-error.gz').read_bytes())
- assert text.endswith(b'</html')
+ text = gunzip((SAMPLEDIR / "truncated-crc-error.gz").read_bytes())
+ assert text.endswith(b"</html")
def test_gunzip_no_gzip_file_raises(self):
- self.assertRaises(IOError, gunzip, (SAMPLEDIR / 'feed-sample1.xml').read_bytes())
+ self.assertRaises(
+ OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes()
+ )
def test_gunzip_truncated_short(self):
- r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'truncated-crc-error-short.gz').read_bytes())
+ r1 = Response(
+ "http://www.example.com",
+ body=(SAMPLEDIR / "truncated-crc-error-short.gz").read_bytes(),
+ )
self.assertTrue(gzip_magic_number(r1))
r2 = Response("http://www.example.com", body=gunzip(r1.body))
- assert r2.body.endswith(b'</html>')
+ assert r2.body.endswith(b"</html>")
self.assertFalse(gzip_magic_number(r2))
def test_is_gzipped_empty(self):
@@ -41,7 +47,11 @@ class GunzipTest(unittest.TestCase):
self.assertFalse(gzip_magic_number(r1))
def test_gunzip_illegal_eof(self):
- text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1]
- expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8")
+ text = html_to_unicode(
+ "charset=cp1252", gunzip((SAMPLEDIR / "unexpected-eof.gz").read_bytes())
+ )[1]
+ expected_text = (SAMPLEDIR / "unexpected-eof-output.txt").read_text(
+ encoding="utf-8"
+ )
self.assertEqual(len(text), len(expected_text))
self.assertEqual(text, expected_text)
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index cf8ad1f23..b824972d5 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -6,7 +6,6 @@ from scrapy.utils.httpobj import urlparse_cached
class HttpobjUtilsTest(unittest.TestCase):
-
def test_urlparse_cached(self):
url = "http://www.example.com/index.html"
request1 = Request(url)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index ba3136b96..3598fa0bb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,13 +1,12 @@
from pytest import mark
from twisted.trial import unittest
-from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
-from scrapy.http import XmlResponse, TextResponse, Response
+from scrapy.http import Response, TextResponse, XmlResponse
+from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
from tests import get_testdata
class XmliterTestCase(unittest.TestCase):
-
xmliter = staticmethod(xmliter)
def test_xmliter(self):
@@ -28,14 +27,18 @@ class XmliterTestCase(unittest.TestCase):
response = XmlResponse(url="http://example.com", body=body)
attrs = []
- for x in self.xmliter(response, 'product'):
- attrs.append((
- x.attrib['id'],
- x.xpath("name/text()").getall(),
- x.xpath("./type/text()").getall()))
+ for x in self.xmliter(response, "product"):
+ attrs.append(
+ (
+ x.attrib["id"],
+ x.xpath("name/text()").getall(),
+ x.xpath("./type/text()").getall(),
+ )
+ )
- self.assertEqual(attrs,
- [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
+ self.assertEqual(
+ attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])]
+ )
def test_xmliter_unusual_node(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -45,8 +48,10 @@ class XmliterTestCase(unittest.TestCase):
</root>
"""
response = XmlResponse(url="http://example.com", body=body)
- nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
- self.assertEqual(nodenames, [['matchme...']])
+ nodenames = [
+ e.xpath("name()").getall() for e in self.xmliter(response, "matchme...")
+ ]
+ self.assertEqual(nodenames, [["matchme..."]])
def test_xmliter_unicode(self):
# example taken from https://github.com/scrapy/scrapy/issues/1665
@@ -88,29 +93,35 @@ class XmliterTestCase(unittest.TestCase):
for r in (
# with bytes
- XmlResponse(url="http://example.com", body=body.encode('utf-8')),
+ XmlResponse(url="http://example.com", body=body.encode("utf-8")),
# Unicode body needs encoding information
- XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
+ XmlResponse(url="http://example.com", body=body, encoding="utf-8"),
):
attrs = []
- for x in self.xmliter(r, 'þingflokkur'):
- attrs.append((x.attrib['id'],
- x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
- x.xpath('./tímabil/fyrstaþing/text()').getall()))
+ for x in self.xmliter(r, "þingflokkur"):
+ attrs.append(
+ (
+ x.attrib["id"],
+ x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
+ x.xpath("./tímabil/fyrstaþing/text()").getall(),
+ )
+ )
- self.assertEqual(attrs,
- [('26', ['-'], ['80']),
- ('21', ['Ab'], ['76']),
- ('27', ['A'], ['27'])])
+ self.assertEqual(
+ attrs,
+ [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])],
+ )
def test_xmliter_text(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
- '<products><product>one</product><product>two</product></products>'
+ "<products><product>one</product><product>two</product></products>"
)
- self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
- [['one'], ['two']])
+ self.assertEqual(
+ [x.xpath("text()").getall() for x in self.xmliter(body, "product")],
+ [["one"], ["two"]],
+ )
def test_xmliter_namespaces(self):
body = b"""
@@ -131,22 +142,25 @@ class XmliterTestCase(unittest.TestCase):
</channel>
</rss>
"""
- response = XmlResponse(url='http://mydummycompany.com', body=body)
- my_iter = self.xmliter(response, 'item')
+ response = XmlResponse(url="http://mydummycompany.com", body=body)
+ my_iter = self.xmliter(response, "item")
node = next(my_iter)
- node.register_namespace('g', 'http://base.google.com/ns/1.0')
- self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
- self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
- self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
+ node.register_namespace("g", "http://base.google.com/ns/1.0")
+ self.assertEqual(node.xpath("title/text()").getall(), ["Item 1"])
+ self.assertEqual(node.xpath("description/text()").getall(), ["This is item 1"])
self.assertEqual(
- node.xpath('g:image_link/text()').getall(),
- ['http://www.mydummycompany.com/images/item1.jpg']
+ node.xpath("link/text()").getall(),
+ ["http://www.mydummycompany.com/items/1"],
)
- self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
- self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
- self.assertEqual(node.xpath('image_link/text()').getall(), [])
- self.assertEqual(node.xpath('id/text()').getall(), [])
- self.assertEqual(node.xpath('price/text()').getall(), [])
+ self.assertEqual(
+ node.xpath("g:image_link/text()").getall(),
+ ["http://www.mydummycompany.com/images/item1.jpg"],
+ )
+ self.assertEqual(node.xpath("g:id/text()").getall(), ["ITEM_1"])
+ self.assertEqual(node.xpath("g:price/text()").getall(), ["400"])
+ self.assertEqual(node.xpath("image_link/text()").getall(), [])
+ self.assertEqual(node.xpath("id/text()").getall(), [])
+ self.assertEqual(node.xpath("price/text()").getall(), [])
def test_xmliter_namespaced_nodename(self):
body = b"""
@@ -167,11 +181,14 @@ class XmliterTestCase(unittest.TestCase):
</channel>
</rss>
"""
- response = XmlResponse(url='http://mydummycompany.com', body=body)
- my_iter = self.xmliter(response, 'g:image_link')
+ response = XmlResponse(url="http://mydummycompany.com", body=body)
+ my_iter = self.xmliter(response, "g:image_link")
node = next(my_iter)
- node.register_namespace('g', 'http://base.google.com/ns/1.0')
- self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+ node.register_namespace("g", "http://base.google.com/ns/1.0")
+ self.assertEqual(
+ node.xpath("text()").extract(),
+ ["http://www.mydummycompany.com/images/item1.jpg"],
+ )
def test_xmliter_namespaced_nodename_missing(self):
body = b"""
@@ -192,45 +209,45 @@ class XmliterTestCase(unittest.TestCase):
</channel>
</rss>
"""
- response = XmlResponse(url='http://mydummycompany.com', body=body)
- my_iter = self.xmliter(response, 'g:link_image')
+ response = XmlResponse(url="http://mydummycompany.com", body=body)
+ my_iter = self.xmliter(response, "g:link_image")
with self.assertRaises(StopIteration):
next(my_iter)
def test_xmliter_exception(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
- '<products><product>one</product><product>two</product></products>'
+ "<products><product>one</product><product>two</product></products>"
)
- iter = self.xmliter(body, 'product')
+ iter = self.xmliter(body, "product")
next(iter)
next(iter)
self.assertRaises(StopIteration, next, iter)
def test_xmliter_objtype_exception(self):
- i = self.xmliter(42, 'product')
+ i = self.xmliter(42, "product")
self.assertRaises(TypeError, next, i)
def test_xmliter_encoding(self):
body = (
b'<?xml version="1.0" encoding="ISO-8859-9"?>\n'
- b'<xml>\n'
- b' <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n'
- b'</xml>\n\n'
+ b"<xml>\n"
+ b" <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n"
+ b"</xml>\n\n"
)
- response = XmlResponse('http://www.example.com', body=body)
+ response = XmlResponse("http://www.example.com", body=body)
self.assertEqual(
- next(self.xmliter(response, 'item')).get(),
- '<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
+ next(self.xmliter(response, "item")).get(),
+ "<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>",
)
class LxmlXmliterTestCase(XmliterTestCase):
xmliter = staticmethod(xmliter_lxml)
- @mark.xfail(reason='known bug of the current implementation')
+ @mark.xfail(reason="known bug of the current implementation")
def test_xmliter_namespaced_nodename(self):
super().test_xmliter_namespaced_nodename()
@@ -252,16 +269,24 @@ class LxmlXmliterTestCase(XmliterTestCase):
</channel>
</rss>
"""
- response = XmlResponse(url='http://mydummycompany.com', body=body)
+ response = XmlResponse(url="http://mydummycompany.com", body=body)
- no_namespace_iter = self.xmliter(response, 'image_link')
+ no_namespace_iter = self.xmliter(response, "image_link")
self.assertEqual(len(list(no_namespace_iter)), 0)
- namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
+ namespace_iter = self.xmliter(
+ response, "image_link", "http://base.google.com/ns/1.0"
+ )
node = next(namespace_iter)
- self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
+ self.assertEqual(
+ node.xpath("text()").getall(),
+ ["http://www.mydummycompany.com/images/item1.jpg"],
+ )
node = next(namespace_iter)
- self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
+ self.assertEqual(
+ node.xpath("text()").getall(),
+ ["http://www.mydummycompany.com/images/item2.jpg"],
+ )
def test_xmliter_namespaces_prefix(self):
body = b"""
@@ -282,125 +307,163 @@ class LxmlXmliterTestCase(XmliterTestCase):
</root>
"""
- response = XmlResponse(url='http://mydummycompany.com', body=body)
- my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
+ response = XmlResponse(url="http://mydummycompany.com", body=body)
+ my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h")
node = next(my_iter)
- self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
- self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
- self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
+ self.assertEqual(len(node.xpath("h:tr/h:td").getall()), 2)
+ self.assertEqual(node.xpath("h:tr/h:td[1]/text()").getall(), ["Apples"])
+ self.assertEqual(node.xpath("h:tr/h:td[2]/text()").getall(), ["Bananas"])
- my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
+ my_iter = self.xmliter(
+ response, "table", "http://www.w3schools.com/furniture", "f"
+ )
node = next(my_iter)
- self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
+ self.assertEqual(node.xpath("f:name/text()").getall(), ["African Coffee Table"])
def test_xmliter_objtype_exception(self):
- i = self.xmliter(42, 'product')
+ i = self.xmliter(42, "product")
self.assertRaises(TypeError, next, i)
class UtilsCsvTestCase(unittest.TestCase):
def test_csviter_defaults(self):
- body = get_testdata('feeds', 'feed-sample3.csv')
+ body = get_testdata("feeds", "feed-sample3.csv")
response = TextResponse(url="http://example.com/", body=body)
csv = csviter(response)
result = [row for row in csv]
- self.assertEqual(result,
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ result,
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
- self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
+ self.assertTrue(all(isinstance(k, str) for k in result_row.keys()))
+ self.assertTrue(all(isinstance(v, str) for v in result_row.values()))
def test_csviter_delimiter(self):
- body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
+ body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")
response = TextResponse(url="http://example.com/", body=body)
- csv = csviter(response, delimiter='\t')
+ csv = csviter(response, delimiter="\t")
- self.assertEqual([row for row in csv],
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ [row for row in csv],
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
def test_csviter_quotechar(self):
- body1 = get_testdata('feeds', 'feed-sample6.csv')
- body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|')
+ body1 = get_testdata("feeds", "feed-sample6.csv")
+ body2 = get_testdata("feeds", "feed-sample6.csv").replace(b",", b"|")
response1 = TextResponse(url="http://example.com/", body=body1)
csv1 = csviter(response1, quotechar="'")
- self.assertEqual([row for row in csv1],
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ [row for row in csv1],
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
response2 = TextResponse(url="http://example.com/", body=body2)
csv2 = csviter(response2, delimiter="|", quotechar="'")
- self.assertEqual([row for row in csv2],
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ [row for row in csv2],
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
def test_csviter_wrong_quotechar(self):
- body = get_testdata('feeds', 'feed-sample6.csv')
+ body = get_testdata("feeds", "feed-sample6.csv")
response = TextResponse(url="http://example.com/", body=body)
csv = csviter(response)
- self.assertEqual([row for row in csv],
- [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
- {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
- {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
- {"'id'": "4", "'name'": "'empty'", "'value'": ""}])
+ self.assertEqual(
+ [row for row in csv],
+ [
+ {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
+ {
+ "'id'": "2",
+ "'name'": "'unicode'",
+ "'value'": "'\xfan\xedc\xf3d\xe9\u203d'",
+ },
+ {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
+ {"'id'": "4", "'name'": "'empty'", "'value'": ""},
+ ],
+ )
def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
- body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
+ body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")
response = Response(url="http://example.com/", body=body)
- csv = csviter(response, delimiter='\t')
+ csv = csviter(response, delimiter="\t")
- self.assertEqual([row for row in csv],
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ [row for row in csv],
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
def test_csviter_headers(self):
- sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
- headers, body = sample[0].split(b','), b'\n'.join(sample[1:])
+ sample = get_testdata("feeds", "feed-sample3.csv").splitlines()
+ headers, body = sample[0].split(b","), b"\n".join(sample[1:])
response = TextResponse(url="http://example.com/", body=body)
- csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
+ csv = csviter(response, headers=[h.decode("utf-8") for h in headers])
- self.assertEqual([row for row in csv],
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ [row for row in csv],
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
def test_csviter_falserow(self):
- body = get_testdata('feeds', 'feed-sample3.csv')
- body = b'\n'.join((body, b'a,b', b'a,b,c,d'))
+ body = get_testdata("feeds", "feed-sample3.csv")
+ body = b"\n".join((body, b"a,b", b"a,b,c,d"))
response = TextResponse(url="http://example.com/", body=body)
csv = csviter(response)
- self.assertEqual([row for row in csv],
- [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
- {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
- {'id': '3', 'name': 'multi', 'value': "foo\nbar"},
- {'id': '4', 'name': 'empty', 'value': ''}])
+ self.assertEqual(
+ [row for row in csv],
+ [
+ {"id": "1", "name": "alpha", "value": "foobar"},
+ {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"},
+ {"id": "3", "name": "multi", "value": "foo\nbar"},
+ {"id": "4", "name": "empty", "value": ""},
+ ],
+ )
def test_csviter_exception(self):
- body = get_testdata('feeds', 'feed-sample3.csv')
+ body = get_testdata("feeds", "feed-sample3.csv")
response = TextResponse(url="http://example.com/", body=body)
iter = csviter(response)
@@ -412,35 +475,41 @@ class UtilsCsvTestCase(unittest.TestCase):
self.assertRaises(StopIteration, next, iter)
def test_csviter_encoding(self):
- body1 = get_testdata('feeds', 'feed-sample4.csv')
- body2 = get_testdata('feeds', 'feed-sample5.csv')
+ body1 = get_testdata("feeds", "feed-sample4.csv")
+ body2 = get_testdata("feeds", "feed-sample5.csv")
- response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
+ response = TextResponse(
+ url="http://example.com/", body=body1, encoding="latin1"
+ )
csv = csviter(response)
self.assertEqual(
list(csv),
[
- {'id': '1', 'name': 'latin1', 'value': 'test'},
- {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
- ]
+ {"id": "1", "name": "latin1", "value": "test"},
+ {"id": "2", "name": "something", "value": "\xf1\xe1\xe9\xf3"},
+ ],
)
- response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
+ response = TextResponse(url="http://example.com/", body=body2, encoding="cp852")
csv = csviter(response)
self.assertEqual(
list(csv),
[
- {'id': '1', 'name': 'cp852', 'value': 'test'},
- {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
- ]
+ {"id": "1", "name": "cp852", "value": "test"},
+ {
+ "id": "2",
+ "name": "something",
+ "value": "\u255a\u2569\u2569\u2569\u2550\u2550\u2557",
+ },
+ ],
)
class TestHelper(unittest.TestCase):
- bbody = b'utf8-body'
- ubody = bbody.decode('utf8')
- txtresponse = TextResponse(url='http://example.org/', body=bbody, encoding='utf-8')
- response = Response(url='http://example.org/', body=bbody)
+ bbody = b"utf8-body"
+ ubody = bbody.decode("utf8")
+ txtresponse = TextResponse(url="http://example.org/", body=bbody, encoding="utf-8")
+ response = Response(url="http://example.org/", body=bbody)
def test_body_or_str(self):
for obj in (self.bbody, self.ubody, self.txtresponse, self.response):
@@ -454,10 +523,7 @@ class TestHelper(unittest.TestCase):
self.assertTrue(type(r1) is not type(r3))
def _assert_type_and_value(self, a, b, obj):
- self.assertTrue(type(a) is type(b),
- f'Got {type(a)}, expected {type(b)} for { obj!r}')
+ self.assertTrue(
+ type(a) is type(b), f"Got {type(a)}, expected {type(b)} for { obj!r}"
+ )
self.assertEqual(a, b)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 535f56691..eae744df5 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,18 +1,21 @@
-import sys
import logging
+import sys
import unittest
from testfixtures import LogCapture
from twisted.python.failure import Failure
-from scrapy.utils.log import (failure_to_exc_info, TopLevelFormatter,
- LogCounterHandler, StreamLogger)
-from scrapy.utils.test import get_crawler
from scrapy.extensions import telnet
+from scrapy.utils.log import (
+ LogCounterHandler,
+ StreamLogger,
+ TopLevelFormatter,
+ failure_to_exc_info,
+)
+from scrapy.utils.test import get_crawler
class FailureToExcInfoTest(unittest.TestCase):
-
def test_failure(self):
try:
0 / 0
@@ -23,48 +26,46 @@ class FailureToExcInfoTest(unittest.TestCase):
self.assertTupleEqual(exc_info, failure_to_exc_info(failure))
def test_non_failure(self):
- self.assertIsNone(failure_to_exc_info('test'))
+ self.assertIsNone(failure_to_exc_info("test"))
class TopLevelFormatterTest(unittest.TestCase):
-
def setUp(self):
self.handler = LogCapture()
- self.handler.addFilter(TopLevelFormatter(['test']))
+ self.handler.addFilter(TopLevelFormatter(["test"]))
def test_top_level_logger(self):
- logger = logging.getLogger('test')
+ logger = logging.getLogger("test")
with self.handler as log:
- logger.warning('test log msg')
- log.check(('test', 'WARNING', 'test log msg'))
+ logger.warning("test log msg")
+ log.check(("test", "WARNING", "test log msg"))
def test_children_logger(self):
- logger = logging.getLogger('test.test1')
+ logger = logging.getLogger("test.test1")
with self.handler as log:
- logger.warning('test log msg')
- log.check(('test', 'WARNING', 'test log msg'))
+ logger.warning("test log msg")
+ log.check(("test", "WARNING", "test log msg"))
def test_overlapping_name_logger(self):
- logger = logging.getLogger('test2')
+ logger = logging.getLogger("test2")
with self.handler as log:
- logger.warning('test log msg')
- log.check(('test2', 'WARNING', 'test log msg'))
+ logger.warning("test log msg")
+ log.check(("test2", "WARNING", "test log msg"))
def test_different_name_logger(self):
- logger = logging.getLogger('different')
+ logger = logging.getLogger("different")
with self.handler as log:
- logger.warning('test log msg')
- log.check(('different', 'WARNING', 'test log msg'))
+ logger.warning("test log msg")
+ log.check(("different", "WARNING", "test log msg"))
class LogCounterHandlerTest(unittest.TestCase):
-
def setUp(self):
- settings = {'LOG_LEVEL': 'WARNING'}
+ settings = {"LOG_LEVEL": "WARNING"}
if not telnet.TWISTED_CONCH_AVAILABLE:
# disable it to avoid the extra warning
- settings['TELNETCONSOLE_ENABLED'] = False
- self.logger = logging.getLogger('test')
+ settings["TELNETCONSOLE_ENABLED"] = False
+ self.logger = logging.getLogger("test")
self.logger.setLevel(logging.NOTSET)
self.logger.propagate = False
self.crawler = get_crawler(settings_dict=settings)
@@ -76,26 +77,25 @@ class LogCounterHandlerTest(unittest.TestCase):
self.logger.removeHandler(self.handler)
def test_init(self):
- self.assertIsNone(self.crawler.stats.get_value('log_count/DEBUG'))
- self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))
- self.assertIsNone(self.crawler.stats.get_value('log_count/WARNING'))
- self.assertIsNone(self.crawler.stats.get_value('log_count/ERROR'))
- self.assertIsNone(self.crawler.stats.get_value('log_count/CRITICAL'))
+ self.assertIsNone(self.crawler.stats.get_value("log_count/DEBUG"))
+ self.assertIsNone(self.crawler.stats.get_value("log_count/INFO"))
+ self.assertIsNone(self.crawler.stats.get_value("log_count/WARNING"))
+ self.assertIsNone(self.crawler.stats.get_value("log_count/ERROR"))
+ self.assertIsNone(self.crawler.stats.get_value("log_count/CRITICAL"))
def test_accepted_level(self):
- self.logger.error('test log msg')
- self.assertEqual(self.crawler.stats.get_value('log_count/ERROR'), 1)
+ self.logger.error("test log msg")
+ self.assertEqual(self.crawler.stats.get_value("log_count/ERROR"), 1)
def test_filtered_out_level(self):
- self.logger.debug('test log msg')
- self.assertIsNone(self.crawler.stats.get_value('log_count/INFO'))
+ self.logger.debug("test log msg")
+ self.assertIsNone(self.crawler.stats.get_value("log_count/INFO"))
class StreamLoggerTest(unittest.TestCase):
-
def setUp(self):
self.stdout = sys.stdout
- logger = logging.getLogger('test')
+ logger = logging.getLogger("test")
logger.setLevel(logging.WARNING)
sys.stdout = StreamLogger(logger, logging.ERROR)
@@ -104,5 +104,5 @@ class StreamLoggerTest(unittest.TestCase):
def test_redirect(self):
with LogCapture() as log:
- print('test log msg')
- log.check(('test', 'ERROR', 'test log msg'))
+ print("test log msg")
+ log.check(("test", "ERROR", "test log msg"))
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index dc5b9e123..ee3314d8e 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,122 +1,127 @@
-import sys
import os
+import sys
import unittest
from pathlib import Path
from unittest import mock
-from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, rel_has_nofollow, set_environ, walk_modules
+from scrapy.item import Field, Item
+from scrapy.utils.misc import (
+ arg_to_iter,
+ build_from_crawler,
+ build_from_settings,
+ create_instance,
+ load_object,
+ rel_has_nofollow,
+ set_environ,
+ walk_modules,
+)
-
-__doctests__ = ['scrapy.utils.misc']
+__doctests__ = ["scrapy.utils.misc"]
class UtilsMiscTestCase(unittest.TestCase):
-
def test_load_object_class(self):
obj = load_object(Field)
self.assertIs(obj, Field)
- obj = load_object('scrapy.item.Field')
+ obj = load_object("scrapy.item.Field")
self.assertIs(obj, Field)
def test_load_object_function(self):
obj = load_object(load_object)
self.assertIs(obj, load_object)
- obj = load_object('scrapy.utils.misc.load_object')
+ obj = load_object("scrapy.utils.misc.load_object")
self.assertIs(obj, load_object)
def test_load_object_exceptions(self):
- self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
- self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
+ self.assertRaises(ImportError, load_object, "nomodule999.mod.function")
+ self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999")
self.assertRaises(TypeError, load_object, {})
def test_walk_modules(self):
- mods = walk_modules('tests.test_utils_misc.test_walk_modules')
+ mods = walk_modules("tests.test_utils_misc.test_walk_modules")
expected = [
- 'tests.test_utils_misc.test_walk_modules',
- 'tests.test_utils_misc.test_walk_modules.mod',
- 'tests.test_utils_misc.test_walk_modules.mod.mod0',
- 'tests.test_utils_misc.test_walk_modules.mod1',
+ "tests.test_utils_misc.test_walk_modules",
+ "tests.test_utils_misc.test_walk_modules.mod",
+ "tests.test_utils_misc.test_walk_modules.mod.mod0",
+ "tests.test_utils_misc.test_walk_modules.mod1",
]
self.assertEqual({m.__name__ for m in mods}, set(expected))
- mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod')
+ mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod")
expected = [
- 'tests.test_utils_misc.test_walk_modules.mod',
- 'tests.test_utils_misc.test_walk_modules.mod.mod0',
+ "tests.test_utils_misc.test_walk_modules.mod",
+ "tests.test_utils_misc.test_walk_modules.mod.mod0",
]
self.assertEqual({m.__name__ for m in mods}, set(expected))
- mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1')
+ mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1")
expected = [
- 'tests.test_utils_misc.test_walk_modules.mod1',
+ "tests.test_utils_misc.test_walk_modules.mod1",
]
self.assertEqual({m.__name__ for m in mods}, set(expected))
- self.assertRaises(ImportError, walk_modules, 'nomodule999')
+ self.assertRaises(ImportError, walk_modules, "nomodule999")
def test_walk_modules_egg(self):
- egg = str(Path(__file__).parent / 'test.egg')
+ egg = str(Path(__file__).parent / "test.egg")
sys.path.append(egg)
try:
- mods = walk_modules('testegg')
+ mods = walk_modules("testegg")
expected = [
- 'testegg.spiders',
- 'testegg.spiders.a',
- 'testegg.spiders.b',
- 'testegg'
+ "testegg.spiders",
+ "testegg.spiders.a",
+ "testegg.spiders.b",
+ "testegg",
]
self.assertEqual({m.__name__ for m in mods}, set(expected))
finally:
sys.path.remove(egg)
def test_arg_to_iter(self):
-
class TestItem(Item):
name = Field()
- assert hasattr(arg_to_iter(None), '__iter__')
- assert hasattr(arg_to_iter(100), '__iter__')
- assert hasattr(arg_to_iter('lala'), '__iter__')
- assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
- assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__')
+ assert hasattr(arg_to_iter(None), "__iter__")
+ assert hasattr(arg_to_iter(100), "__iter__")
+ assert hasattr(arg_to_iter("lala"), "__iter__")
+ assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
+ assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
self.assertEqual(list(arg_to_iter(None)), [])
- self.assertEqual(list(arg_to_iter('lala')), ['lala'])
+ self.assertEqual(list(arg_to_iter("lala")), ["lala"])
self.assertEqual(list(arg_to_iter(100)), [100])
- self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c'])
+ self.assertEqual(list(arg_to_iter(c for c in "abc")), ["a", "b", "c"])
self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
- self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
- self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
+ self.assertEqual(list(arg_to_iter({"a": 1})), [{"a": 1}])
+ self.assertEqual(
+ list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]
+ )
def test_create_instance(self):
settings = mock.MagicMock()
- crawler = mock.MagicMock(spec_set=['settings'])
- args = (True, 100.)
- kwargs = {'key': 'val'}
+ crawler = mock.MagicMock(spec_set=["settings"])
+ args = (True, 100.0)
+ kwargs = {"key": "val"}
def _test_with_settings(mock, settings):
create_instance(mock, settings, None, *args, **kwargs)
- if hasattr(mock, 'from_crawler'):
+ if hasattr(mock, "from_crawler"):
self.assertEqual(mock.from_crawler.call_count, 0)
- if hasattr(mock, 'from_settings'):
- mock.from_settings.assert_called_once_with(settings, *args,
- **kwargs)
+ if hasattr(mock, "from_settings"):
+ mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
self.assertEqual(mock.call_count, 0)
else:
mock.assert_called_once_with(*args, **kwargs)
def _test_with_crawler(mock, settings, crawler):
create_instance(mock, settings, crawler, *args, **kwargs)
- if hasattr(mock, 'from_crawler'):
- mock.from_crawler.assert_called_once_with(crawler, *args,
- **kwargs)
- if hasattr(mock, 'from_settings'):
+ if hasattr(mock, "from_crawler"):
+ mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
+ if hasattr(mock, "from_settings"):
self.assertEqual(mock.from_settings.call_count, 0)
self.assertEqual(mock.call_count, 0)
- elif hasattr(mock, 'from_settings'):
- mock.from_settings.assert_called_once_with(settings, *args,
- **kwargs)
+ elif hasattr(mock, "from_settings"):
+ mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
self.assertEqual(mock.call_count, 0)
else:
mock.assert_called_once_with(*args, **kwargs)
@@ -127,10 +132,10 @@ class UtilsMiscTestCase(unittest.TestCase):
# 3. with from_crawler() constructor
# 4. with from_settings() and from_crawler() constructor
spec_sets = (
- ['__qualname__'],
- ['__qualname__', 'from_settings'],
- ['__qualname__', 'from_crawler'],
- ['__qualname__', 'from_settings', 'from_crawler'],
+ ["__qualname__"],
+ ["__qualname__", "from_settings"],
+ ["__qualname__", "from_crawler"],
+ ["__qualname__", "from_settings", "from_crawler"],
)
for specs in spec_sets:
m = mock.MagicMock(spec_set=specs)
@@ -139,10 +144,9 @@ class UtilsMiscTestCase(unittest.TestCase):
_test_with_crawler(m, settings, crawler)
# Check adoption of crawler settings
- m = mock.MagicMock(spec_set=['__qualname__', 'from_settings'])
+ m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
create_instance(m, None, crawler, *args, **kwargs)
- m.from_settings.assert_called_once_with(crawler.settings, *args,
- **kwargs)
+ m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs)
with self.assertRaises(ValueError):
create_instance(m, None, None)
@@ -151,26 +155,98 @@ class UtilsMiscTestCase(unittest.TestCase):
with self.assertRaises(TypeError):
create_instance(m, settings, None)
- def test_set_environ(self):
- assert os.environ.get('some_test_environ') is None
- with set_environ(some_test_environ='test_value'):
- assert os.environ.get('some_test_environ') == 'test_value'
- assert os.environ.get('some_test_environ') is None
+ def test_build_from_crawler(self):
+ settings = mock.MagicMock()
+ crawler = mock.MagicMock(spec_set=["settings"])
+ args = (True, 100.0)
+ kwargs = {"key": "val"}
- os.environ['some_test_environ'] = 'test'
- assert os.environ.get('some_test_environ') == 'test'
- with set_environ(some_test_environ='test_value'):
- assert os.environ.get('some_test_environ') == 'test_value'
- assert os.environ.get('some_test_environ') == 'test'
+ def _test_with_crawler(mock, settings, crawler):
+ build_from_crawler(mock, crawler, *args, **kwargs)
+ if hasattr(mock, "from_crawler"):
+ mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
+ if hasattr(mock, "from_settings"):
+ self.assertEqual(mock.from_settings.call_count, 0)
+ self.assertEqual(mock.call_count, 0)
+ elif hasattr(mock, "from_settings"):
+ mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
+ self.assertEqual(mock.call_count, 0)
+ else:
+ mock.assert_called_once_with(*args, **kwargs)
+
+ # Check usage of correct constructor using three mocks:
+ # 1. with no alternative constructors
+ # 2. with from_crawler() constructor
+ # 3. with from_settings() and from_crawler() constructor
+ spec_sets = (
+ ["__qualname__"],
+ ["__qualname__", "from_crawler"],
+ ["__qualname__", "from_settings", "from_crawler"],
+ )
+ for specs in spec_sets:
+ m = mock.MagicMock(spec_set=specs)
+ _test_with_crawler(m, settings, crawler)
+ m.reset_mock()
+
+ # Check adoption of crawler
+ m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
+ m.from_crawler.return_value = None
+ with self.assertRaises(TypeError):
+ build_from_crawler(m, crawler, *args, **kwargs)
+
+ def test_build_from_settings(self):
+ settings = mock.MagicMock()
+ args = (True, 100.0)
+ kwargs = {"key": "val"}
+
+ def _test_with_settings(mock, settings):
+ build_from_settings(mock, settings, *args, **kwargs)
+ if hasattr(mock, "from_settings"):
+ mock.from_settings.assert_called_once_with(settings, *args, **kwargs)
+ self.assertEqual(mock.call_count, 0)
+ else:
+ mock.assert_called_once_with(*args, **kwargs)
+
+ # Check usage of correct constructor using three mocks:
+ # 1. with no alternative constructors
+ # 2. with from_settings() constructor
+ # 3. with from_settings() and from_crawler() constructor
+ spec_sets = (
+ ["__qualname__"],
+ ["__qualname__", "from_settings"],
+ ["__qualname__", "from_settings", "from_crawler"],
+ )
+ for specs in spec_sets:
+ m = mock.MagicMock(spec_set=specs)
+ _test_with_settings(m, settings)
+ m.reset_mock()
+
+ # Check adoption of crawler settings
+ m = mock.MagicMock(spec_set=["__qualname__", "from_settings"])
+ m.from_settings.return_value = None
+ with self.assertRaises(TypeError):
+ build_from_settings(m, settings, *args, **kwargs)
+
+ def test_set_environ(self):
+ assert os.environ.get("some_test_environ") is None
+ with set_environ(some_test_environ="test_value"):
+ assert os.environ.get("some_test_environ") == "test_value"
+ assert os.environ.get("some_test_environ") is None
+
+ os.environ["some_test_environ"] = "test"
+ assert os.environ.get("some_test_environ") == "test"
+ with set_environ(some_test_environ="test_value"):
+ assert os.environ.get("some_test_environ") == "test_value"
+ assert os.environ.get("some_test_environ") == "test"
def test_rel_has_nofollow(self):
- assert rel_has_nofollow('ugc nofollow') is True
- assert rel_has_nofollow('ugc,nofollow') is True
- assert rel_has_nofollow('ugc') is False
- assert rel_has_nofollow('nofollow') is True
- assert rel_has_nofollow('nofollowfoo') is False
- assert rel_has_nofollow('foonofollow') is False
- assert rel_has_nofollow('ugc, , nofollow') is True
+ assert rel_has_nofollow("ugc nofollow") is True
+ assert rel_has_nofollow("ugc,nofollow") is True
+ assert rel_has_nofollow("ugc") is False
+ assert rel_has_nofollow("nofollow") is True
+ assert rel_has_nofollow("nofollowfoo") is False
+ assert rel_has_nofollow("foonofollow") is False
+ assert rel_has_nofollow("ugc, , nofollow") is True
if __name__ == "__main__":
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 562f72fee..484757035 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -3,7 +3,10 @@ import warnings
from functools import partial
from unittest import mock
-from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value
+from scrapy.utils.misc import (
+ is_generator_with_return_value,
+ warn_on_generator_with_return_value,
+)
def _indentation_error(*args, **kwargs):
@@ -12,7 +15,7 @@ def _indentation_error(*args, **kwargs):
def top_level_return_something():
"""
-docstring
+ docstring
"""
url = """
https://example.org
@@ -23,7 +26,7 @@ https://example.org
def top_level_return_none():
"""
-docstring
+ docstring
"""
url = """
https://example.org
@@ -39,7 +42,6 @@ def generator_that_returns_stuff():
class UtilsMiscPy3TestCase(unittest.TestCase):
-
def test_generators_return_something(self):
def f1():
yield 1
@@ -60,7 +62,7 @@ class UtilsMiscPy3TestCase(unittest.TestCase):
def i1():
"""
-docstring
+ docstring
"""
url = """
https://example.org
@@ -77,7 +79,10 @@ https://example.org
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, top_level_return_something)
self.assertEqual(len(w), 1)
- self.assertIn('The "NoneType.top_level_return_something" method is a generator', str(w[0].message))
+ self.assertIn(
+ 'The "NoneType.top_level_return_something" method is a generator',
+ str(w[0].message),
+ )
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, f1)
self.assertEqual(len(w), 1)
@@ -121,7 +126,7 @@ https://example.org
def k2():
"""
-docstring
+ docstring
"""
url = """
https://example.org
@@ -170,6 +175,7 @@ https://example.org
def decorator(func):
def inner_func():
func()
+
return inner_func
@decorator
@@ -203,7 +209,7 @@ https://example.org
@decorator
def k3():
"""
-docstring
+ docstring
"""
url = """
https://example.org
@@ -249,12 +255,14 @@ https://example.org
warn_on_generator_with_return_value(None, l3)
self.assertEqual(len(w), 0)
- @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
+ @mock.patch(
+ "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
+ )
def test_indentation_error(self):
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(None, top_level_return_none)
self.assertEqual(len(w), 1)
- self.assertIn('Unable to determine', str(w[0].message))
+ self.assertIn("Unable to determine", str(w[0].message))
def test_partial(self):
def cb(arg1, arg2):
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 867e12103..90bd350a5 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,8 +1,8 @@
-import unittest
-import os
-import tempfile
-import shutil
import contextlib
+import os
+import shutil
+import tempfile
+import unittest
import warnings
from pathlib import Path
@@ -16,7 +16,7 @@ def inside_a_project():
try:
os.chdir(project_dir)
- Path('scrapy.cfg').touch()
+ Path("scrapy.cfg").touch()
yield project_dir
finally:
@@ -26,21 +26,15 @@ def inside_a_project():
class ProjectUtilsTest(unittest.TestCase):
def test_data_path_outside_project(self):
- self.assertEqual(
- str(Path('.scrapy', 'somepath')),
- data_path('somepath')
- )
- abspath = str(Path(os.path.sep, 'absolute', 'path'))
+ self.assertEqual(str(Path(".scrapy", "somepath")), data_path("somepath"))
+ abspath = str(Path(os.path.sep, "absolute", "path"))
self.assertEqual(abspath, data_path(abspath))
def test_data_path_inside_project(self):
with inside_a_project() as proj_path:
- expected = Path(proj_path, '.scrapy', 'somepath')
- self.assertEqual(
- expected.resolve(),
- Path(data_path('somepath')).resolve()
- )
- abspath = str(Path(os.path.sep, 'absolute', 'path').resolve())
+ expected = Path(proj_path, ".scrapy", "somepath")
+ self.assertEqual(expected.resolve(), Path(data_path("somepath")).resolve())
+ abspath = str(Path(os.path.sep, "absolute", "path").resolve())
self.assertEqual(abspath, data_path(abspath))
@@ -59,22 +53,21 @@ def set_env(**update):
class GetProjectSettingsTestCase(unittest.TestCase):
-
def test_valid_envvar(self):
- value = 'tests.test_cmdline.settings'
+ value = "tests.test_cmdline.settings"
envvars = {
- 'SCRAPY_SETTINGS_MODULE': value,
+ "SCRAPY_SETTINGS_MODULE": value,
}
with warnings.catch_warnings():
warnings.simplefilter("error")
with set_env(**envvars):
settings = get_project_settings()
- assert settings.get('SETTINGS_MODULE') == value
+ assert settings.get("SETTINGS_MODULE") == value
def test_invalid_envvar(self):
envvars = {
- 'SCRAPY_FOO': 'bar',
+ "SCRAPY_FOO": "bar",
}
with set_env(**envvars):
settings = get_project_settings()
@@ -82,12 +75,12 @@ class GetProjectSettingsTestCase(unittest.TestCase):
assert settings.get("SCRAPY_FOO") is None
def test_valid_and_invalid_envvars(self):
- value = 'tests.test_cmdline.settings'
+ value = "tests.test_cmdline.settings"
envvars = {
- 'SCRAPY_FOO': 'bar',
- 'SCRAPY_SETTINGS_MODULE': value,
+ "SCRAPY_FOO": "bar",
+ "SCRAPY_SETTINGS_MODULE": value,
}
with set_env(**envvars):
settings = get_project_settings()
- assert settings.get('SETTINGS_MODULE') == value
- assert settings.get('SCRAPY_FOO') is None
+ assert settings.get("SETTINGS_MODULE") == value
+ assert settings.get("SCRAPY_FOO") is None
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 403e4f8fe..80d2e8da1 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -5,14 +5,20 @@ import platform
from twisted.trial import unittest
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback
+from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f
from scrapy.utils.python import (
- memoizemethod_noargs, binary_is_text, equal_attributes,
- get_func_args, to_bytes, to_unicode,
- without_none_values, MutableChain, MutableAsyncChain)
+ MutableAsyncChain,
+ MutableChain,
+ binary_is_text,
+ equal_attributes,
+ get_func_args,
+ memoizemethod_noargs,
+ to_bytes,
+ to_unicode,
+ without_none_values,
+)
-
-__doctests__ = ['scrapy.utils.python']
+__doctests__ = ["scrapy.utils.python"]
class MutableChainTest(unittest.TestCase):
@@ -79,48 +85,41 @@ class MutableAsyncChainTest(unittest.TestCase):
class ToUnicodeTest(unittest.TestCase):
def test_converting_an_utf8_encoded_string_to_unicode(self):
- self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
+ self.assertEqual(to_unicode(b"lel\xc3\xb1e"), "lel\xf1e")
def test_converting_a_latin_1_encoded_string_to_unicode(self):
- self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
+ self.assertEqual(to_unicode(b"lel\xf1e", "latin-1"), "lel\xf1e")
def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
- self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
+ self.assertEqual(to_unicode("\xf1e\xf1e\xf1e"), "\xf1e\xf1e\xf1e")
def test_converting_a_strange_object_should_raise_TypeError(self):
self.assertRaises(TypeError, to_unicode, 423)
def test_errors_argument(self):
- self.assertEqual(
- to_unicode(b'a\xedb', 'utf-8', errors='replace'),
- 'a\ufffdb'
- )
+ self.assertEqual(to_unicode(b"a\xedb", "utf-8", errors="replace"), "a\ufffdb")
class ToBytesTest(unittest.TestCase):
def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
- self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
+ self.assertEqual(to_bytes("\xa3 49"), b"\xc2\xa3 49")
def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
- self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
+ self.assertEqual(to_bytes("\xa3 49", "latin-1"), b"\xa3 49")
def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
- self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
+ self.assertEqual(to_bytes(b"lel\xf1e"), b"lel\xf1e")
def test_converting_a_strange_object_should_raise_TypeError(self):
self.assertRaises(TypeError, to_bytes, unittest)
def test_errors_argument(self):
- self.assertEqual(
- to_bytes('a\ufffdb', 'latin-1', errors='replace'),
- b'a?b'
- )
+ self.assertEqual(to_bytes("a\ufffdb", "latin-1", errors="replace"), b"a?b")
class MemoizedMethodTest(unittest.TestCase):
def test_memoizemethod_noargs(self):
class A:
-
@memoizemethod_noargs
def cached(self):
return object()
@@ -141,7 +140,7 @@ class BinaryIsTextTest(unittest.TestCase):
assert binary_is_text(b"hello")
def test_utf_16_strings_contain_null_bytes(self):
- assert binary_is_text("hello".encode('utf-16'))
+ assert binary_is_text("hello".encode("utf-16"))
def test_one_with_encoding(self):
assert binary_is_text(b"<div>Price \xa3</div>")
@@ -151,7 +150,6 @@ class BinaryIsTextTest(unittest.TestCase):
class UtilsPythonTestCase(unittest.TestCase):
-
def test_equal_attributes(self):
class Obj:
pass
@@ -160,45 +158,45 @@ class UtilsPythonTestCase(unittest.TestCase):
b = Obj()
# no attributes given return False
self.assertFalse(equal_attributes(a, b, []))
- # not existent attributes
- self.assertFalse(equal_attributes(a, b, ['x', 'y']))
+ # nonexistent attributes
+ self.assertFalse(equal_attributes(a, b, ["x", "y"]))
a.x = 1
b.x = 1
# equal attribute
- self.assertTrue(equal_attributes(a, b, ['x']))
+ self.assertTrue(equal_attributes(a, b, ["x"]))
b.y = 2
# obj1 has no attribute y
- self.assertFalse(equal_attributes(a, b, ['x', 'y']))
+ self.assertFalse(equal_attributes(a, b, ["x", "y"]))
a.y = 2
# equal attributes
- self.assertTrue(equal_attributes(a, b, ['x', 'y']))
+ self.assertTrue(equal_attributes(a, b, ["x", "y"]))
a.y = 1
# differente attributes
- self.assertFalse(equal_attributes(a, b, ['x', 'y']))
+ self.assertFalse(equal_attributes(a, b, ["x", "y"]))
# test callable
a.meta = {}
b.meta = {}
- self.assertTrue(equal_attributes(a, b, ['meta']))
+ self.assertTrue(equal_attributes(a, b, ["meta"]))
# compare ['meta']['a']
- a.meta['z'] = 1
- b.meta['z'] = 1
+ a.meta["z"] = 1
+ b.meta["z"] = 1
- get_z = operator.itemgetter('z')
- get_meta = operator.attrgetter('meta')
+ get_z = operator.itemgetter("z")
+ get_meta = operator.attrgetter("meta")
def compare_z(obj):
return get_z(get_meta(obj))
- self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
+ self.assertTrue(equal_attributes(a, b, [compare_z, "x"]))
# fail z equality
- a.meta['z'] = 2
- self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
+ a.meta["z"] = 2
+ self.assertFalse(equal_attributes(a, b, [compare_z, "x"]))
def test_get_func_args(self):
def f1(a, b, c):
@@ -218,7 +216,6 @@ class UtilsPythonTestCase(unittest.TestCase):
pass
class Callable:
-
def __call__(self, a, b, c):
pass
@@ -228,34 +225,31 @@ class UtilsPythonTestCase(unittest.TestCase):
partial_f2 = functools.partial(f1, b=None)
partial_f3 = functools.partial(partial_f2, None)
- self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
- self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
- self.assertEqual(get_func_args(f3), ['a', 'b', 'c'])
- self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
- self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
- self.assertEqual(get_func_args(partial_f1), ['b', 'c'])
- self.assertEqual(get_func_args(partial_f2), ['a', 'c'])
- self.assertEqual(get_func_args(partial_f3), ['c'])
- self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
+ self.assertEqual(get_func_args(f1), ["a", "b", "c"])
+ self.assertEqual(get_func_args(f2), ["a", "b", "c"])
+ self.assertEqual(get_func_args(f3), ["a", "b", "c"])
+ self.assertEqual(get_func_args(A), ["a", "b", "c"])
+ self.assertEqual(get_func_args(a.method), ["a", "b", "c"])
+ self.assertEqual(get_func_args(partial_f1), ["b", "c"])
+ self.assertEqual(get_func_args(partial_f2), ["a", "c"])
+ self.assertEqual(get_func_args(partial_f3), ["c"])
+ self.assertEqual(get_func_args(cal), ["a", "b", "c"])
self.assertEqual(get_func_args(object), [])
+ self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"])
+ self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"])
- if platform.python_implementation() == 'CPython':
- # TODO: how do we fix this to return the actual argument names?
- self.assertEqual(get_func_args(str.split), [])
- self.assertEqual(get_func_args(" ".join), [])
+ if platform.python_implementation() == "CPython":
+ # doesn't work on CPython: https://bugs.python.org/issue42785
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
- elif platform.python_implementation() == 'PyPy':
- self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
- self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
- self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
+ elif platform.python_implementation() == "PyPy":
+ self.assertEqual(
+ get_func_args(operator.itemgetter(2), stripself=True), ["obj"]
+ )
def test_without_none_values(self):
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
self.assertEqual(
- without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
- {'one': 1, 'three': 3, 'four': 4})
-
-
-if __name__ == "__main__":
- unittest.main()
+ without_none_values({"one": 1, "none": None, "three": 3, "four": 4}),
+ {"one": 1, "three": 3, "four": 4},
+ )
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index a92d9a0ac..e6d1abe3f 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,3 +1,4 @@
+import json
import unittest
import warnings
from hashlib import sha1
@@ -18,29 +19,38 @@ from scrapy.utils.request import (
request_authenticate,
request_fingerprint,
request_httprepr,
+ request_to_curl,
)
from scrapy.utils.test import get_crawler
class UtilsRequestTest(unittest.TestCase):
-
def test_request_authenticate(self):
r = Request("http://www.example.com")
- request_authenticate(r, 'someuser', 'somepass')
- self.assertEqual(r.headers['Authorization'], b'Basic c29tZXVzZXI6c29tZXBhc3M=')
+ request_authenticate(r, "someuser", "somepass")
+ self.assertEqual(r.headers["Authorization"], b"Basic c29tZXVzZXI6c29tZXBhc3M=")
def test_request_httprepr(self):
r1 = Request("http://www.example.com")
- self.assertEqual(request_httprepr(r1), b'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
+ self.assertEqual(
+ request_httprepr(r1), b"GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n"
+ )
r1 = Request("http://www.example.com/some/page.html?arg=1")
- self.assertEqual(request_httprepr(r1), b'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')
-
- r1 = Request("http://www.example.com", method='POST',
- headers={"Content-type": b"text/html"}, body=b"Some body")
self.assertEqual(
request_httprepr(r1),
- b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
+ b"GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n",
+ )
+
+ r1 = Request(
+ "http://www.example.com",
+ method="POST",
+ headers={"Content-type": b"text/html"},
+ body=b"Some body",
+ )
+ self.assertEqual(
+ request_httprepr(r1),
+ b"POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body",
)
def test_request_httprepr_for_non_http_request(self):
@@ -61,77 +71,77 @@ class FingerprintTest(unittest.TestCase):
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
(
Request("http://example.org"),
- b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
+ b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",
{},
),
(
Request("https://example.org"),
- b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+ b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l",
{},
),
(
Request("https://example.org?a"),
- b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
+ b"G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c",
{},
),
(
Request("https://example.org?a=b"),
- b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
+ b"\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17",
{},
),
(
Request("https://example.org?a=b&a"),
- b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
+ b"t+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96",
{},
),
(
Request("https://example.org?a=b&a=c"),
- b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
+ b"\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef",
{},
),
(
- Request("https://example.org", method='POST'),
- b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
+ Request("https://example.org", method="POST"),
+ b"\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d",
{},
),
(
- Request("https://example.org", body=b'a'),
- b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
+ Request("https://example.org", body=b"a"),
+ b"\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_",
{},
),
(
- Request("https://example.org", method='POST', body=b'a'),
- b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
+ Request("https://example.org", method="POST", body=b"a"),
+ b"5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7",
{},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+ Request("https://example.org#a", headers={"A": b"B"}),
+ b"\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l",
{},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
- {'include_headers': ['A']},
+ Request("https://example.org#a", headers={"A": b"B"}),
+ b"]\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.",
+ {"include_headers": ["A"]},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
- {'keep_fragments': True},
+ Request("https://example.org#a", headers={"A": b"B"}),
+ b"<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f",
+ {"keep_fragments": True},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
- {'include_headers': ['A'], 'keep_fragments': True},
+ Request("https://example.org#a", headers={"A": b"B"}),
+ b"\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11",
+ {"include_headers": ["A"], "keep_fragments": True},
),
(
Request("https://example.org/ab"),
- b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
+ b"N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6",
{},
),
(
- Request("https://example.org/a", body=b'b'),
- b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
+ Request("https://example.org/a", body=b"b"),
+ b"_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5",
{},
),
)
@@ -143,42 +153,42 @@ class FingerprintTest(unittest.TestCase):
self.assertEqual(self.function(r1), self.function(r2))
def test_query_string_key_without_value(self):
- r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
- r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+ r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78132,199")
+ r2 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
self.assertNotEqual(self.function(r1), self.function(r2))
def test_caching(self):
- r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
- self.assertEqual(
- self.function(r1),
- self.cache[r1][self.default_cache_key]
- )
+ r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
+ self.assertEqual(self.function(r1), self.cache[r1][self.default_cache_key])
def test_header(self):
r1 = Request("http://www.example.com/members/offers.html")
r2 = Request("http://www.example.com/members/offers.html")
- r2.headers['SESSIONID'] = b"somehash"
+ r2.headers["SESSIONID"] = b"somehash"
self.assertEqual(self.function(r1), self.function(r2))
def test_headers(self):
r1 = Request("http://www.example.com/")
r2 = Request("http://www.example.com/")
- r2.headers['Accept-Language'] = b'en'
+ r2.headers["Accept-Language"] = b"en"
r3 = Request("http://www.example.com/")
- r3.headers['Accept-Language'] = b'en'
- r3.headers['SESSIONID'] = b"somehash"
+ r3.headers["Accept-Language"] = b"en"
+ r3.headers["SESSIONID"] = b"somehash"
self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
- self.assertEqual(self.function(r1),
- self.function(r1, include_headers=['Accept-Language']))
+ self.assertEqual(
+ self.function(r1), self.function(r1, include_headers=["Accept-Language"])
+ )
self.assertNotEqual(
- self.function(r1),
- self.function(r2, include_headers=['Accept-Language']))
+ self.function(r1), self.function(r2, include_headers=["Accept-Language"])
+ )
- self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
- self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
+ self.assertEqual(
+ self.function(r3, include_headers=["accept-language", "sessionid"]),
+ self.function(r3, include_headers=["SESSIONID", "Accept-Language"]),
+ )
def test_fragment(self):
r1 = Request("http://www.example.com/test.html")
@@ -190,8 +200,8 @@ class FingerprintTest(unittest.TestCase):
def test_method_and_body(self):
r1 = Request("http://www.example.com")
- r2 = Request("http://www.example.com", method='POST')
- r3 = Request("http://www.example.com", method='POST', body=b'request body')
+ r2 = Request("http://www.example.com", method="POST")
+ r3 = Request("http://www.example.com", method="POST", body=b"request body")
self.assertNotEqual(self.function(r1), self.function(r2))
self.assertNotEqual(self.function(r2), self.function(r3))
@@ -209,7 +219,7 @@ class FingerprintTest(unittest.TestCase):
# would put the body right after the URL.
r1 = Request("http://www.example.com/foo")
fp1 = self.function(r1)
- r2 = Request("http://www.example.com/f", body=b'oo')
+ r2 = Request("http://www.example.com/f", body=b"oo")
fp2 = self.function(r2)
self.assertNotEqual(fp1, fp2)
@@ -217,13 +227,9 @@ class FingerprintTest(unittest.TestCase):
"""Test hardcoded hashes, to make sure future changes to not introduce
backward incompatibilities."""
actual = [
- self.function(request, **kwargs)
- for request, _, kwargs in self.known_hashes
- ]
- expected = [
- _fingerprint
- for _, _fingerprint, _ in self.known_hashes
+ self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
]
+ expected = [_fingerprint for _, _fingerprint, _ in self.known_hashes]
self.assertEqual(actual, expected)
@@ -233,77 +239,77 @@ class RequestFingerprintTest(FingerprintTest):
known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
(
Request("http://example.org"),
- 'b2e5245ef826fd9576c93bd6e392fce3133fab62',
+ "b2e5245ef826fd9576c93bd6e392fce3133fab62",
{},
),
(
Request("https://example.org"),
- 'bd10a0a89ea32cdee77917320f1309b0da87e892',
+ "bd10a0a89ea32cdee77917320f1309b0da87e892",
{},
),
(
Request("https://example.org?a"),
- '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
+ "2fb7d48ae02f04b749f40caa969c0bc3c43204ce",
{},
),
(
Request("https://example.org?a=b"),
- '42e5fe149b147476e3f67ad0670c57b4cc57856a',
+ "42e5fe149b147476e3f67ad0670c57b4cc57856a",
{},
),
(
Request("https://example.org?a=b&a"),
- 'd23a9787cb56c6375c2cae4453c5a8c634526942',
+ "d23a9787cb56c6375c2cae4453c5a8c634526942",
{},
),
(
Request("https://example.org?a=b&a=c"),
- '9a18a7a8552a9182b7f1e05d33876409e421e5c5',
+ "9a18a7a8552a9182b7f1e05d33876409e421e5c5",
{},
),
(
- Request("https://example.org", method='POST'),
- 'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
+ Request("https://example.org", method="POST"),
+ "ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6",
{},
),
(
- Request("https://example.org", body=b'a'),
- '4bb136e54e715a4ea7a9dd1101831765d33f2d60',
+ Request("https://example.org", body=b"a"),
+ "4bb136e54e715a4ea7a9dd1101831765d33f2d60",
{},
),
(
- Request("https://example.org", method='POST', body=b'a'),
- '6c6595374a304b293be762f7b7be3f54e9947c65',
+ Request("https://example.org", method="POST", body=b"a"),
+ "6c6595374a304b293be762f7b7be3f54e9947c65",
{},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- 'bd10a0a89ea32cdee77917320f1309b0da87e892',
+ Request("https://example.org#a", headers={"A": b"B"}),
+ "bd10a0a89ea32cdee77917320f1309b0da87e892",
{},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- '515b633cb3ca502a33a9d8c890e889ec1e425e65',
- {'include_headers': ['A']},
+ Request("https://example.org#a", headers={"A": b"B"}),
+ "515b633cb3ca502a33a9d8c890e889ec1e425e65",
+ {"include_headers": ["A"]},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- '505c96e7da675920dfef58725e8c957dfdb38f47',
- {'keep_fragments': True},
+ Request("https://example.org#a", headers={"A": b"B"}),
+ "505c96e7da675920dfef58725e8c957dfdb38f47",
+ {"keep_fragments": True},
),
(
- Request("https://example.org#a", headers={'A': b'B'}),
- 'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
- {'include_headers': ['A'], 'keep_fragments': True},
+ Request("https://example.org#a", headers={"A": b"B"}),
+ "d6f673cdcb661b7970c2b9a00ee63e87d1e2e5da",
+ {"include_headers": ["A"], "keep_fragments": True},
),
(
Request("https://example.org/ab"),
- '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+ "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
{},
),
(
- Request("https://example.org/a", body=b'b'),
- '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+ Request("https://example.org/a", body=b"b"),
+ "4e2870fee58582d6f81755e9b8fdefe3cba0c951",
{},
),
)
@@ -314,36 +320,29 @@ class RequestFingerprintTest(FingerprintTest):
def tearDown(self) -> None:
warnings.simplefilter("default", ScrapyDeprecationWarning)
- @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+ @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
def test_part_separation(self):
super().test_part_separation()
class RequestFingerprintDeprecationTest(unittest.TestCase):
-
def test_deprecation_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
request_fingerprint(Request("http://www.example.com"))
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
- any(
- 'Call to deprecated function' in message
- for message in messages
- )
+ any("Call to deprecated function" in message for message in messages)
)
- self.assertFalse(any('non-default' in message for message in messages))
+ self.assertFalse(any("non-default" in message for message in messages))
def test_deprecation_non_default_parameters(self):
with pytest.warns(ScrapyDeprecationWarning) as warnings:
request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
messages = [str(warning.message) for warning in warnings]
self.assertTrue(
- any(
- 'Call to deprecated function' in message
- for message in messages
- )
+ any("Call to deprecated function" in message for message in messages)
)
- self.assertTrue(any('non-default' in message for message in messages))
+ self.assertTrue(any("non-default" in message for message in messages))
class RequestFingerprintAsBytesTest(FingerprintTest):
@@ -352,24 +351,21 @@ class RequestFingerprintAsBytesTest(FingerprintTest):
known_hashes = RequestFingerprintTest.known_hashes
def test_caching(self):
- r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+ r1 = Request("http://www.example.com/hnnoticiaj1.aspx?78160,199")
self.assertEqual(
- self.function(r1),
- bytes.fromhex(self.cache[r1][self.default_cache_key])
+ self.function(r1), bytes.fromhex(self.cache[r1][self.default_cache_key])
)
- @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+ @pytest.mark.xfail(reason="known bug kept for backward compatibility", strict=True)
def test_part_separation(self):
super().test_part_separation()
def test_hashes(self):
actual = [
- self.function(request, **kwargs)
- for request, _, kwargs in self.known_hashes
+ self.function(request, **kwargs) for request, _, kwargs in self.known_hashes
]
expected = [
- bytes.fromhex(_fingerprint)
- for _, _fingerprint, _ in self.known_hashes
+ bytes.fromhex(_fingerprint) for _, _fingerprint, _ in self.known_hashes
]
self.assertEqual(actual, expected)
@@ -385,8 +381,10 @@ def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False)
if cache_key not in cache:
fp = sha1()
fp.update(to_bytes(request.method))
- fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
- fp.update(request.body or b'')
+ fp.update(
+ to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments))
+ )
+ fp.update(request.body or b"")
if include_headers:
for hdr in include_headers:
if hdr in request.headers:
@@ -401,39 +399,38 @@ REQUEST_OBJECTS_TO_TEST = (
Request("http://www.example.com/"),
Request("http://www.example.com/query?id=111&cat=222"),
Request("http://www.example.com/query?cat=222&id=111"),
- Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
- Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
+ Request("http://www.example.com/hnnoticiaj1.aspx?78132,199"),
+ Request("http://www.example.com/hnnoticiaj1.aspx?78160,199"),
Request("http://www.example.com/members/offers.html"),
Request(
"http://www.example.com/members/offers.html",
- headers={'SESSIONID': b"somehash"},
+ headers={"SESSIONID": b"somehash"},
),
Request(
"http://www.example.com/",
- headers={'Accept-Language': b"en"},
+ headers={"Accept-Language": b"en"},
),
Request(
"http://www.example.com/",
headers={
- 'Accept-Language': b"en",
- 'SESSIONID': b"somehash",
+ "Accept-Language": b"en",
+ "SESSIONID": b"somehash",
},
),
Request("http://www.example.com/test.html"),
Request("http://www.example.com/test.html#fragment"),
- Request("http://www.example.com", method='POST'),
- Request("http://www.example.com", method='POST', body=b'request body'),
+ Request("http://www.example.com", method="POST"),
+ Request("http://www.example.com", method="POST", body=b"request body"),
)
class BackwardCompatibilityTestCase(unittest.TestCase):
-
def test_function_backward_compatibility(self):
include_headers_to_test = (
None,
- ['Accept-Language'],
- ['accept-language', 'sessionid'],
- ['SESSIONID', 'Accept-Language'],
+ ["Accept-Language"],
+ ["accept-language", "sessionid"],
+ ["SESSIONID", "Accept-Language"],
)
for request_object in REQUEST_OBJECTS_TO_TEST:
for include_headers in include_headers_to_test:
@@ -467,7 +464,6 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
warning to be logged."""
class RequestFingerprinter:
-
cache = WeakKeyDictionary()
def fingerprint(self, request):
@@ -475,14 +471,14 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
fp = sha1()
fp.update(to_bytes(request.method))
fp.update(to_bytes(canonicalize_url(request.url)))
- fp.update(request.body or b'')
+ fp.update(request.body or b"")
self.cache[request] = fp.digest()
return self.cache[request]
for request_object in REQUEST_OBJECTS_TO_TEST:
with warnings.catch_warnings() as logged_warnings:
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
crawler = get_crawler(settings_dict=settings)
fp = crawler.request_fingerprinter.fingerprint(request_object)
@@ -492,11 +488,10 @@ class BackwardCompatibilityTestCase(unittest.TestCase):
class RequestFingerprinterTestCase(unittest.TestCase):
-
def test_default_implementation(self):
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(prevent_warnings=False)
- request = Request('https://example.com')
+ request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
_request_fingerprint_as_bytes(request),
@@ -505,11 +500,11 @@ class RequestFingerprinterTestCase(unittest.TestCase):
def test_deprecated_implementation(self):
settings = {
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6',
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6",
}
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(settings_dict=settings)
- request = Request('https://example.com')
+ request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
_request_fingerprint_as_bytes(request),
@@ -518,11 +513,11 @@ class RequestFingerprinterTestCase(unittest.TestCase):
def test_recommended_implementation(self):
settings = {
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
}
with warnings.catch_warnings(record=True) as logged_warnings:
crawler = get_crawler(settings_dict=settings)
- request = Request('https://example.com')
+ request = Request("https://example.com")
self.assertEqual(
crawler.request_fingerprinter.fingerprint(request),
fingerprint(request),
@@ -531,34 +526,30 @@ class RequestFingerprinterTestCase(unittest.TestCase):
def test_unknown_implementation(self):
settings = {
- 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
+ "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5",
}
with self.assertRaises(ValueError):
get_crawler(settings_dict=settings)
class CustomRequestFingerprinterTestCase(unittest.TestCase):
-
def test_include_headers(self):
-
class RequestFingerprinter:
-
def fingerprint(self, request):
- return fingerprint(request, include_headers=['X-ID'])
+ return fingerprint(request, include_headers=["X-ID"])
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
crawler = get_crawler(settings_dict=settings)
- r1 = Request("http://www.example.com", headers={'X-ID': '1'})
+ r1 = Request("http://www.example.com", headers={"X-ID": "1"})
fp1 = crawler.request_fingerprinter.fingerprint(r1)
- r2 = Request("http://www.example.com", headers={'X-ID': '2'})
+ r2 = Request("http://www.example.com", headers={"X-ID": "2"})
fp2 = crawler.request_fingerprinter.fingerprint(r2)
self.assertNotEqual(fp1, fp2)
def test_dont_canonicalize(self):
-
class RequestFingerprinter:
cache = WeakKeyDictionary()
@@ -570,7 +561,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
return self.cache[request]
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
crawler = get_crawler(settings_dict=settings)
@@ -581,26 +572,24 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
self.assertNotEqual(fp1, fp2)
def test_meta(self):
-
class RequestFingerprinter:
-
def fingerprint(self, request):
- if 'fingerprint' in request.meta:
- return request.meta['fingerprint']
+ if "fingerprint" in request.meta:
+ return request.meta["fingerprint"]
return fingerprint(request)
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
}
crawler = get_crawler(settings_dict=settings)
r1 = Request("http://www.example.com")
fp1 = crawler.request_fingerprinter.fingerprint(r1)
- r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+ r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
fp2 = crawler.request_fingerprinter.fingerprint(r2)
- r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+ r3 = Request("http://www.example.com", meta={"fingerprint": "a"})
fp3 = crawler.request_fingerprinter.fingerprint(r3)
- r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
+ r4 = Request("http://www.example.com", meta={"fingerprint": "b"})
fp4 = crawler.request_fingerprinter.fingerprint(r4)
self.assertNotEqual(fp1, fp2)
self.assertNotEqual(fp1, fp4)
@@ -608,57 +597,51 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
self.assertEqual(fp2, fp3)
def test_from_crawler(self):
-
class RequestFingerprinter:
-
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def __init__(self, crawler):
- self._fingerprint = crawler.settings['FINGERPRINT']
+ self._fingerprint = crawler.settings["FINGERPRINT"]
def fingerprint(self, request):
return self._fingerprint
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
- 'FINGERPRINT': b'fingerprint',
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
+ "FINGERPRINT": b"fingerprint",
}
crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
- self.assertEqual(fingerprint, settings['FINGERPRINT'])
+ self.assertEqual(fingerprint, settings["FINGERPRINT"])
def test_from_settings(self):
-
class RequestFingerprinter:
-
@classmethod
def from_settings(cls, settings):
return cls(settings)
def __init__(self, settings):
- self._fingerprint = settings['FINGERPRINT']
+ self._fingerprint = settings["FINGERPRINT"]
def fingerprint(self, request):
return self._fingerprint
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
- 'FINGERPRINT': b'fingerprint',
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
+ "FINGERPRINT": b"fingerprint",
}
crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
- self.assertEqual(fingerprint, settings['FINGERPRINT'])
+ self.assertEqual(fingerprint, settings["FINGERPRINT"])
def test_from_crawler_and_settings(self):
-
class RequestFingerprinter:
-
# This method is ignored due to the presence of from_crawler
@classmethod
def from_settings(cls, settings):
@@ -669,20 +652,82 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase):
return cls(crawler)
def __init__(self, crawler):
- self._fingerprint = crawler.settings['FINGERPRINT']
+ self._fingerprint = crawler.settings["FINGERPRINT"]
def fingerprint(self, request):
return self._fingerprint
settings = {
- 'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
- 'FINGERPRINT': b'fingerprint',
+ "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter,
+ "FINGERPRINT": b"fingerprint",
}
crawler = get_crawler(settings_dict=settings)
request = Request("http://www.example.com")
fingerprint = crawler.request_fingerprinter.fingerprint(request)
- self.assertEqual(fingerprint, settings['FINGERPRINT'])
+ self.assertEqual(fingerprint, settings["FINGERPRINT"])
+
+
+class RequestToCurlTest(unittest.TestCase):
+ def _test_request(self, request_object, expected_curl_command):
+ curl_command = request_to_curl(request_object)
+ self.assertEqual(curl_command, expected_curl_command)
+
+ def test_get(self):
+ request_object = Request("https://www.example.com")
+ expected_curl_command = "curl -X GET https://www.example.com"
+ self._test_request(request_object, expected_curl_command)
+
+ def test_post(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_headers(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ headers={"Content-Type": "application/json", "Accept": "application/json"},
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ ' --data-raw \'{"foo": "bar"}\''
+ " -H 'Content-Type: application/json' -H 'Accept: application/json'"
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_cookies_dict(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ cookies={"foo": "bar"},
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_cookies_list(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ cookies=[{"foo": "bar"}],
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+ )
+ self._test_request(request_object, expected_curl_command)
if __name__ == "__main__":
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index c056d3682..838169e4f 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,5 +1,4 @@
import unittest
-import warnings
from itertools import chain
from pathlib import Path
from urllib.parse import urlparse
@@ -7,54 +6,47 @@ from urllib.parse import urlparse
import pytest
from xtractmime import BINARY_BYTES, RESOURCE_HEADER_BUFFER_LENGTH
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.http import HtmlResponse, Response, TextResponse, XmlResponse
+from scrapy.http import HtmlResponse, JsonResponse, Response, TextResponse, XmlResponse
from scrapy.http.headers import Headers
from scrapy.responsetypes import ResponseTypes
from scrapy.utils.misc import load_object
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (
+ _get_encoding_or_mime_type_from_headers,
get_meta_refresh,
get_response_class,
open_in_browser,
- response_httprepr,
response_status_message,
- _get_encoding_or_mime_type_from_headers,
)
-
-__doctests__ = ['scrapy.utils.response']
+__doctests__ = ["scrapy.utils.response"]
# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
PRE_XTRACTMIME_HTML_STARTS = (
- b'<!DOCTYPE HTML',
- b'<HTML',
+ b"<!DOCTYPE HTML",
+ b"<HTML",
)
POST_XTRACTMIME_HTML_STARTS = (
- b'<HEAD',
- b'<SCRIPT',
- b'<IFRAME',
- b'<H1',
- b'<DIV',
- b'<FONT',
- b'<TABLE',
- b'<A',
- b'<STYLE',
- b'<TITLE',
- b'<B',
- b'<BODY',
- b'<BR',
- b'<P',
- b'<!--',
+ b"<HEAD",
+ b"<SCRIPT",
+ b"<IFRAME",
+ b"<H1",
+ b"<DIV",
+ b"<FONT",
+ b"<TABLE",
+ b"<A",
+ b"<STYLE",
+ b"<TITLE",
+ b"<B",
+ b"<BODY",
+ b"<BR",
+ b"<P",
+ b"<!--",
)
NON_BINARY_ASCII_BYTES = (
- byte
- for byte in (
- bytes([byte]) for byte in range(128)
- )
- if byte not in BINARY_BYTES
+ byte for byte in (bytes([byte]) for byte in range(128)) if byte not in BINARY_BYTES
)
# https://mimesniff.spec.whatwg.org/#whitespace-byte
@@ -104,67 +96,56 @@ PRE_XTRACTMIME_SCENARIOS = (
*(
(mime_type, load_object(class_path))
for mime_type, class_path in ResponseTypes.CLASSES.items()
- if mime_type not in (
+ if mime_type
+ not in (
# “Note that XHTML is best parsed as XML”
# https://lxml.de/parsing.html
"application/xhtml+xml",
"application/vnd.wap.xhtml+xml",
)
),
-
# JavaScript MIME types should trigger a TextResponse.
#
# https://mimesniff.spec.whatwg.org/#javascript-mime-type
*(
(mime_type, TextResponse)
for mime_type in (
- 'application/javascript',
- 'application/x-javascript',
- 'text/ecmascript',
- 'text/javascript',
- 'text/javascript1.0',
- 'text/javascript1.1',
- 'text/javascript1.2',
- 'text/javascript1.3',
- 'text/javascript1.4',
- 'text/javascript1.5',
- 'text/jscript',
- 'text/livescript',
- 'text/x-ecmascript',
- 'text/x-javascript',
-
+ "application/javascript",
+ "application/x-javascript",
+ "text/ecmascript",
+ "text/javascript",
+ "text/javascript1.0",
+ "text/javascript1.1",
+ "text/javascript1.2",
+ "text/javascript1.3",
+ "text/javascript1.4",
+ "text/javascript1.5",
+ "text/jscript",
+ "text/livescript",
+ "text/x-ecmascript",
+ "text/x-javascript",
# Unofficial
- 'application/x-javascript',
+ "application/x-javascript",
)
),
-
- # JSON MIME types should trigger a TextResponse.
+ # JSON MIME types should trigger a JsonResponse.
#
# https://mimesniff.spec.whatwg.org/#json-mime-type
*(
- (mime_type, TextResponse)
+ (mime_type, JsonResponse)
for mime_type in (
- 'application/json',
- 'text/json',
-
+ "application/json",
# Unofficial
- 'application/json-amazonui-streaming',
- 'application/x-json',
+ "application/json-amazonui-streaming",
+ "application/x-json",
)
),
-
# Binary MIME types should trigger a Response.
#
# https://mimesniff.spec.whatwg.org/#json-mime-type
- *(
- (mime_type, Response)
- for mime_type in (
- 'application/pdf',
- )
- ),
+ *((mime_type, Response) for mime_type in ("application/pdf",)),
)
),
-
# Content-Type triumphs body, except for:
#
# - Binary content mislabeled as plain text due to an Apache bug
@@ -176,58 +157,54 @@ PRE_XTRACTMIME_SCENARIOS = (
*(
(
{
- 'body': body,
- 'headers': Headers({'Content-Type': [content_type]}),
+ "body": body,
+ "headers": Headers({"Content-Type": [content_type]}),
},
response_class,
)
for body, content_type, response_class in (
*(
- (b'\x00\x01\xff', content_type, TextResponse)
+ (b"\x00\x01\xff", content_type, TextResponse)
for content_type in (
- 'text/json',
# text/plain variants *not* affected by the Apache bug
- 'text/plain; charset=Iso-8859-1',
- 'text/plain; charset=utf-8',
- 'text/plain; charset=windows-1252',
+ "text/plain; charset=Iso-8859-1",
+ "text/plain; charset=utf-8",
+ "text/plain; charset=windows-1252",
)
),
)
),
-
# Compressed content should be of type Response until uncompressed.
(
{
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Encoding': ['zip'],
- 'Content-Type': ['text/html'],
+ "Content-Encoding": ["zip"],
+ "Content-Type": ["text/html"],
}
)
},
Response,
),
-
# We take the file extension of URL paths into account, except for HTTP
# responses, because “they are unreliable and easily spoofed”.
#
# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
*(
(
- {'url': f'{protocol}://example.com/a.{extension}'},
+ {"url": f"{protocol}://example.com/a.{extension}"},
response_class,
)
for protocol in ("file", "ftp")
for extension, response_class in (
("gz", Response),
("html", HtmlResponse),
- ("json", TextResponse),
+ ("json", JsonResponse),
("pdf", Response),
("txt", TextResponse),
("xml", XmlResponse),
)
),
-
# Unlike in a web browser, where an attachment Content-Disposition header
# causes the response to be downloaded, and hence MIME sniffing becomes
# irrelevant, in Scrapy those responses are handled the same as any, and
@@ -237,9 +214,9 @@ PRE_XTRACTMIME_SCENARIOS = (
(
{
"url": f"{protocol}://example.com/a",
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Disposition': [
+ "Content-Disposition": [
'attachment; filename="a.xml"',
]
}
@@ -253,9 +230,9 @@ PRE_XTRACTMIME_SCENARIOS = (
(
{
"url": f"{protocol}://example.com/a",
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Disposition': [
+ "Content-Disposition": [
'attachment; filename="a.html"',
],
"Content-Type": "text/xml",
@@ -271,9 +248,9 @@ PRE_XTRACTMIME_SCENARIOS = (
{
"url": f"{protocol}://example.com/a",
"body": b"<?xml",
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Disposition': [
+ "Content-Disposition": [
'attachment; filename="a.html"',
],
}
@@ -283,17 +260,14 @@ PRE_XTRACTMIME_SCENARIOS = (
)
for protocol in ("http", "https")
),
-
# Without anything else, the body determines the response class.
*(
({"body": body}, response_class)
for body, response_class in (
- (b'<html><head><title>Hello ', HtmlResponse),
+ (b"Hello ", HtmlResponse),
(b'\n. ', HtmlResponse),
-
+ (b"\n. ", HtmlResponse),
# https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type
*(
(prefix + start + b">", HtmlResponse)
@@ -345,7 +319,6 @@ PRE_XTRACTMIME_SCENARIOS = (
(b"\x1f\x8b\x08", Response),
(b"PK\x03\x04", Response),
(b"Rar \x1a\x07\x00", Response),
-
# A body is considered binary if its header (first 1445 bytes)
# contains any binary data byte.
*((byte, Response) for byte in BINARY_BYTES[1:]),
@@ -361,7 +334,6 @@ PRE_XTRACTMIME_SCENARIOS = (
(b"a" * RESOURCE_HEADER_BUFFER_LENGTH + BINARY_BYTES[0], TextResponse),
)
),
-
# A Content-Type whose essence is "unknown/unknown", "application/unknown",
# or "*/*" has the same effect as no Content-Type being defined.
#
@@ -369,9 +341,9 @@ PRE_XTRACTMIME_SCENARIOS = (
*(
(
{
- 'body': b'",
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Encoding': ['zip'],
+ "Content-Encoding": ["zip"],
}
- )
+ ),
},
Response,
),
-
# If the body is empty, it contains no binary data bytes, hence body-based
# MIME type detection must interpret the result as text.
#
# https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type
({}, TextResponse),
-
# We take the file extension of URL paths into account, except for HTTP
# responses, because “they are unreliable and easily spoofed”.
#
# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata
*(
(
- {'url': f'{protocol}://example.com/a.{extension}'},
+ {"url": f"{protocol}://example.com/a.{extension}"},
response_class,
)
for protocol in ("file", "ftp")
@@ -575,22 +536,21 @@ POST_XTRACTMIME_SCENARIOS = (
),
*(
(
- {'url': f'{protocol}://example.com/a.html'},
+ {"url": f"{protocol}://example.com/a.html"},
response_class,
)
for protocol, response_class in (
*((protocol, TextResponse) for protocol in ("http", "https")),
)
),
-
# Content-Type triumphes Content-Disposition.
*(
(
{
"url": f"{protocol}://example.com/a",
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Disposition': [
+ "Content-Disposition": [
f'attachment; filename="a.{file_extension}"',
],
"Content-Type": [content_type],
@@ -604,29 +564,27 @@ POST_XTRACTMIME_SCENARIOS = (
("xml", "application/octet-stream", Response),
)
),
-
# File extension triumphs body.
(
{
"body": b"",
- 'headers': Headers(
+ "headers": Headers(
{
- 'Content-Disposition': [
+ "Content-Disposition": [
'attachment; filename="a.gz"',
],
}
- )
+ ),
},
Response,
),
(
{
"body": b"",
- 'url': 'file:///a.gz',
+ "url": "file:///a.gz",
},
Response,
),
-
# Without anything else, the body determines the response class.
*(
({"body": body}, response_class)
@@ -660,7 +618,7 @@ POST_XTRACTMIME_SCENARIOS = (
)
),
(b"\x0c', TextResponse),
- (b'a", TextResponse),
+ (b"a', bbody)
return True
+
response = HtmlResponse(url, body=body)
assert open_in_browser(response, _openfunc=browser_open), "Browser not called"
@@ -764,33 +704,42 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertRaises(TypeError, open_in_browser, resp, debug=True)
def test_get_meta_refresh(self):
- r1 = HtmlResponse("http://www.example.com", body=b"""
+ r1 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
Dummy
blahablsdfsal&
- """)
- r2 = HtmlResponse("http://www.example.com", body=b"""
+ """,
+ )
+ r2 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
Dummy
blahablsdfsal&
- """)
- r3 = HtmlResponse("http://www.example.com", body=b"""
+ """,
+ )
+ r3 = HtmlResponse(
+ "http://www.example.com",
+ body=b"""
![""" response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text='asd'), - ]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.org/page.html", text="asd"), + ], + ) # override denied extensions - lx = self.extractor_cls(deny_extensions=['html']) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/photo.jpg'), - ]) + lx = self.extractor_cls(deny_extensions=["html"]) + self.assertEqual( + lx.extract_links(response), + [ + Link(url="http://example.org/photo.jpg"), + ], + ) def test_process_value(self): """Test restrict_xpaths with encodings""" @@ -280,7 +427,11 @@ class Base:](photo.jpg){kind=link}
Hello
Hello
Hello
Hello
Hello
an Jos\xe9 de
', - encoding='utf-8') - Selector(r1).xpath('//text()').getall() + r1 = TextResponse( + "http://www.example.com", + body=b"an Jos\xe9 de
", + encoding="utf-8", + ) + Selector(r1).xpath("//text()").getall() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" - x = Selector(text='') + x = Selector(text="") weakref.ref(x) - assert not hasattr(x, '__dict__'), f"{x.__class__.__name__} does not use __slots__" + assert not hasattr( + x, "__dict__" + ), f"{x.__class__.__name__} does not use __slots__" def test_selector_bad_args(self): - with self.assertRaisesRegex(ValueError, 'received both response and text'): - Selector(TextResponse(url='http://example.com', body=b''), text='') + with self.assertRaisesRegex(ValueError, "received both response and text"): + Selector(TextResponse(url="http://example.com", body=b""), text="") + + +class JMESPathTestCase(unittest.TestCase): + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) + def test_json_has_html(self) -> None: + """Sometimes the information is returned in a json wrapper""" + + body = """ + { + "content": [ + { + "name": "A", + "value": "a" + }, + { + "name": { + "age": 18 + }, + "value": "b" + }, + { + "name": "C", + "value": "c" + }, + { + "name": "D", + "value": "d
"
+ }
+ ],
+ "html": "a
bc
bc
def
"
+ }
+ """
+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+ self.assertEqual(
+ resp.jmespath("html").get(),
+ "a
bc
bc
def
",
+ )
+ self.assertEqual(
+ resp.jmespath("html").xpath("//div/a/text()").getall(),
+ ["a", "b", "d"],
+ )
+ self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"])
+ self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18")
+
+ @pytest.mark.skipif(
+ not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath"
+ )
+ def test_html_has_json(self) -> None:
+ body = """
+
+
+ {
+ "user": [
+ {
+ "name": "A",
+ "age": 18
+ },
+ {
+ "name": "B",
+ "age": 32
+ },
+ {
+ "name": "C",
+ "age": 22
+ },
+ {
+ "name": "D",
+ "age": 25
+ }
+ ],
+ "total": 4,
+ "status": "ok"
+ }
+
+
+ """
+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+ self.assertEqual(
+ resp.xpath("//div/content/text()").jmespath("user[*].name").getall(),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(
+ resp.xpath("//div/content").jmespath("user[*].name").getall(),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4")
+
+ @pytest.mark.skipif(
+ not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath"
+ )
+ def test_jmestpath_with_re(self) -> None:
+ body = """
+ Information
+
+
+ {
+ "user": [
+ {
+ "name": "A",
+ "age": 18
+ },
+ {
+ "name": "B",
+ "age": 32
+ },
+ {
+ "name": "C",
+ "age": 22
+ },
+ {
+ "name": "D",
+ "age": 25
+ }
+ ],
+ "total": 4,
+ "status": "ok"
+ }
+
+
+ """
+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+ self.assertEqual(
+ resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(
+ resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"),
+ ["A", "B", "C", "D"],
+ )
+
+ self.assertEqual(
+ resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), []
+ )
+
+ self.assertEqual(
+ resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"),
+ None,
+ )
+
+ self.assertEqual(
+ resp.xpath("//div/content")
+ .jmespath("user[*].age.to_string(@)")
+ .re(r"(\d+)"),
+ ["18", "32", "22", "25"],
+ )
+
+ @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath")
+ def test_jmespath_not_available(my_json_page) -> None:
+ body = """
+ {
+ "website": {"name": "Example"}
+ }
+ """
+ resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+ with pytest.raises(AttributeError):
+ resp.jmespath("website.name").get()
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 916fe012a..3fde5e8c5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,13 +1,20 @@
import unittest
from unittest import mock
-from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
- SETTINGS_PRIORITIES, get_settings_priority)
+import pytest
+
+from scrapy.settings import (
+ SETTINGS_PRIORITIES,
+ BaseSettings,
+ Settings,
+ SettingsAttribute,
+ get_settings_priority,
+)
+
from . import default_settings
class SettingsGlobalFuncsTest(unittest.TestCase):
-
def test_get_settings_priority(self):
for prio_str, prio_num in SETTINGS_PRIORITIES.items():
self.assertEqual(get_settings_priority(prio_str), prio_num)
@@ -15,127 +22,142 @@ class SettingsGlobalFuncsTest(unittest.TestCase):
class SettingsAttributeTest(unittest.TestCase):
-
def setUp(self):
- self.attribute = SettingsAttribute('value', 10)
+ self.attribute = SettingsAttribute("value", 10)
def test_set_greater_priority(self):
- self.attribute.set('value2', 20)
- self.assertEqual(self.attribute.value, 'value2')
+ self.attribute.set("value2", 20)
+ self.assertEqual(self.attribute.value, "value2")
self.assertEqual(self.attribute.priority, 20)
def test_set_equal_priority(self):
- self.attribute.set('value2', 10)
- self.assertEqual(self.attribute.value, 'value2')
+ self.attribute.set("value2", 10)
+ self.assertEqual(self.attribute.value, "value2")
self.assertEqual(self.attribute.priority, 10)
def test_set_less_priority(self):
- self.attribute.set('value2', 0)
- self.assertEqual(self.attribute.value, 'value')
+ self.attribute.set("value2", 0)
+ self.assertEqual(self.attribute.value, "value")
self.assertEqual(self.attribute.priority, 10)
def test_overwrite_basesettings(self):
- original_dict = {'one': 10, 'two': 20}
+ original_dict = {"one": 10, "two": 20}
original_settings = BaseSettings(original_dict, 0)
attribute = SettingsAttribute(original_settings, 0)
- new_dict = {'three': 11, 'four': 21}
+ new_dict = {"three": 11, "four": 21}
attribute.set(new_dict, 10)
self.assertIsInstance(attribute.value, BaseSettings)
self.assertCountEqual(attribute.value, new_dict)
self.assertCountEqual(original_settings, original_dict)
- new_settings = BaseSettings({'five': 12}, 0)
+ new_settings = BaseSettings({"five": 12}, 0)
attribute.set(new_settings, 0) # Insufficient priority
self.assertCountEqual(attribute.value, new_dict)
attribute.set(new_settings, 10)
self.assertCountEqual(attribute.value, new_settings)
def test_repr(self):
- self.assertEqual(repr(self.attribute),
- "