From f096f17fa4ac1307fa1c81ae082bb52e9f86653a Mon Sep 17 00:00:00 2001 From: Elias Ram Date: Tue, 20 Feb 2024 20:32:02 +0100 Subject: [PATCH 01/45] test #6 added tests for check command --- tests/test_command_check.py | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 592494aba..d503628b8 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,3 +1,8 @@ +import sys +from io import StringIO +from unittest.mock import Mock, PropertyMock, patch + +from scrapy.commands.check import Command from tests.test_commands import CommandTest @@ -94,3 +99,53 @@ class CheckSpider(scrapy.Spider): raise Exception('SCRAPY_CHECK not set') """ self._test_contract(parse_def=parse_def) + + @patch("scrapy.commands.check.ContractsManager") + def test_run_with_opts_list_prints_spider(self, cm_cls_mock): + output = StringIO() + sys.stdout = output + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + type(spider_cls_mock).name = PropertyMock(return_value=spider_name) + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = ["fakeMethod1", "fakeMethod2"] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=True)) + + self.assertEqual( + "FakeSpider\n * fakeMethod1\n * fakeMethod2\n", output.getvalue() + ) + sys.stdout = sys.__stdout__ + + @patch("scrapy.commands.check.ContractsManager") + def test_run_without_opts_list_does_not_crawl_spider_with_no_tested_methods( + self, cm_cls_mock + ): + cmd = Command() + cmd.settings = Mock(getwithbase=Mock(return_value={})) + cm_cls_mock.return_value = cm_mock = Mock() + spider_loader_mock = Mock() + cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + spider_name = "FakeSpider" + spider_cls_mock = Mock() + spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ + x + ] + tested_methods = [] + cm_mock.tested_methods_from_spidercls.side_effect = lambda x: { + spider_cls_mock: tested_methods + }[x] + + cmd.run([spider_name], Mock(list=False)) + + cmd.crawler_process.crawl.assert_not_called() From e27d320c3cde3c965e61a674e8880043943cf17a Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Tue, 20 Feb 2024 23:58:39 +0100 Subject: [PATCH 02/45] test #3 Increased branch coverage for form.py --- tests/test_http_request.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..510ae74ba 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1642,6 +1642,62 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a form response with invalid form data throws a type error""" + response = _buildresponse( + """ +
+ +
+ """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a form response with invalid form data throws a value error""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + class CustomFormdata: + def __iter__(self): + raise ValueError("Custom iteration error for testing") + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=CustomFormdata()) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def tearDown(self): warnings.resetwarnings() super().tearDown() From e2a0c85f1167c7c32219eaf095c1818b2c74702c Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:21:24 +0100 Subject: [PATCH 03/45] doc #3 Clarified test comments --- tests/test_http_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 510ae74ba..95e4a7be0 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1643,7 +1643,7 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["allow_nan"], True) def test_form_response_with_invalid_formdata_type_error(self): - """Test that a form response with invalid form data throws a type error""" + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" response = _buildresponse( """ @@ -1659,7 +1659,7 @@ class JsonRequestTest(RequestTest): ) def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a form response with invalid form data throws a value error""" + """Test that a ValueError is raised for fault-inducing iterable formdata input""" response = _buildresponse( """ From 12b4417c56d8aa76cbe3a36c026962612453ee6e Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:30:50 +0100 Subject: [PATCH 04/45] test #22 Improve json_request.py coverage --- tests/test_http_request.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 6dc9ec8b7..d1c435468 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1642,6 +1642,26 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) + def test_replacement_both_body_and_data_warns(self): + """Test that we can get a warning if both body and data are passed for branch coverage""" + body1 = None + body2 = b"body" + data1 = { + "name1": "value1", + } + data2 = { + "name2": "value2", + } + r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) + + with mock.patch("warnings.warn") as mock_warn: + r1.replace(data=data2, body=body2) + mock_warn.assert_called_once() + (warning_message,), _ = mock_warn.call_args + self.assertIn( + "Both body and data passed. data will be ignored", warning_message + ) + def tearDown(self): warnings.resetwarnings() super().tearDown() From bc036542a82ec054dd6a36b4b928a9bc6ae48e63 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 02:35:45 +0100 Subject: [PATCH 05/45] refactor #3 Moved tests to FormRequestTest --- tests/test_http_request.py | 112 ++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 95e4a7be0..39afc5fd1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1426,6 +1426,62 @@ class FormRequestTest(RequestTest): r = self.request_class.from_response(response) self.assertEqual(r.method, expected) + def test_form_response_with_invalid_formdata_type_error(self): + """Test that a ValueError is raised for non-iterable and non-dict formdata input""" + response = _buildresponse( + """ + + +
+ """ + ) + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=123) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_form_response_with_custom_invalid_formdata_value_error(self): + """Test that a ValueError is raised for fault-inducing iterable formdata input""" + response = _buildresponse( + """ +
+ +
+ """ + ) + + class CustomFormdata: + def __iter__(self): + raise ValueError("Custom iteration error for testing") + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formdata=CustomFormdata()) + + self.assertIn( + "formdata should be a dict or iterable of tuples", str(context.exception) + ) + + def test_get_form_with_xpath_no_form_parent(self): + """Test that _get_from raised a ValueError when an XPath selects an element + not nested within a
and no parent is found""" + response = _buildresponse( + """ +
+

This paragraph is not inside a form.

+
+ + +
+ """ + ) + + with self.assertRaises(ValueError) as context: + FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') + + self.assertIn("No
element found with", str(context.exception)) + def _buildresponse(body, **kwargs): kwargs.setdefault("body", body) @@ -1642,62 +1698,6 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["ensure_ascii"], True) self.assertEqual(kwargs["allow_nan"], True) - def test_form_response_with_invalid_formdata_type_error(self): - """Test that a ValueError is raised for non-iterable and non-dict formdata input""" - response = _buildresponse( - """ - - -
- """ - ) - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=123) - - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - - def test_form_response_with_custom_invalid_formdata_value_error(self): - """Test that a ValueError is raised for fault-inducing iterable formdata input""" - response = _buildresponse( - """ -
- -
- """ - ) - - class CustomFormdata: - def __iter__(self): - raise ValueError("Custom iteration error for testing") - - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=CustomFormdata()) - - self.assertIn( - "formdata should be a dict or iterable of tuples", str(context.exception) - ) - - def test_get_form_with_xpath_no_form_parent(self): - """Test that _get_from raised a ValueError when an XPath selects an element - not nested within a
and no parent is found""" - response = _buildresponse( - """ -
-

This paragraph is not inside a form.

-
- - -
- """ - ) - - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formxpath='//div[@id="outside-form"]/p') - - self.assertIn("No
element found with", str(context.exception)) - def tearDown(self): warnings.resetwarnings() super().tearDown() From b7a7ae7dbbaddd9d14b50c1257370af51e1ac1b5 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:04:45 +0100 Subject: [PATCH 06/45] refactor #22 Change comment and warning catching --- tests/test_http_request.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d1c435468..a45293695 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1643,7 +1643,7 @@ class JsonRequestTest(RequestTest): self.assertEqual(kwargs["allow_nan"], True) def test_replacement_both_body_and_data_warns(self): - """Test that we can get a warning if both body and data are passed for branch coverage""" + """Test that we get a warning if both body and data are passed""" body1 = None body2 = b"body" data1 = { @@ -1654,12 +1654,11 @@ class JsonRequestTest(RequestTest): } r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - with mock.patch("warnings.warn") as mock_warn: + with warnings.catch_warnings(record=True) as _warnings: r1.replace(data=data2, body=body2) - mock_warn.assert_called_once() - (warning_message,), _ = mock_warn.call_args self.assertIn( - "Both body and data passed. data will be ignored", warning_message + "Both body and data passed. data will be ignored", + str(_warnings[0].message), ) def tearDown(self): From 877398a3dee8e92300ef52177b986be16a55f277 Mon Sep 17 00:00:00 2001 From: noon <14049705+noon-io@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:13:57 +0100 Subject: [PATCH 07/45] refactor #3 Remove inner class in form test --- tests/test_http_request.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 39afc5fd1..71d442a81 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1452,12 +1452,8 @@ class FormRequestTest(RequestTest): """ ) - class CustomFormdata: - def __iter__(self): - raise ValueError("Custom iteration error for testing") - with self.assertRaises(ValueError) as context: - FormRequest.from_response(response, formdata=CustomFormdata()) + FormRequest.from_response(response, formdata=("a",)) self.assertIn( "formdata should be a dict or iterable of tuples", str(context.exception) From e208f82076182e71a4eb8470eb527363158f4b0c Mon Sep 17 00:00:00 2001 From: vishesh10 Date: Thu, 22 Feb 2024 16:46:24 +0530 Subject: [PATCH 08/45] Add support for multiple-compressed responses (#6063) --- .../downloadermiddlewares/httpcompression.py | 34 ++++++++-- .../compressed/html-gzip-deflate-gzip.bin | Bin 0 -> 8014 bytes .../compressed/html-gzip-deflate.bin | Bin 0 -> 7991 bytes ...st_downloadermiddleware_httpcompression.py | 58 ++++++++++++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 tests/sample_data/compressed/html-gzip-deflate-gzip.bin create mode 100644 tests/sample_data/compressed/html-gzip-deflate.bin diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 709333948..1e340abb6 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from itertools import chain from logging import getLogger from typing import TYPE_CHECKING, List, Optional, Union @@ -102,18 +103,18 @@ class HttpCompressionMiddleware: if isinstance(response, Response): content_encoding = response.headers.getlist("Content-Encoding") if content_encoding: - encoding = content_encoding.pop() max_size = request.meta.get("download_maxsize", self._max_size) warn_size = request.meta.get("download_warnsize", self._warn_size) try: - decoded_body = self._decode( - response.body, encoding.lower(), max_size + decoded_body, content_encoding = self._handle_encoding( + response.body, content_encoding, max_size ) except _DecompressionMaxSizeExceeded: raise IgnoreRequest( f"Ignored response {response} because its body " - f"({len(response.body)} B) exceeded DOWNLOAD_MAXSIZE " - f"({max_size} B) during decompression." + f"({len(response.body)} B compressed) exceeded " + f"DOWNLOAD_MAXSIZE ({max_size} B) during " + f"decompression." ) if len(response.body) < warn_size <= len(decoded_body): logger.warning( @@ -121,6 +122,7 @@ class HttpCompressionMiddleware: f"({len(decoded_body)} B) is larger than the " f"download warning size ({warn_size} B)." ) + response.headers["Content-Encoding"] = content_encoding if self.stats: self.stats.inc_value( "httpcompression/response_bytes", @@ -144,6 +146,28 @@ class HttpCompressionMiddleware: return response + def _handle_encoding(self, body, content_encoding, max_size): + to_decode, to_keep = self._split_encodings(content_encoding) + for encoding in to_decode: + body = self._decode(body, encoding, max_size) + return body, to_keep + + def _split_encodings(self, content_encoding): + to_keep = [ + encoding.strip().lower() + for encoding in chain.from_iterable( + encodings.split(b",") for encodings in content_encoding + ) + ] + to_decode = [] + while to_keep: + encoding = to_keep.pop() + if encoding not in ACCEPTED_ENCODINGS: + to_keep.append(encoding) + return to_decode, to_keep + to_decode.append(encoding) + return to_decode, to_keep + def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: if encoding == b"gzip" or encoding == b"x-gzip": return gunzip(body, max_size=max_size) diff --git a/tests/sample_data/compressed/html-gzip-deflate-gzip.bin b/tests/sample_data/compressed/html-gzip-deflate-gzip.bin new file mode 100644 index 0000000000000000000000000000000000000000..d66f4c5a03df80085208d8990df59e1702c63b50 GIT binary patch literal 8014 zcmV-UAF<#ciwFP!000000|7T5$l!RKHC1(7lmGWskY!}j3`dRLA?64iIN z(Bm6+BEw8}=7FI~PdU-NoI$sYi==Cu_J|W)>riui_5yjZAmBi- za14I9JL=!~z9y>ZG1)S5g}nXRqrXC3vsTsJxlC(lv|L*QFL5BGB|t7A5yV z8)A3oNJKx+@#>nN_hRjk)tc|^i3jX>&B*BY@$Ev^66`9i!wQBVG74zW%9*8cbkc6?RKdAV?G-_trm>*sW}ar*a&5LpX`354B2 zJVft*L#}$lU@P1H7kM35$fXWsR*Qqj@tWcB+nQ6xzO@V ze?a^36_WPKCXE`Ns`Negn^k~p+;;T*8$ie;1bp6St z(w-Ipec=t_y1GQ*KveU{lv3?#F=k0kbRY>0A4F~1xD=?@M3uRBHJka-ks|i%Zhnt8W>_q2-r0 z^-u!V5+n|%pa!682{6Q>_06P`h@o0wBYJVSaF=L&RRKcrX_*L8RPz(U)gdu0GGdDR zmc_)V1l1k>bRUH7XWSo5PdMZvi$h1BEM-6_=In`%3tnn}lq#Bg^$gCISUqXVBb1=* zd=tDfApw0l+JMM+?{BO)BF&g7SF7Y^v$HLN3tN#W8E16ETYD2oYs&_?H2}6vR3UK( z_{)kAF7wb}OL;&Fv4dgPP)f^a3N7o&phv&nA!Y*U!Q7&zuaeqQZM719B2RW`lsx3o zn}CT^%CI5x5-raR$+spIROp~2V919=AR9CgHJ-+tBW9%yxo|vdE`0z*Kq8?Yy!_nCd0S zh&z!TM(H1d`T$FT@GA2%irjKTzDys+x{4~d>X-f>r5ug~b{GdfW>#P(#egxpc%qf6 z>DP)e>+TYNEZc`qf&?IRFykM(p%xoY;UW7o?z&cs=h!}l(6Jr)`SWdK%$PEnW3qZN(FbDwU!XPf{z^VG<~FX z&xfzFGk;zP0l}tQw4^=M2HBeW_}kj`4nrkCy!EuEN`yGtlr9ASNb>n1Y8?{hyUM@n!6o3f z6SG?oNxgcFz!&||FlnvZyt!%0ht!C`uw6er024X2R##+YlH+!Jlk?0>v5gEL1%8^X z#)~KRTHSOzRcp-VW^=~w(7#UAm9^kQx;Vr2PGE?Fi@jVzteIk@LTaV7eJLKzoOm^= zPO$GM*vFVAZU!MA4#ARF2CfGRScR@PcD$ zx534=Hc-ue8xs{+-&UcPpC)Xd7P2{PfTKW?JyEN9_`8l>uA(3n5>A+qtPXu`_XUVQ zSssHW4u@`d^A|e*kKaHzuJTpiGM-8+Ly1c<2iC3JFUo< zuG0h_{fJ>0^G(Byut|Y%JNI)2#|sQRp@sAHH!(o6B>-kztNl)s#BD}o67^i9h602L z{~(>?#Z}a6wPtl!Tjv=t0QYO=sSl_g)aK1@h0XK2QUM>Bq8mX8455VR*bA`qLcE{P zM=BU97HPKOP!*(gOI%nFIMs&%*7TC1QXyPlr!Br|VtiyOO^K>`nL%uB-=Xf-N=YvA zinAzm&>c!yr$(75NdAB;6JUoc<4hR$PCiEiLK4#-XXaeqq`;M74STk+$|gl@#TbsT z#1dxlq1DSMz+?M2UO`98lP=Pz75o=3rSO~9friQqcBbNCTAjSQS6nH$OTla{J~mI8 zOpmodeX0B7fihrZv|maZ?9PR=U{+SH8C)Sl+Y#C*^iBvRu}u;!hnbWd*Hx)Tu`HOM zpUC<>_^F2Vzj7;UJsc)}GSxslTdO>3RL8-k77`?Shp`IDGp?Wdh{L469oaO;TUu@%0*;vkQn?|LgUZR| zJe2~e7&)3r#>c=AgZ8%M%8nW>k}Mg<)`AFoN#oKU=p_!*+2@p3Xb%dplB7c6#m^F^ zJVQQCvP+cc#vyt!(Qj zKjP3Qeu8C3XBwpx(-o(>uH$xE#F%SBX{0P~fp#%uib4XImbHj=Ouiy|w#+_)3nwU9 z?5uu@tThe(elbD&;ORsRtBxK!DNJFn5ZE(U86;msFc&>QK`mydl9MQdHFgNK>8sXa zv@MeNjEsL}qsm5UR>g7L?ljS^nA3SIHTN^E_RnREelLy@BWanbG zquU4T#~@qxdWf@1^+a{AB@u`3=0~=CRJK)7#IEy#nH@7?UUhTzK$}MJpb^ABP%5Ij zeyxZ=Q?VVUj2q`eTk7^z$5IzT(`v2hR{h8uiWx+0IY-wX5YPI@DHNT$RJ+Z-4#@T@#!%X`l_lvrgE4Q>mq6fs`- zQ7EgmGApdZRm8D;n2*b)*`R7UB^_0^4*|O6d*eZcO^iioi|TN;?fXLNS(GU>@ALV< zOzmx4P+Q!J`I<6umbbnlFN+XJ<38VdIe!3dHSCX)tCWekKgVA0xc0cP8n<2zCq7r$_&$>#yt9zxNF&WewuA9^hBUskUknYcv^aAdSlL~f@qSeOj>tl zVEbAjyk$2R1oR-)T=4d+VlRSjUk3G+hNh@*o6aZ)?S0D5mer{6QycM9yAffGQ z_VfUD341ZN6LUGQYHkxKjMO(-%?t3jeq9#h`Z1?@3ws{3yVFxHzwrI1@#QmG5?T?+ zWe&>tDBWs5H5zyh=5c6GdF9-uF_%_A9hj$#cHxkcnxlmfpP$c**KPEpW0H~ineFyY zip9_NO-^rd@+Oq-OJxNMwHMm<$;d^yBBwL|p^?v{WPtrA`x8ZGOECmwOIY;NFG+by z*}{S=lWfcneHg3#RF^R3UctdXf+7_Hil z6d-11AO0ks`-HCq9g(iP!1M31F>XJUY8eXcUucF_sfX>x2&+L~Kz@>3Tj#Dd5|E8C z!<QWOGi~F~SlKb3yxK$ymy9d}zWl`l#B5^3K`vs+c}trIdW$lDBndDt@2D zAAMtE7xhRUTw*`pAYj%+w8T`suGak0TPL=(UI-C=oz|u!rVBXp`0dGpB-Si#RhMN! zN#F3di4856Utsk=PRy-c8(&Yetg~DcF>F{alA!UE-UGbm35gyo?ME3=LZtoedinK} z6hGfNW|0!~!P=Y(Hfb9Rrsa?tjUZx|fK6pV5+eD0P$K-ABaiq#O107{&BfC~vH7m{M zEC2n~Nqh5o@x<3SMxK1^wLTzxrAByM(n>h?42QnZb`}w21xROuo`^hOdWvF$Qh{9Q zuUVQKHFUQ|cPmodPVCV>dHRw@%=A7S({K@6t9j}1jnLDs(f|zo z3`P&G#bJ_kPC8u~c~e~-VMN95EkcP*^IyNYgJlT>P(9T7xV23Qk31N3wYT{-^xSRE z-rC&nn)djd=@^j1ma_7W{g+F!!pfFtyYJ7&&acShdj6RrC(azS1Aca`2Ha;IU!Nfk zvc|%Ey0hC$z1r_)=lD+cE|Ibe(Y4t|zsOZJv|kr^n26bl`3s&!o`n6q4vW3NJb4j! z%;VdBEn!t8X6XBI#fIGOHL-%w{H-s@{yke0RhYnZ)B5&M@b;AWAVAzR6@Po)Oth!Jcvi1Eobj`;5XZY}OZ_ zlnNN&{hQE{J|j%T9HpVJVMTq+=L_|;jY@jbEU0Dae0XnZGhk)CR^a`&lb3}-6`qfD zZ4E#J{ai|AvuO5gy?GADo|;yexy_EptN~b8j5h}^V0N15{~ZeKMWs3+hCyF(2JfW~ zGrMcxT|fH6=X5l_rE)(3FaHslsKQ+Q)E4SQHVNePoJfs)VDdO!qXRU4jK31&1mJGq z3c`x?vVs=|8V;E~P95$V4U$%ReJ7K79qpcwB$o71pyfVeo>e;9TH}b>ptL4rm!DLG z;+pZ)_acn`lI&_=*Qy?F^+*B#lJSWhhoD$$4B%6e5IVhmNh+JbfN5}+>_?r1*G@F>u%?zVnIk-zqoT}p!1WZdl~ft~!*?-K1S@Pm>jaowvWTzM3M z3BO3Q?cBgXz-ghNjex$)Dyy}b^}|<#e!ZPG_y=}x|A_)rabLSx-L3Fveo_CZ*(l&Y zD-PmNR`wb_gMTng=Jr(dN|pQ^@_U)=5Ad>nB1uM>`*TdVu_VwMedMxb*nyPzg`whV zkp!u@25{G8YUy=6CQ1wNN)g&6{F=>QtTR>ZABWAWo}llHD)P_;%+YYq&gY@8@cO_s zgsbW7fYdvsb3{I${S2^W_|oxnh~E+hdmPVOOZg(**+rdNR_}~QcPu)+`Bfzp9$76y z6T+np{EKgEE;^}z?xk*7^H=Rw_Ys&}O=pgIx*((K=_d(n<;Q-?yDPf$immw+a9f!H z&u^NUNJVA#re0KVsW!U&hkgFmIcdB0{_ z#Bl%6wRy!kdZG*EcO>Zk^tvU+>`RLvYZ(XNwi&{ID`}XlJ5jhWcb(`*Lva1(KUTBV zhfaSma5(B6EQ06UYGaH6pkq)+gpv0f;?D?Z<(D_BQI+_A-n$!-Rx+%bNmE5C^N6sZ z?!#B=k!8R0OqYJba7(V4f z>$L&v0%`mTRKf(w9UN05)wyYYrqA$yrzw6Dy=dAet5K{Gj|r9|XeeqX^HPspIr%#f z-Aw!@o>~Pv&;!?>3M34&iT~68Cnva*Wr1jYc~RHe=R+*6RqTW*oNBxbKRqZb9+QW% z+~M1SJh1f^$Bx-%y(xrahiea@_nRT6?LEfN`chIKuKs7I=8dB3HQ$^*Vq_U4V;+(w z&xG)L+F8d!V#+C|rY7Nm0t0<2|A#65JFnvXdz1cP70V_MJz=!LJ0Xg_V*T+sRYv;R zk6NoD&MlCTj4_HDj!_nEA@WQ$2I`K>-@yWCZO+TFwM`)3rU>GemRv%4iXAX)Dm$R- za*9iw5l!Q?@hHkwSSVMuBT)PUJq%QA%gVpgTKg&{h&TAP(v%`-=@0$|LgRkkMv*{8b~W^|czkMAmIW zmlU}Cp)R`JX^iwkhun`=j|IJMt*!gDfG2X2|n?_@z@CA|)6 z3`06Xrc9BdI_ki=huE7)wOrRUO1{@QOe>+%_%A2EQQWFNd=WDq<&a z+_E-HUS;ZI-9l`!iYI!%9(=|zhQ77j+{T2b5cw5!bbDUf=X&-JWBK8lV86803mq~1R=iDU`^3$m7bIF+9 z1>vegA@&$FPKA@#v43`Eu62Rt3%EZq4I{GSipQk~{<(ySVwZW*v7nY8DCIaGya}hV zb^LoCY$}$>rHhKM+9xQUJjFN)ix3ju;lz;|4_8E@aq8(rEQ`u5(1-rvY8O*hm9TM) z?v19F-4lL0hz@HJwCY-)YAfY8wL@DPP!sFt@}Zl+BIlB3eakrVV!u0lfz`qaC+TFa zo}g~k#v4}w^-$=`d?OmxcKMV^k9F?V$tm<+F`dJ-+9d2#zYW^bZ}=`Ae|+H{TIepN zl-j37Gk^rkAH_*WbxV5sq#3&T_Y;W{Hh&k5?X!wpJ5ztb&+T^4JdK=dGS%b;M3lok z@2Yrle?&a;Kd8JjC2RWrb~TWLyojo%L=OInJrbKQS=DL^zTULK{mimaP~o>9B`!G9 z1rZ|Y`F3|MTRC2aZin8z6E*r5Y@{V=g`>eKzeWSR-;Hm$VN7-k9`}SlnRez0?PW_HBspUFdbM}- z`4+T860?A9h}Ey4oA{2x3_H41NP%T%chNw{(GIq9EQ*Y9xGp-=#g3S%a<~fBOt0%KB-SuxU8W zdyDjj-LJI#r2m8#xQ%lF^Upn6O;t$R)PT|G8FgluVbv@@`NnC&62)cDzqs-1ki!e2 zQLpvFTNk`&ItA-gK|IUd6W$WE`OV2^VtdeoRF$(4HPoZ{lnn~If%mEl(;L+Y<6f`68G1^B8dYC&;MZOJ$>6i> zf)IOGg!+1{gSn9E?Bai$VjF=Y#bq>rVp4u97H}lC1h(a+7xTRT1R8SZT zjjA;JE2E<+!gMD)?ge)hV97F(Hu!bu2fU$dP(%Q$vFOokK4LJ5o~bmuh}Tv7!&50^ z2RR8~I!V7qCEjqF?N_u*)FCy>W4a0ARS>d}F&eizQK56ySvR6MAU@p&Rt5gNuE;ok z+%4f_^-8>64gPq2)FWXPIDL;JVuG2lZBs)hSEL#BEj;puW0iGW@UO~-99A96VP4=A z7vEFbl7a>Z20k~mk2MEdGaQ{L>X)uj`mt@oggAT5M?lu3kE=B-y^pm{{fl6rkLNJ7 zKt^x8KUavhaoJ{Qng2|8e0|)w2p2y6!scm*n_g9g=;I|AEpsSX7>LxdbZi~UQ;pL$t#qTaOdBB3>-Lu`E&L;goo6E^}JZg0%i1 zzRKR8DoM>^WCz;RRey$cp8V!!MKh-uH3Z4v=PJU%cenZLe%~3n_3%XWvv!dd!_z1K Q142u<+9f5~j5i+u01a&HYybcN literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-gzip-deflate.bin b/tests/sample_data/compressed/html-gzip-deflate.bin new file mode 100644 index 0000000000000000000000000000000000000000..5066842ef7d137786f43682c99c63e24eae261ec GIT binary patch literal 7991 zcmV-7AIRW%oHbQ-T$BIzRgh+cATb04X;45K35n4uFiN_IbaxEt?(QBT-6bI%gMrdW zNC-&B1is_*d;R`>&U5a$aqfN7MqxiOdBgVM33^wKK@!z>x6tDob|S-J=M>3F*C}#i zpg=;RAY+OFu42tH27|;~He|t~fdY4cUpv;8DWh17mkiDKZ8oR6255#$`;^y|kJ#;? zxZmCH0IwUPfE&w;W6mC=J92Aq@D5>dKjnXAi8OEWb_!_iS#~fwGCbtuyjp-AtQ6Qh!9x|h6#k-K|Dn7e?zW%!eA@g{ug;2 zSIDIfWLArV$MKrs@!OhH#^cL0WYl^0UI%=kR9k=B_A)Qm$E~I3!7;!$o1qk8bai$0 z(9m&vV&{Fq&xAfh} zCx_;QKOonFU+l7@A9jL(uMon{A@W9RA`k8t>d`GQ2X~JEpZi1Q=MQO|SK8vM+5Rqb zNB)|77G#M1rOm7EfR2YdZP>Ajwtq{<%8JqQr+_AJuiL*C-u)d>S!0bV?h74vhZC@3 z=mNR;@3eoAl-w)#i(S#AN(b?X)64iR@2lOVp%xs(en(=nItgp9 zteho0ETa9jxz^azT6KFto8yn`)^wyQe)r2mxqRX9r0n;eZthlqzi)lf$gpawo<-4_ zWs@Ru?>0`==(vF)ue8i7)q!`hc=KVs(4C}xBq~%nyIW($VR27?5HdSAd$p+PG+Q<3 z=JX?E5BbG)ru1mgZQ`p<*2uyA&2`VF8!ev;n7CS9nkbTpoDqag%nX{F-B*ajmZ_E} zzAMc%WH8aFop{dxa@|W+WG@82m*hw`(NKl)veJk1wX8NmWJ8o0yIn~1i|&iVSO4r6 zhg*`A_INbYkH?l5sY&xufWq||LC>6Om9^Nn9Za7uH?qn{#^c%s->hatVnbxHDVago zvvYQfHn3dM0Lrh!Y|4ihp`{x%M@qkdR4w;u5)PNxb4msS?e+R8M%i zx|G#;|7Sj>+Kr(nNvl=s#h7&K(^Q)&bFZS6yX0k5*0Z+GlJo|jkF3i4$;^OQ57^5= z(o-G30nwx@Q5gl<(wK&8d+RxS>(ZVU0e#^O;<~y-;6POK z$dpp;YB6R>O>`g$4Ie~p+PD;`*F=@Mcjah1+f^y4DG$KRv4gK@@*42rs>OBK-tJk# zBo0IOuH@KtboRk*yOW*Pu;72iq4OfT!*m6U0IviN6^JWD09zwx?v-QI^BnyiyP^*g zk*m=WcxE50YsMf3%ODSkfHa8!#d4t`9g9oS+^cUH#i8YwHuX>f))FKRr=SL)YY8yK zqV>(Bl8B*NU?X~Qw{Vwed{qHL@oAX|QdIL3!qp)$Eiz(?`mjtgFDf0Qbkd-V*?mRLP$$|IDZ?0gfvG9dwdI@*B9ckgek zI3mrMDOaoHX0x*`f(u)bC>dvT!&`e3NNdXmxitW`O;jOq2l&g15H9o3U`u&G3bBJ> z*HB8!XbLUs$)HES-XUfJ>A~EhrmvFPQf;*oej-nHXp}tU(VKvYQ_8R*^Aau349T}9 z6;$Y;Bw)ygL?9b95H+60oFitX4Y_bUYc72NL_jIPp-ix({Hqu5WWBCKa0J%=WL4po z_59n&8e0u^R4PobS19xvc?i?yj1BpKWtr@ZO6|O|N0{m*$cQ_U9Y*ONg8BeUf$%Ex zGK$=CL%vKO#=43sx9XSvAEg|Q1a=q)K4w;6CdGg;yLh6Ns_EB?G3)LUe=OUFPl5y> zbTM8LgD@UoQj|SD`V1>%D(IVWwuNEWsDO0RdliTp;sd1$<>lv6pwe66%xrI~f1Y6I zld8TzRFtsfVAg~R7eJzpoSC7ilS&15o3)k@yn>G$@HBm-be|jwv^!mBhuF5Ldwqin+(Ejek3mwEyKHNNTa*p-HyOc_60M+ zaCzS!OmznYn@;+HVl+_cx)fJtzwTb{g>KDC&4>kLHi|*)Ehrrf9Rb0nTePG-)CSp_ z`uN-0^$tTNK)m&|rb>i3+LSH?|48!rA!;2G=DW(j>cJ)8wiB~k5J|myjldWE(J*PP z+q}7H%7@g5z_49EJpdCqwN_VTWs>7|dz16bOtFm&9|eAzt;UNd_FCO^J5_7U=4Nxo z?$EzZ)s?m2L%KM_^-f@jf{VReLado$q(W+?w0$Wa&762Os!p))C)mfBCT<2H9}dBi zR|c*J3Rs1%H+HG@lRIb;>$kzhwKh=Aej5`NSl?Em zmY*hUpBAz?Y=EOcl08wYdHB1IU9O@a6%tOEkgN`UZTAI;KUp4wB@ZFu1!;_7t`2cm zwHX^~U1`AuLc#`lD?;bD3af$7>?>0C^s>S`E0j~d|B$UF(yGz|dMKp=i#7A+Eatv0 z+Y@oB!$ceSO?)(1$=G2M#J}2v6p4R9z1o+ruH!j!9y_hbm#)(U9{q@681qfTjIc?8 za69*N2FD8wJfVg2^*1p&3R;9@OT|ZiUVBx>5ljn4%j&2@Iix=-3Oe^g_I!&qpd4Di&$B;ZPN%bxT}W z4>;9_0oL@AqEaDTU#Bg;X<~e2Dou&1d6_|MZr`Eq)=Eh(@`|%4bkH42S*J#sC`kT* zD-&RcE8|QU_f9@X140thA7|!V-lV{lVGVn>vC1YzY{eLku*4E(@uAhrD8OU;H(o(U z%abnBs1^JdFQxFC)`5n~40fjCVOpKMx>sB&xJ$uoEj~6+nM{whKz*tE`iR4%za7~$$6H!%9RiM-1X8&nl7q_0gEe*twdt$YVze!jDP!O7)T)*T{@n^Bs`>R0NXRS0ODLK^4 zQZ0+L9I3`&^y=k5k(1>-dmlV!_*)w-t5wh_*^>v{=w#<&wxinz>&GBl_j-u4O7%o_ zuO$(O@8(Cgd{nkoQN*tEf|(sNVqSG~^+20O@SqXIKTs;7x_+&QKvS_Dri>fsLtEZT#Eb{4E8b2D$k6=_^BS(=abXFVCRhuW!R+ttjZes(H`C{g>|LktN7X2U zH}sx~x?Dzg;m><8L&a^R&eRVqJj;|X8M-iQO};NXDLa_^tO3RcU*0#Ye4l;VDXzm+ z!7=(y%fq)?kGKd}lSCz4Oz>&1Q;*g}RFU2H1G5Iz)bCJTu@uwiusx{ahA8f zA}@;&NaH@=dO3dpZZ+(WlB<-7xj)BV@3{83uo|~s4deWO@RJ}YoA_s~?5Z92w)t)+ zt~g*Nk>n`}lyzawX;Hn;?NLGNLG;PyZ6LhBou#BapR?V*KwCT|i6Uq$J zpvFAFMZ=2322km{z&X(1v@KYP{Q@astGvdgr!uIq4b_shiwi9zXuWD`+ zD2&uMS@@Lc#dNanGl7tgyg z2?~~d)P82jl<>(=w_blhtVD-x?dg5@uk)?Xw5*Y-UKp*~juaqfW*`0}p8JHa1RasC zyTJ4Bu`zBxlxi6Y?O$kyR;h>W#t5rHUqF76TwCX^H4>1GF~girA&aurTV!)i*fGKq z4s${KW64;`aeQdPGWw|6hVstY@~W6VV5O9N-jcU_LK}p~6w}}lcmtSD@KTgc8 zT^nCdv#hgR6ftaAE|Q?}limZo<_U=&EbT`bQ9`8s?Rxq3lN3MSIcAX(^ugMB=T>94 z{U+4FQEyUwVpSgHnN9>91Lu8#Q#dMt3Vx+)nJ# zK6(0*M$Gg+9Mf58F^D( z9brVp?kz%zO!HsAxr1d11W-NH`M9-B36DG&bhWqnHT2wV&fePG@0#}boaq>l!XNM!(2aHMCzBc$kRUiTMkjMV^HHy$*}LzdU&ncg*A4el1~DBxdOQam9w* z?lrN3(EP10$o@TB6IGbNbkq9wQ1JGY_#i;wV74aUvQ6}*qT1xj22WOE$VHbf)e)Yr*=2gk%Lvj9F(8Ly?r%OLTOq6~U8;Jr;x z|G9We8Si~}H%!JH#=gm9ES?eBOu?RV1Ouf-Kl_ZrEo{~opOgw1;QgD>kv=0##2lre zuVF=f%;yXBw2ex7(k!TD>3n!^X)|DDy;k7;x09EJK^2~lbZrel1N~e|WwU7ZY`u97 z$DW#2n7Pf4$E*QZSBy6YE?{<==>HuG>_w$IA%;O;aR%?D4l}!J;9WoZ!{>A~zNKi8U84guevH2o;{@Pt;0nTu^s<5%1{w~TJx(3& z8V!OsS>3JhXMR!tsM#psKPwL6P*(ODJ%fKROy>4f z^h%Zd9P)da><{p=ej-Umnfr50xUnSA8hzxlW!Qm~_=Tb3YLNt~xCU_7WNPVkJSIvD z@JbQdCH$JrU#v4#?H`BDtDd0mj4JZb1ggv5Y~{y(%DXGN^NOwc6mVOa0ncxmnMg%t_NHD`aH%%B z{D*!1*5q&a7hdmARB!x*=>UdQ*P)Hf{}CqEa)NB*OLU7$r%)1!WIPj~+L7?cLrjGu z-%A~N^rNr#EVW#8sKpuBe9^_E5DV5m^=j1^cA471qN-ZcF>}AH;#I4BnTG;jvw5cR z>AinxCK^Sn1P{D2G;?=Em#$&ri{mXXMwko4GA66iR0qDAGI7s_({ql_ur<5` z`O5FIlm5Xp^U1L=NoA5ZD_yb1>txaQI?o)gSU2yDell%OI$gCBnE7FftFhq_HH7gK zvQ8#vJb+$q2nyuNyjXTh#O%D99&e^X)xkCtVWSJQ78pL|LhH2w>jG)~3RJ=b$sHV1 zBh|TSex}dxf2S#a6TN8KC#zAc5swL$BWNgUCi7B{T{-zX5Zz4tCZ1XaJJ18yp9&-l zvWfrG|0gH7lVyQueR)yW+UG+ou2t-WDV%D&3_m?6D;|@FvfSa@fjqGF7RQd+X1ytd zV~1-Gp!b_0rtLk(&-zkQAFlpqr{;~K>own;K4N4UBx4?uCeMWMdfHjXLSo7(rluz0 zfdT`4D*uNm{yVSY{d<%CU=_5Rv9(Pg-=+xSmX=&Xd5Rq{Y$`jT>vD=qoDogqwDBm)Rahuj zwIfjc13e5>Y|F~O(^~r~CWtrqwbGO#Y9(Tc9r@j2MgIauWSK=mJfWs$vB*exer}OH z|7tJNF**>7XChJ*bL;$dp~@rjPop4H@^oMQ!K3h!_~eG<*+*|y6uLoHxlpo>n={v~tv z4BZfGvX2QN7D*)5KQCpwzi^g&;gNUVi&_gBnP7pt#> zZf=OA8ZjCtbTSq&t#o~}gMoVqhdA3dExN#$;_qZZVkNx}Y79dK@d%MZHt6!bA4~h+&xgOLmAz@KQ)oN|wNLRk|Dk2|ziV(Uw5Bj?;D5%SZh>~qPO-38&QLm~DUG){$+*Rg+g zX0COC<_ow#F%2WK$fb*lui7Ul zo;<}k3X2dD-{HiO8V^@QqH*f!L@bNSEzpPl;c6FCR+X@EjP8x5mfaJ6JBSWz5wz-B zpK2@RH?>1s8c-AK=klSOz#`|8W_`;z@?yU`e1X-%3Mc7guAZQ7)y5lF0rgPm%X}jm z)^_=nNso2z)yXOJUNN1+wAv)>Q@;(`(r@@K9)En{9$M%wrIgyIMKgc|%OAx_M|Dej z`lK1U`S%lv5;lJqjqS6FTsu>L!O!h>&peHsYckd321Jy@JnyP_aeqWS@;|7&GbL;K z{&qExgS?2UrbG_@iaipWFIm-U3clX7!Trp#QBdKxA0;k0(ghJB>G^hdE?YTXhHi)6 zy%RP17i^>@X@#S~DZfSoyx)y)xM56o3Lf`_Kbdys3GHP`FFGe^NcJoR1<`HjQWFdH zFaUDNKc>nA!3wN1+YY;@T_n}L!`!|HomkMhsQL^18i*iBaA3)Ur=i?~E)K%yeyw1& zMWw*6ubnD=zMM@csBC~{lV4(DUl2%VDZNK+NhCR9`Fgc?^!XOFLlU!qZHU#cpqu!P z!VEjQR7iny@WSXC$;@tStk;l`SA*b?S zP+Xv44!3lHWTGJ6PHH54ecz=$m05$X9)J4`Bg*<|m#}F#&3lXVhTX5U{G|Vc7PyUb z0Q1j1T1{0*+SGv2=oxipm|@i{Kl#RK!V<-0&%e0w>yX0>qEWB)!dn--XgUS!R6#t; z-4ostwE4}+XJUKMgH)BX5jE7K_~YGksK)T!bE_E>ch=;8LLNy8-3wcydeP{pA08b~ z%1vMzE|d)lyMgzr3)36b2;*L_z8QK-f*Ms{bKuuq{K??6?1B(`SA_a{tb@6b>g?iw zn_?S*BgJJjfMQa9E97+z@bA@{rZ@A+42VK83%I3ONmNi63yrEY`zxcPDZ+FoJMIN{ z6=2CSkv8~s=m)%^Y*0i1tFh?OY(8QziJqx6yNK6S`@>TyV+T12U^+>^MkU^Gn(bG# zOVlAX%450-;#Cl`kTDv!I#Ho>)mb;9I3PaV237_ByspSNe%vkLWA#eBUJd?uebggi z6*zs5BVvM?ux(RACs(8y^({Q|hGUg=T=1{Th8$KM%VA#N6c^u9+LD3>2nIekw2w6h zTQeM;DC(E4QTnlM!h|?`%tt`hq>rmLEWMAlPW_8uppWM;v_M90ygyfnwsF~JXqo>^ zcYJ-^xCj?M{leyHhnrqih3MlY7%g)sSQv=Zv2<)5%2SQxP=Im(-$gIq_-1-Vz^C05 zFlr+*WGx-rB}25sBU_IXw<2CFy|FA&<>8cgY%X(J(So%8Aim1ppDIbsVq^!})K!0m tb)Nj@W<@in7&QdR;O8pB!FRX$>we!Ex%Kcw^s{!67Q@pg{{uoxxY|2R=d%C+ literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 9deb81c37..ae5569d0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -27,6 +27,8 @@ FORMAT = { "x-gzip": ("html-gzip.bin", "gzip"), "rawdeflate": ("html-rawdeflate.bin", "deflate"), "zlibdeflate": ("html-zlibdeflate.bin", "deflate"), + "gzip-deflate": ("html-gzip-deflate.bin", "gzip, deflate"), + "gzip-deflate-gzip": ("html-gzip-deflate-gzip.bin", "gzip, deflate, gzip"), "br": ("html-br.bin", "br"), # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin "zstd-static-content-size": ("html-zstd-static-content-size.bin", "zstd"), @@ -205,6 +207,62 @@ class HttpCompressionTest(TestCase): assert newresponse is not response self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"]) + def test_multi_compression_single_header(self): + response = self._getresponse("gzip-deflate") + request = response.request + newresponse = self.mw.process_response(request, response, self.spider) + assert newresponse is not response + assert "Content-Encoding" not in newresponse.headers + assert newresponse.body.startswith(b" Date: Mon, 26 Feb 2024 10:53:06 -0800 Subject: [PATCH 09/45] Remove usage of deprecated mktemp (#5285) --- .bandit.yml | 1 - tests/test_commands.py | 5 ++--- tests/test_downloader_handlers.py | 28 ++++++++++++---------------- tests/test_http2_client_protocol.py | 4 ++-- tests/test_pipeline_crawl.py | 4 ++-- tests/test_spiderloader/__init__.py | 4 ++-- tests/test_spiderstate.py | 5 ++--- tests/test_squeues_request.py | 2 +- tests/test_webclient.py | 7 +++---- 9 files changed, 26 insertions(+), 34 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 2aae8a0aa..8c6a08e1b 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -4,7 +4,6 @@ skips: - B105 - B301 - B303 -- B306 - B307 - B311 - B320 diff --git a/tests/test_commands.py b/tests/test_commands.py index 36f800850..2f36baa87 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,13 +6,12 @@ import platform import re import subprocess import sys -import tempfile from contextlib import contextmanager from itertools import chain from pathlib import Path from shutil import copytree, rmtree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import mkdtemp +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import Dict, Generator, Optional, Union from unittest import skipIf @@ -82,7 +81,7 @@ class ProjectTest(unittest.TestCase): rmtree(self.temp_path) def call(self, *new_args, **kwargs): - with tempfile.TemporaryFile() as out: + with TemporaryFile() as out: args = (sys.executable, "-m", "scrapy.cmdline") + new_args return subprocess.call( args, stdout=out, stderr=out, cwd=self.cwd, env=self.env, **kwargs diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index dd07d33f1..d3fd63847 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,8 +2,8 @@ import contextlib import os import shutil import sys -import tempfile from pathlib import Path +from tempfile import mkdtemp, mkstemp from typing import Optional, Type from unittest import SkipTest, mock @@ -107,13 +107,14 @@ class LoadTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly - self.tmpname = Path(self.mktemp() + "^") + self.fd, self.tmpname = mkstemp(suffix="^") Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = build_from_crawler(FileDownloadHandler, get_crawler()) self.download_request = handler.download_request def tearDown(self): - self.tmpname.unlink() + os.close(self.fd) + os.remove(self.tmpname) def test_download(self): def _test(response): @@ -122,12 +123,12 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.body, b"0123456789") self.assertEqual(response.protocol, None) - request = Request(path_to_file_uri(str(self.tmpname))) + request = Request(path_to_file_uri(self.tmpname)) assert request.url.upper().endswith("%5E") return self.download_request(request, Spider("foo")).addCallback(_test) def test_non_existent(self): - request = Request(path_to_file_uri(self.mktemp())) + request = Request(path_to_file_uri(mkdtemp())) d = self.download_request(request, Spider("foo")) return self.assertFailure(d, OSError) @@ -224,8 +225,7 @@ class HttpTestCase(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -651,8 +651,7 @@ class Https11CustomCiphers(unittest.TestCase): certfile = "keys/localhost.crt" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) @@ -1015,8 +1014,7 @@ class BaseFTPTestCase(unittest.TestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() + self.directory = Path(mkdtemp()) userdir = self.directory / self.username userdir.mkdir() for filename, content in self.test_files: @@ -1092,7 +1090,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() fname_bytes = to_bytes(local_fname) local_fname = Path(local_fname) os.close(f) @@ -1113,7 +1111,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def _test_response_class(self, filename, response_class): - f, local_fname = tempfile.mkstemp() + f, local_fname = mkstemp() local_fname = Path(local_fname) os.close(f) meta = {} @@ -1163,9 +1161,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file - self.directory = Path(self.mktemp()) - self.directory.mkdir() - + self.directory = Path(mkdtemp()) for filename, content in self.test_files: (self.directory / filename).write_bytes(content) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 8fdf3d56f..995c02a1a 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -5,6 +5,7 @@ import shutil import string from ipaddress import IPv4Address from pathlib import Path +from tempfile import mkdtemp from typing import Dict from unittest import mock, skipIf from urllib.parse import urlencode @@ -185,8 +186,7 @@ class Https2ClientProtocolTestCase(TestCase): certificate_file = Path(__file__).parent / "keys" / "localhost.crt" def _init_resource(self): - self.temp_directory = self.mktemp() - Path(self.temp_directory).mkdir() + self.temp_directory = mkdtemp() r = File(self.temp_directory) r.putChild(b"get-data-html-small", GetDataHtmlSmall()) r.putChild(b"get-data-html-large", GetDataHtmlLarge()) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index c41ab483f..be9811980 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,6 @@ import shutil from pathlib import Path +from tempfile import mkdtemp from typing import Optional, Set from testfixtures import LogCapture @@ -67,8 +68,7 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.__enter__() # prepare a directory for storing files - self.tmpmediastore = Path(self.mktemp()) - self.tmpmediastore.mkdir() + self.tmpmediastore = Path(mkdtemp()) self.settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", "ITEM_PIPELINES": {self.pipeline_class: 1}, diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 04025d30d..f950739f2 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import sys import tempfile import warnings from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -139,8 +140,7 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = module_dir / "test_spiders" - self.tmpdir = Path(self.mktemp()) - self.tmpdir.mkdir() + self.tmpdir = Path(mkdtemp()) self.spiders_dir = self.tmpdir / "test_spiders_xxx" _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(str(self.tmpdir)) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index f97125b76..59d18d92e 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,6 +1,6 @@ import shutil from datetime import datetime, timezone -from pathlib import Path +from tempfile import mkdtemp from twisted.trial import unittest @@ -12,8 +12,7 @@ from scrapy.utils.test import get_crawler class SpiderStateTest(unittest.TestCase): def test_store_load(self): - jobdir = self.mktemp() - Path(jobdir).mkdir() + jobdir = mkdtemp() try: spider = Spider(name="default") dt = datetime.now(tz=timezone.utc) diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index b444c32b7..499ca46b8 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -25,7 +25,7 @@ class BaseQueueTestCase(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix="scrapy-queue-tests-") self.qpath = self.tempfilename() - self.qdir = self.mkdtemp() + self.qdir = tempfile.mkdtemp() self.crawler = get_crawler(Spider) def tearDown(self): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index d4b6ba15b..53558814d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,6 +4,7 @@ Tests borrowed from the twisted.web.client tests. """ import shutil from pathlib import Path +from tempfile import mkdtemp import OpenSSL.SSL from twisted.internet import defer, reactor @@ -274,8 +275,7 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) @@ -440,8 +440,7 @@ class WebClientSSLTestCase(unittest.TestCase): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = Path(self.mktemp()) - self.tmpname.mkdir() + self.tmpname = Path(mkdtemp()) (self.tmpname / "file").write_bytes(b"0123456789") r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) From 2d46b4acf5855faaf2d6baa36615f08bd8aefccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Feb 2024 09:28:02 +0100 Subject: [PATCH 10/45] Complete coverage for the AutoThrottle extension (#6245) --- docs/topics/autothrottle.rst | 2 +- scrapy/extensions/throttle.py | 5 + tests/test_extension_throttle.py | 340 +++++++++++++++++++++++++++++++ 3 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 tests/test_extension_throttle.py diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 8e6aae65c..5370d77b3 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -131,7 +131,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY Default: ``1.0`` Average number of requests Scrapy should be sending in parallel to remote -websites. +websites. It must be higher than ``0.0``. By default, AutoThrottle adjusts the delay to send a single concurrent request to each of the remote websites. Set this option to diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 396800775..d217c7a69 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -16,6 +16,11 @@ class AutoThrottle: self.target_concurrency = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) + if self.target_concurrency <= 0.0: + raise NotConfigured( + f"AUTOTHROTTLE_TARGET_CONCURRENCY " + f"({self.target_concurrency!r}) must be higher than 0." + ) crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py new file mode 100644 index 000000000..dae4ea966 --- /dev/null +++ b/tests/test_extension_throttle.py @@ -0,0 +1,340 @@ +from logging import INFO +from unittest.mock import Mock + +import pytest + +from scrapy import Request, Spider +from scrapy.exceptions import NotConfigured +from scrapy.extensions.throttle import AutoThrottle +from scrapy.http.response import Response +from scrapy.settings.default_settings import ( + AUTOTHROTTLE_MAX_DELAY, + AUTOTHROTTLE_START_DELAY, + DOWNLOAD_DELAY, +) +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler as _get_crawler + +UNSET = object() + + +class TestSpider(Spider): + name = "test" + + +def get_crawler(settings=None, spidercls=None): + settings = settings or {} + settings["AUTOTHROTTLE_ENABLED"] = True + return _get_crawler(settings_dict=settings, spidercls=spidercls) + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, False), + (False, False), + (True, True), + ), +) +def test_enabled(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_ENABLED"] = value + crawler = _get_crawler(settings_dict=settings) + if expected: + build_from_crawler(AutoThrottle, crawler) + else: + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + "value", + ( + 0.0, + -1.0, + ), +) +def test_target_concurrency_invalid(value): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": value} + crawler = get_crawler(settings) + with pytest.raises(NotConfigured): + build_from_crawler(AutoThrottle, crawler) + + +@pytest.mark.parametrize( + ("spider", "setting", "expected"), + ( + (UNSET, UNSET, DOWNLOAD_DELAY), + (1.0, UNSET, 1.0), + (UNSET, 1.0, 1.0), + (1.0, 2.0, 1.0), + (3.0, 2.0, 3.0), + ), +) +def test_mindelay_definition(spider, setting, expected): + settings = {} + if setting is not UNSET: + settings["DOWNLOAD_DELAY"] = setting + + class _TestSpider(Spider): + name = "test" + + if spider is not UNSET: + _TestSpider.download_delay = spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(_TestSpider()) + assert at.mindelay == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + (UNSET, AUTOTHROTTLE_MAX_DELAY), + (1.0, 1.0), + ), +) +def test_maxdelay_definition(value, expected): + settings = {} + if value is not UNSET: + settings["AUTOTHROTTLE_MAX_DELAY"] = value + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + at._spider_opened(TestSpider()) + assert at.maxdelay == expected + + +@pytest.mark.parametrize( + ("min_spider", "min_setting", "start_setting", "expected"), + ( + (UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY), + (AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY), + (UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0), + (UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 2.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 1.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ( + AUTOTHROTTLE_START_DELAY + 1.0, + UNSET, + AUTOTHROTTLE_START_DELAY + 2.0, + AUTOTHROTTLE_START_DELAY + 2.0, + ), + ), +) +def test_startdelay_definition(min_spider, min_setting, start_setting, expected): + settings = {} + if min_setting is not UNSET: + settings["DOWNLOAD_DELAY"] = min_setting + if start_setting is not UNSET: + settings["AUTOTHROTTLE_START_DELAY"] = start_setting + + class _TestSpider(Spider): + name = "test" + + if min_spider is not UNSET: + _TestSpider.download_delay = min_spider + + crawler = get_crawler(settings, _TestSpider) + at = build_from_crawler(AutoThrottle, crawler) + spider = _TestSpider() + at._spider_opened(spider) + assert spider.download_delay == expected + + +@pytest.mark.parametrize( + ("meta", "slot"), + ( + ({}, None), + ({"download_latency": 1.0}, None), + ({"download_slot": "foo"}, None), + ({"download_slot": "foo"}, "foo"), + ({"download_latency": 1.0, "download_slot": "foo"}, None), + ), +) +def test_skipped(meta, slot): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + request = Request("https://example.com", meta=meta) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + if slot is not None: + crawler.engine.downloader.slots[slot] = object() + at._adjust_delay = None # Raise exception if called. + + at._response_downloaded(None, request, spider) + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 0.75), + (4.0, 2.0, 1.0, 2.0), + (2.0, 1.0, 1.0, 2.0), + (2.0, 4.0, 1.0, 0.75), + (2.0, 2.0, 0.5, 1.0), + (2.0, 2.0, 2.0, 1.5), + ), +) +def test_adjustment(download_latency, target_concurrency, slot_delay, expected): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("mindelay", "maxdelay", "expected"), + ( + (0.5, 2.0, 1.0), + (0.25, 0.5, 0.5), + (2.0, 4.0, 2.0), + ), +) +def test_adjustment_limits(mindelay, maxdelay, expected): + download_latency, target_concurrency, slot_delay = (2.0, 2.0, 1.0) + # expected adjustment without limits with these values: 1.0 + settings = { + "AUTOTHROTTLE_MAX_DELAY": maxdelay, + "AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency, + "DOWNLOAD_DELAY": mindelay, + } + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +@pytest.mark.parametrize( + ("download_latency", "target_concurrency", "slot_delay", "expected"), + ( + (2.0, 2.0, 1.0, 1.0), + (1.0, 2.0, 1.0, 1.0), # Instead of 0.75 + (4.0, 2.0, 1.0, 2.0), + ), +) +def test_adjustment_bad_response( + download_latency, target_concurrency, slot_delay, expected +): + settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": download_latency, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, status=400) + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = slot_delay + crawler.engine.downloader.slots["foo"] = slot + + at._response_downloaded(response, request, spider) + + assert slot.delay == expected, f"{slot.delay} != {expected}" + + +def test_debug(caplog): + settings = {"AUTOTHROTTLE_DEBUG": True} + crawler = get_crawler(settings) + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [ + ( + "scrapy.extensions.throttle", + INFO, + "slot: foo | conc: 2 | delay: 1500 ms (-500) | latency: 1000 ms | size: 3 bytes", + ), + ] + + +def test_debug_disabled(caplog): + crawler = get_crawler() + at = build_from_crawler(AutoThrottle, crawler) + spider = TestSpider() + at._spider_opened(spider) + meta = {"download_latency": 1.0, "download_slot": "foo"} + request = Request("https://example.com", meta=meta) + response = Response(request.url, body=b"foo") + + crawler.engine = Mock() + crawler.engine.downloader = Mock() + crawler.engine.downloader.slots = {} + slot = Mock() + slot.delay = 2.0 + slot.transferring = (None, None) + crawler.engine.downloader.slots["foo"] = slot + + caplog.clear() + with caplog.at_level(INFO): + at._response_downloaded(response, request, spider) + + assert caplog.record_tuples == [] From d87f949526470fc4847c99f58e1dcc40d4e9ed00 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 17:08:13 -0300 Subject: [PATCH 11/45] Use defusedxml.xmlrpc --- scrapy/http/request/rpc.py | 4 ++++ setup.py | 1 + 2 files changed, 5 insertions(+) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bde860a66..2bf5ba4b6 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,9 +7,13 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional +from defusedxml import xmlrpc + from scrapy.http.request import Request from scrapy.utils.python import get_func_args +xmlrpc.monkey_patch() + DUMPS_ARGS = get_func_args(xmlrpclib.dumps) diff --git a/setup.py b/setup.py index 405633f55..2d6d26b0c 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ install_requires = [ "packaging", "tldextract", "lxml>=4.4.1", + "defusedxml>=0.7.1", ] extras_require = { ':platform_python_implementation == "CPython"': ["PyDispatcher>=2.0.5"], From 7f945ad6db728234987629b2299750abee5c1781 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 17:39:29 -0300 Subject: [PATCH 12/45] Import defusedxml.xmlrpc using alias --- scrapy/http/request/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 2bf5ba4b6..5a2107f76 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,12 +7,12 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional -from defusedxml import xmlrpc +import defusedxml.xmlrpc as xml_rpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args -xmlrpc.monkey_patch() +xml_rpc.monkey_patch() DUMPS_ARGS = get_func_args(xmlrpclib.dumps) From 008ebb65fc2f0e7cfe9fa43d7ac938a94b3098fb Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 27 Feb 2024 18:10:28 -0300 Subject: [PATCH 13/45] Change immport style --- scrapy/http/request/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 5a2107f76..84b433990 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -7,12 +7,12 @@ See documentation in docs/topics/request-response.rst import xmlrpc.client as xmlrpclib from typing import Any, Optional -import defusedxml.xmlrpc as xml_rpc +import defusedxml.xmlrpc from scrapy.http.request import Request from scrapy.utils.python import get_func_args -xml_rpc.monkey_patch() +defusedxml.xmlrpc.monkey_patch() DUMPS_ARGS = get_func_args(xmlrpclib.dumps) From 282767f23b2e71969bea3bd5492abde88d2054c6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:49:06 +0500 Subject: [PATCH 14/45] Bump black. --- .flake8 | 2 +- .pre-commit-config.yaml | 4 ++-- docs/topics/addons.rst | 3 +-- scrapy/commands/__init__.py | 1 + scrapy/commands/shell.py | 1 + scrapy/core/downloader/handlers/__init__.py | 6 +++--- scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/middleware.py | 1 + scrapy/core/engine.py | 1 + scrapy/core/http2/stream.py | 12 ++++++------ scrapy/core/scraper.py | 1 + scrapy/core/spidermw.py | 1 + scrapy/downloadermiddlewares/defaultheaders.py | 1 + scrapy/downloadermiddlewares/downloadtimeout.py | 1 + scrapy/downloadermiddlewares/retry.py | 1 + scrapy/exceptions.py | 1 + scrapy/extension.py | 1 + scrapy/extensions/corestats.py | 1 + scrapy/extensions/memusage.py | 1 + scrapy/extensions/postprocessing.py | 1 + scrapy/http/request/__init__.py | 17 +++++++++++------ scrapy/http/request/rpc.py | 1 + scrapy/http/response/__init__.py | 1 + scrapy/http/response/text.py | 1 + scrapy/link.py | 1 + scrapy/linkextractors/__init__.py | 1 + scrapy/linkextractors/lxmlhtml.py | 1 + scrapy/loader/__init__.py | 1 + scrapy/mail.py | 1 + scrapy/pipelines/__init__.py | 1 + scrapy/pipelines/files.py | 1 + scrapy/pipelines/images.py | 1 + scrapy/responsetypes.py | 1 + scrapy/selector/unified.py | 1 + scrapy/settings/__init__.py | 1 - scrapy/shell.py | 1 + scrapy/spidermiddlewares/httperror.py | 1 + scrapy/spidermiddlewares/offsite.py | 1 + scrapy/spidermiddlewares/referer.py | 1 + scrapy/spiders/__init__.py | 1 + scrapy/spiders/feed.py | 1 + scrapy/statscollectors.py | 1 + scrapy/utils/defer.py | 7 +++---- scrapy/utils/deprecate.py | 6 ++---- scrapy/utils/iterators.py | 11 +++++------ scrapy/utils/misc.py | 1 + scrapy/utils/python.py | 7 +++---- scrapy/utils/request.py | 7 ++++--- scrapy/utils/response.py | 7 ++++--- scrapy/utils/signal.py | 1 + scrapy/utils/sitemap.py | 1 + scrapy/utils/spider.py | 15 +++++---------- scrapy/utils/url.py | 1 + tests/mocks/dummydbm.py | 1 + tests/spiders.py | 1 + tests/test_exporters.py | 4 +++- tests/test_pipeline_media.py | 6 +++--- tests/test_responsetypes.py | 5 ++++- tests/test_webclient.py | 1 + 59 files changed, 102 insertions(+), 60 deletions(-) diff --git a/.flake8 b/.flake8 index 544d72956..62ccad9cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,7 @@ [flake8] max-line-length = 119 -ignore = W503, E203 +ignore = E203, E501, E701, E704, W503 exclude = docs/conf.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0cff5cc73..83bc65b67 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==24.2.0 diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 1bf2172bd..d2fc41003 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -150,8 +150,7 @@ Access the crawler instance: def from_crawler(cls, crawler): return cls(crawler) - def update_settings(self, settings): - ... + def update_settings(self, settings): ... Use a fallback component: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2aa569cdd..27993710e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -1,6 +1,7 @@ """ Base class for Scrapy commands """ + import argparse import os from pathlib import Path diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 12e37babc..f72a23c6a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -3,6 +3,7 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ + from argparse import Namespace from threading import Thread from typing import List, Type diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 416669b7f..ade51ca63 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -21,9 +21,9 @@ logger = logging.getLogger(__name__) class DownloadHandlers: def __init__(self, crawler: "Crawler"): self._crawler: "Crawler" = crawler - self._schemes: Dict[ - str, Union[str, Callable] - ] = {} # stores acceptable schemes on instancing + self._schemes: Dict[str, Union[str, Callable]] = ( + {} + ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers handlers: Dict[str, Union[str, Callable]] = without_none_values( diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index b6ac7a251..d168c2b2e 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,5 +1,6 @@ """Download handlers for http and https schemes """ + from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index dca13c01e..52ebe4e22 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,6 +3,7 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ + from typing import Any, Callable, Generator, List, Union, cast from twisted.internet.defer import Deferred, inlineCallbacks diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 545cd401f..2db085081 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider. For more information see docs/topics/architecture.rst """ + import logging from time import time from typing import ( diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 0f282d83d..4132fc385 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -111,17 +111,17 @@ class Stream: # Metadata of an HTTP/2 connection stream # initialized when stream is instantiated self.metadata: Dict = { - "request_content_length": 0 - if self._request.body is None - else len(self._request.body), + "request_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether the stream has initiated the request "request_sent": False, # Flag to track whether we have logged about exceeding download warnsize "reached_warnsize": False, # Each time we send a data frame, we will decrease value by the amount send. - "remaining_content_length": 0 - if self._request.body is None - else len(self._request.body), + "remaining_content_length": ( + 0 if self._request.body is None else len(self._request.body) + ), # Flag to keep track whether client (self) have closed this stream "stream_closed_local": False, # Flag to keep track whether the server has closed the stream diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8fb16b8a9..272841e01 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,5 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" + from __future__ import annotations import logging diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 031a0be36..1ccfd08a2 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ + import logging from inspect import isasyncgenfunction, iscoroutine from itertools import islice diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 8aec37cf1..58fd415b9 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -3,6 +3,7 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Iterable, Tuple, Union diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 1c904c05b..fd7c03a38 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -3,6 +3,7 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ + from __future__ import annotations from typing import TYPE_CHECKING, Union diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 3c494de78..46587a898 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ + from __future__ import annotations import warnings diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 6d188c489..e7ecdbe0c 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -4,6 +4,7 @@ Scrapy core exceptions These exceptions are documented in docs/topics/exceptions.rst. Please don't add new exceptions here without documenting them there. """ + from typing import Any # Internal diff --git a/scrapy/extension.py b/scrapy/extension.py index 4e365cfa1..6be14450c 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -3,6 +3,7 @@ The Extension Manager See documentation in docs/topics/extensions.rst """ + from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 302a615f2..717c249d9 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -1,6 +1,7 @@ """ Extension for collecting core stats like items scraped and start/finish times """ + from datetime import datetime, timezone from scrapy import signals diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ca766c938..4d4501c44 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -3,6 +3,7 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ + import logging import socket import sys diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index 17969c5b0..f8b59827b 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -1,6 +1,7 @@ """ Extension for processing data before they are exported to feeds. """ + from bz2 import BZ2File from gzip import GzipFile from io import IOBase diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..6269ee86a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,6 +4,7 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ + import inspect from typing import ( Any, @@ -231,12 +232,16 @@ class Request(object_ref): """ d = { "url": self.url, # urls are safe (safe_string_url) - "callback": _find_method(spider, self.callback) - if callable(self.callback) - else self.callback, - "errback": _find_method(spider, self.errback) - if callable(self.errback) - else self.errback, + "callback": ( + _find_method(spider, self.callback) + if callable(self.callback) + else self.callback + ), + "errback": ( + _find_method(spider, self.errback) + if callable(self.errback) + else self.errback + ), "headers": dict(self.headers), } for attr in self.attributes: diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 84b433990..e20e7c438 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,6 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ + import xmlrpc.client as xmlrpclib from typing import Any, Optional diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..d73dfce4b 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,6 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations from ipaddress import IPv4Address, IPv6Address diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6596d8a5c..2816610fb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ + from __future__ import annotations import json diff --git a/scrapy/link.py b/scrapy/link.py index 0868ae5ef..4bdbc1823 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + from typing import Any diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3774430a7..73a63651c 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -5,6 +5,7 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ + import re # common file extensions that are not followed if they occur in links diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 23cbd0116..d76db20ba 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,7 @@ """ Link extractor based on lxml.html """ + import logging import operator from functools import partial diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 1042a3d48..529fa279e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,6 +3,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ + import itemloaders from scrapy.item import Item diff --git a/scrapy/mail.py b/scrapy/mail.py index 237327451..4b18b6003 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,6 +3,7 @@ Mail sending helpers See documentation in docs/topics/email.rst """ + import logging from email import encoders as Encoders from email.mime.base import MIMEBase diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index c97d71fb6..f9544d329 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -3,6 +3,7 @@ Item pipeline See documentation in docs/item-pipeline.rst """ + from typing import Any, List from twisted.internet.defer import Deferred diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 73064ad10..1d7625299 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,6 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ + import base64 import functools import hashlib diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 02c4b1361..8169583f8 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -3,6 +3,7 @@ Images Pipeline See documentation in topics/media-pipeline.rst """ + import functools import hashlib import warnings diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 0d127d851..702e50536 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,6 +2,7 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ + from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 863fb6032..75d5e9fbd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,6 +1,7 @@ """ XPath selectors based on lxml """ + from typing import Any, Optional, Type, Union from parsel import Selector as _ParselSelector diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b5d8fdb12..d270a72f4 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int: class SettingsAttribute: - """Class for storing data related to settings attributes. This class is intended for internal usage, you should try Settings class diff --git a/scrapy/shell.py b/scrapy/shell.py index bb3b1461c..05909977a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,6 +3,7 @@ See documentation in docs/topics/shell.rst """ + import os import signal diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94450b35b..35c869a75 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -3,6 +3,7 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a5214702d..dd2fccfcb 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -3,6 +3,7 @@ Offsite Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a29e0ebb5..a0b6851e5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ + from __future__ import annotations import warnings diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e16d71727..72c2aaba7 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + from __future__ import annotations import logging diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 599af7360..5caf8c79e 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -4,6 +4,7 @@ for scraping from an XML feed. See documentation in docs/topics/spiders.rst """ + from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 15193aac5..ab571a3ab 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -1,6 +1,7 @@ """ Scrapy extension for collecting scraping stats """ + import logging import pprint from typing import TYPE_CHECKING, Any, Dict, Optional diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bf3c5ef5b..c391db9fd 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -1,6 +1,7 @@ """ Helper functions for dealing with Twisted deferreds """ + import asyncio import inspect from asyncio import Future @@ -304,13 +305,11 @@ _T = TypeVar("_T") @overload -def deferred_from_coro(o: _CT) -> Deferred: - ... +def deferred_from_coro(o: _CT) -> Deferred: ... @overload -def deferred_from_coro(o: _T) -> _T: - ... +def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Union[Deferred, _T]: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ea577c44a..e0f2ac763 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = [] @overload -def update_classpath(path: str) -> str: - ... +def update_classpath(path: str) -> str: ... @overload -def update_classpath(path: Any) -> Any: - ... +def update_classpath(path: Any) -> Any: ... def update_classpath(path: Any) -> Any: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c56be5ea2..93a2ba7a1 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -225,18 +225,17 @@ def csviter( @overload -def _body_or_str(obj: Union[Response, str, bytes]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: - ... +def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ... @overload -def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes: - ... +def _body_or_str( + obj: Union[Response, str, bytes], unicode: Literal[False] +) -> bytes: ... def _body_or_str( diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b38190cb3..7b43760a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,5 @@ """Helper functions which don't fit anywhere else""" + import ast import hashlib import inspect diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 68ca96b69..7b408c49c 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ + import collections.abc import gc import inspect @@ -285,13 +286,11 @@ def equal_attributes( @overload -def without_none_values(iterable: Mapping) -> dict: - ... +def without_none_values(iterable: Mapping) -> dict: ... @overload -def without_none_values(iterable: Iterable) -> Iterable: - ... +def without_none_values(iterable: Iterable) -> Iterable: ... def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index db0b44cf4..e99d1eeb5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,7 +44,9 @@ def _serialize_headers( yield from request.headers.getlist(header) -_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +_fingerprint_cache: ( + "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]" +) _fingerprint_cache = WeakKeyDictionary() @@ -114,8 +116,7 @@ def fingerprint( class RequestFingerprinterProtocol(Protocol): - def fingerprint(self, request: Request) -> bytes: - ... + def fingerprint(self, request: Request) -> bytes: ... class RequestFingerprinter: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 33cd692bf..63a484b42 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -2,6 +2,7 @@ This module provides some useful functions for working with scrapy.http.Response objects """ + import os import re import tempfile @@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = ( - WeakKeyDictionary() -) +_metaref_cache: ( + "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" +) = WeakKeyDictionary() def get_meta_refresh( diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 21a12a19e..a25100c03 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,4 +1,5 @@ """Helper functions for working with signals""" + import collections.abc import logging from typing import Any as TypingAny diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 3d2ecc9a7..8bf941eb2 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,6 +4,7 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ + from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 704df8657..855bc8f87 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -39,13 +39,11 @@ def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ig @overload -def iterate_spider_output(result: CoroutineType) -> Deferred: - ... +def iterate_spider_output(result: CoroutineType) -> Deferred: ... @overload -def iterate_spider_output(result: _T) -> Iterable: - ... +def iterate_spider_output(result: _T) -> Iterable: ... def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]: @@ -83,8 +81,7 @@ def spidercls_for_request( default_spidercls: Type[Spider], log_none: bool = ..., log_multiple: bool = ..., -) -> Type[Spider]: - ... +) -> Type[Spider]: ... @overload @@ -94,8 +91,7 @@ def spidercls_for_request( default_spidercls: Literal[None], log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... @overload @@ -105,8 +101,7 @@ def spidercls_for_request( *, log_none: bool = ..., log_multiple: bool = ..., -) -> Optional[Type[Spider]]: - ... +) -> Optional[Type[Spider]]: ... def spidercls_for_request( diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 22b4197f9..9d97cb12f 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,6 +5,7 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ + import re from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 2869ff8f7..bde3de228 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -1,4 +1,5 @@ """DBM-like dummy module""" + import collections from typing import Any, DefaultDict diff --git a/tests/spiders.py b/tests/spiders.py index 3df153a12..94969db99 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,6 +1,7 @@ """ Some spiders used for testing and benchmarking """ + import asyncio import time from urllib.parse import urlencode diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c11913365..59b724495 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, "John\xa3") ie = self._get_exporter(fields_to_export={"name": "名稱"}) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")]) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")] + ) def test_field_custom_serializer(self): i = self.custom_field_item_class(name="John\xa3", age="22") diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 820484565..d477b59be 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler try: from PIL import Image # noqa: imported just to check for the import error except ImportError: - skip_pillow: Optional[ - str - ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + skip_pillow: Optional[str] = ( + "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow" + ) else: skip_pillow = None diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 713a83d52..2633cca5b 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -33,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase): ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response), ("attachment;filename=data高.doc".encode("gbk"), Response), ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse), - ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse), + ( + "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), + XmlResponse, + ), ] for source, cls in mappings: retcls = responsetypes.from_content_disposition(source) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 53558814d..cce119001 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,6 +2,7 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ + import shutil from pathlib import Path from tempfile import mkdtemp From 6e5918345b8eb10674e11d9c4c5db8c9028be674 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 19:50:54 +0500 Subject: [PATCH 15/45] Bump bandit, flake8 and isort. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83bc65b67..a911d4cfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: [-r, -c, .bandit.yml] - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 - repo: https://github.com/psf/black.git @@ -13,7 +13,7 @@ repos: hooks: - id: black - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/adamchainz/blacken-docs From 68104b9f48802d1ecc1c892c61aaa197500435b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 20:21:27 +0500 Subject: [PATCH 16/45] Update .bandit.yml, add problem names. --- .bandit.yml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 8c6a08e1b..6e8331c0f 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,20 +1,19 @@ skips: -- B101 -- B113 # https://github.com/PyCQA/bandit/issues/1010 -- B105 -- B301 -- B303 -- B307 -- B311 -- B320 -- B321 -- B324 -- B402 # https://github.com/scrapy/scrapy/issues/4180 -- B403 -- B404 -- B406 -- B410 -- B503 -- B603 -- B605 +- B101 # assert_used +- B105 # hardcoded_password_string +- B301 # pickle +- B307 # eval +- B311 # random +- B320 # xml_bad_etree +- B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B324 # hashlib "Use of weak SHA1 hash for security" +- B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 +- B403 # import_pickle +- B404 # import_subprocess +- B406 # import_xml_sax +- B410 # import_lxml +- B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 +- B503 # ssl_with_bad_defaults +- B603 # subprocess_without_shell_equals_true +- B605 # start_process_with_a_shell exclude_dirs: ['tests'] From d2c05d9d96394e111ead1aa097402cdbc18c0859 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 20:43:57 +0500 Subject: [PATCH 17/45] Bump mypy and type stubs. --- scrapy/utils/spider.py | 3 +-- scrapy/utils/ssl.py | 2 +- tox.ini | 14 +++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 855bc8f87..cbbb01d85 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -34,8 +34,7 @@ _T = TypeVar("_T") # https://stackoverflow.com/questions/60222982 @overload -def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: # type: ignore[misc] - ... +def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ... # type: ignore[overload-overlap] @overload diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index e74769c65..d520ef809 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,6 +1,6 @@ from typing import Any, Optional -import OpenSSL._util as pyOpenSSLutil # type: ignore[import-untyped] +import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version from OpenSSL.crypto import X509Name diff --git a/tox.ini b/tox.ini index 359ff0f73..e787c7bf3 100644 --- a/tox.ini +++ b/tox.ini @@ -29,14 +29,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.8.0 + typing-extensions==4.10.0 types-attrs==19.1.0 - types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 - types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-lxml==2024.2.9 + types-Pillow==10.2.0.20240213 + types-Pygments==2.17.0.20240106 + types-pyOpenSSL==24.0.0.20240130 + types-setuptools==69.1.0.20240223 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From 4f9dd998dcf01c003bc2a053b6bd78091d12ecd5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 27 Feb 2024 22:01:36 +0500 Subject: [PATCH 18/45] Bump pylint, cleanup the ignored tags. --- docs/conf.py | 2 +- pylintrc | 17 +----------- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/http/headers.py | 4 ++- scrapy/utils/ossignal.py | 6 ++++- scrapy/utils/signal.py | 5 +++- tests/test_commands.py | 27 ------------------- tests/test_item.py | 4 ++- tests/test_linkextractors.py | 3 --- tests/test_loader_deprecated.py | 6 ++--- tests/test_settings/__init__.py | 2 +- tests/test_utils_datatypes.py | 2 +- tests/test_utils_signal.py | 6 ----- tests/test_utils_spider.py | 2 +- tox.ini | 2 +- 15 files changed, 25 insertions(+), 65 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9ca0f817a..399078010 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,7 +227,7 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", "http://directory.google.com/", ] diff --git a/pylintrc b/pylintrc index c8654b8d3..78004e78a 100644 --- a/pylintrc +++ b/pylintrc @@ -4,21 +4,14 @@ jobs=1 # >1 hides results [MESSAGES CONTROL] disable=abstract-method, - anomalous-backslash-in-string, arguments-differ, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-mcs-classmethod-argument, bare-except, broad-except, broad-exception-raised, c-extension-no-member, - catching-non-exception, - cell-var-from-loop, - comparison-with-callable, - consider-using-dict-items, - consider-using-in, consider-using-with, cyclic-import, dangerous-default-value, @@ -32,7 +25,6 @@ disable=abstract-method, implicit-str-concat, import-error, import-outside-toplevel, - import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -44,7 +36,6 @@ disable=abstract-method, logging-fstring-interpolation, logging-not-lazy, lost-exception, - method-hidden, missing-docstring, no-else-raise, no-else-return, @@ -52,7 +43,7 @@ disable=abstract-method, no-method-argument, no-name-in-module, no-self-argument, - no-value-for-parameter, + no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, pointless-exception-statement, pointless-statement, @@ -77,14 +68,10 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - undefined-variable, - undefined-loop-variable, - unexpected-special-method-signature, unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, - unsubscriptable-object, unused-argument, unused-import, unused-private-member, @@ -92,8 +79,6 @@ disable=abstract-method, unused-wildcard-import, use-dict-literal, used-before-assignment, - useless-object-inheritance, # Required for Python 2 support useless-return, - useless-super-delegation, wildcard-import, wrong-import-position diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 1e340abb6..f0ad24f72 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -169,7 +169,7 @@ class HttpCompressionMiddleware: return to_decode, to_keep def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: - if encoding == b"gzip" or encoding == b"x-gzip": + if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 21eb9fb73..73aee7178 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -113,7 +113,9 @@ class Headers(CaselessDict): return ((k, self.getlist(k)) for k in self.keys()) def values(self) -> List[Optional[bytes]]: # type: ignore[override] - return [self[k] for k in self.keys()] + return [ + self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + ] def to_string(self) -> bytes: # cast() can be removed if the headers_dict_to_raw() hint is improved diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index db9a71273..5985a847e 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -24,7 +24,11 @@ def install_shutdown_handlers( (e.g. Pdb) """ signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: + if ( + signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable + == signal.default_int_handler + or override_sigint + ): signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, "SIGBREAK"): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index a25100c03..89cfbd2ec 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -98,7 +98,10 @@ def send_catch_log_deferred( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) - d.addBoth(lambda result: (receiver, result)) + # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d.addBoth( + lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + ) dfds.append(d) d = DeferredList(dfds) d.addCallback(lambda out: [x[1] for x in out]) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2f36baa87..febad21da 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,38 +991,11 @@ class MySpider(scrapy.Spider): class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" - def setUp(self): - super().setUp() - def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - def test_run_good_spider(self): - super().test_run_good_spider() - - def test_runspider(self): - super().test_runspider() - - def test_runspider_dnscache_disabled(self): - super().test_runspider_dnscache_disabled() - - def test_runspider_log_level(self): - super().test_runspider_log_level() - - def test_runspider_log_short_names(self): - super().test_runspider_log_short_names() - - def test_runspider_no_spider_found(self): - super().test_runspider_no_spider_found() - - def test_output(self): - super().test_output() - - def test_overwrite_output(self): - super().test_overwrite_output() - def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_item.py b/tests/test_item.py index ce2b4fd15..daf5d4f59 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__(self, *args, **kwargs): + def __init__( + self, *args, **kwargs + ): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 55ea9eed2..6b4df90d8 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - def test_restrict_xpaths_with_html_entities(self): - super().test_restrict_xpaths_with_html_entities() - @mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d7f773d5c..99cdf88d9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase): } def test_output(self): - for tl in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[tl] + for k, v in self.test_list_equals.items(): + expr, test_list, expected = v test = SelectJmes(expr)(test_list) self.assertEqual( - test, expected, msg=f'test "{tl}" got {test} expected {expected}' + test, expected, msg=f'test "{k}" got {test} expected {expected}' ) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3fde5e8c5..9ee248538 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase): mydict = settings.get("TEST_DICT") self.assertIsInstance(mydict, BaseSettings) self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object self.assertEqual(mydict.getpriority("key"), 0) @mock.patch("scrapy.settings.default_settings", default_settings) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 9e5f88f48..be5c6de81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for i, r in enumerate(refs): self.assertIn(r, cache) self.assertEqual(cache[r], i) - del r # delete reference to the last object in the list + del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache for _ in range(max // 2): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 65b99e0c4..60232f10b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" - def test_send_catch_log(self): - return super().test_send_catch_log() - @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") - def test_send_catch_log(self): - return super().test_send_catch_log() - class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 460ae40c3..dd1d26448 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import tests.test_utils_spider + import tests.test_utils_spider # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tox.ini b/tox.ini index e787c7bf3..4ed9b3bd7 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.0.1 + pylint==3.1.0 commands = pylint conftest.py docs extras scrapy setup.py tests From 63acd0720970c87450fdbcb9aa6967118c9c1cf2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 28 Feb 2024 16:14:08 -0300 Subject: [PATCH 19/45] Fix and re-enable unnecessary-comprehension and use-dict-literal pylint tags --- pylintrc | 2 -- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/utils/python.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pylintrc b/pylintrc index 78004e78a..c60e4e16a 100644 --- a/pylintrc +++ b/pylintrc @@ -68,7 +68,6 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, @@ -77,7 +76,6 @@ disable=abstract-method, unused-private-member, unused-variable, unused-wildcard-import, - use-dict-literal, used-before-assignment, useless-return, wildcard-import, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f0ad24f72..aa3abe853 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -135,7 +135,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs = {"cls": respcls, "body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ebb4f5984..2a3913da5 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -85,7 +85,7 @@ class CrawlSpider(Spider): url=link.url, callback=self._callback, errback=self._errback, - meta=dict(rule=rule_index, link_text=link.text), + meta={"rule": rule_index, "link_text": link.text}, ) def _requests_to_follow(self, response): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7b408c49c..1e7364e49 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -162,7 +162,7 @@ def re_rsearch( pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): - matches = [match for match in pattern.finditer(chunk)] + matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() return offset + start, offset + end From 26a16f2c43dc96fe33d0b0fc8846402e9ae97e9a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 28 Feb 2024 16:36:19 -0300 Subject: [PATCH 20/45] Fix tests --- tests/test_crawl.py | 10 ++--- tests/test_downloadermiddleware_cookies.py | 2 +- tests/test_downloadermiddleware_httpauth.py | 4 +- tests/test_exporters.py | 24 +++++------ tests/test_linkextractors.py | 38 ++++++++-------- tests/test_loader_deprecated.py | 20 ++++----- tests/test_mail.py | 2 +- tests/test_pipeline_crawl.py | 2 +- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 18 ++++---- tests/test_pipeline_media.py | 48 ++++++++++----------- tests/test_scheduler.py | 26 +++++------ tests/test_spidermiddleware_offsite.py | 17 +++++--- tests/test_utils_iterators.py | 16 +++---- tests/test_utils_template.py | 2 +- 15 files changed, 117 insertions(+), 114 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 96d43b2b9..6cde4ed8c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -76,11 +76,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def _test_delay(self, total, delay, randomize=False): - crawl_kwargs = dict( - maxlatency=delay * 2, - mockserver=self.mockserver, - total=total, - ) + crawl_kwargs = { + "maxlatency": delay * 2, + "mockserver": self.mockserver, + "total": total, + } tolerance = 1 - (0.6 if randomize else 0.2) settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 4a81a638e..425fabcc7 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -320,7 +320,7 @@ class CookiesMiddlewareTest(TestCase): @pytest.mark.xfail(reason="Cookie header is not currently being processed") def test_keep_cookie_from_default_request_headers_middleware(self): - DEFAULT_REQUEST_HEADERS = dict(Cookie="default=value; asdf=qwerty") + DEFAULT_REQUEST_HEADERS = {"Cookie": "default=value; asdf=qwerty"} mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) # overwrite with values from 'cookies' request argument req1 = Request("http://example.org", cookies={"default": "something"}) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index fc110e6cc..500af6536 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -59,7 +59,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") @@ -79,6 +79,6 @@ class HttpAuthAnyMiddlewareTest(unittest.TestCase): self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar")) def test_auth_already_set(self): - req = Request("http://example.com/", headers=dict(Authorization="Digest 123")) + req = Request("http://example.com/", headers={"Authorization": "Digest 123"}) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers["Authorization"], b"Digest 123") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 59b724495..fa9389044 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -152,7 +152,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) ie = self._get_exporter() exported = ie.export_item(i3) @@ -185,7 +185,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_item_dict_list(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=[i1]) + i2 = {"name": "Maria", "age": [i1]} i3 = self.item_class(name="Jesus", age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) @@ -373,7 +373,7 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_join_multivalue_not_strings(self): self.assertExportResult( - item=dict(name="John", friends=[4, 8]), + item={"name": "John", "friends": [4, 8]}, include_headers_line=False, expected='"[4, 8]",John\r\n', ) @@ -388,14 +388,14 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_errors_default(self): with self.assertRaises(UnicodeEncodeError): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, expected=None, encoding="windows-1251", ) def test_errors_xmlcharrefreplace(self): self.assertExportResult( - item=dict(text="W\u0275\u200Brd"), + item={"text": "W\u0275\u200Brd"}, include_headers_line=False, expected="Wɵ​rd\r\n", encoding="windows-1251", @@ -455,8 +455,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name="foo\xa3hoo", age="22") - i2 = dict(name="bar", age=i1) + i1 = {"name": "foo\xa3hoo", "age": "22"} + i2 = {"name": "bar", "age": i1} i3 = self.item_class(name="buz", age=i2) self.assertExportResult( @@ -478,8 +478,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name="foo") - i2 = dict(name="bar", v2={"egg": ["spam"]}) + i1 = {"name": "foo"} + i2 = {"name": "bar", "v2": {"egg": ["spam"]}} i3 = self.item_class(name="buz", age=[i1, i2]) self.assertExportResult( @@ -534,7 +534,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def test_nested_item(self): i1 = self.item_class(name="Joseph", age="22") - i2 = dict(name="Maria", age=i1) + i2 = {"name": "Maria", "age": i1} i3 = self.item_class(name="Jesus", age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -622,9 +622,9 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name="Joseph\xa3", age="22") + i1 = {"name": "Joseph\xa3", "age": "22"} i2 = self.item_class(name="Maria", age=i1) - i3 = dict(name="Jesus", age=i2) + i3 = {"name": "Jesus", "age": i2} self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 6b4df90d8..217c7a299 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -37,7 +37,7 @@ class Base: page4_url = "http://example.com/page%204.html" self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -55,7 +55,7 @@ class Base: def test_extract_filter_allow(self): lx = self.extractor_cls(allow=("sample",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -70,7 +70,7 @@ class Base: def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=("sample",), unique=False) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -93,7 +93,7 @@ class Base: def test_extract_filter_allow_with_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=False, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -116,7 +116,7 @@ class Base: def test_extract_filter_allow_no_duplicates_canonicalize(self): lx = self.extractor_cls(allow=("sample",), unique=True, canonicalize=True) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -127,7 +127,7 @@ class Base: def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=("sample",), deny=("3",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -137,7 +137,7 @@ class Base: def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=("google.com",)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -148,7 +148,7 @@ class Base: lx = self.extractor_cls(allow="sample") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -162,7 +162,7 @@ class Base: lx = self.extractor_cls(allow="sample", deny="3") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -171,7 +171,7 @@ class Base: lx = self.extractor_cls(allow_domains="google.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -179,7 +179,7 @@ class Base: lx = self.extractor_cls(deny_domains="example.com") self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://www.google.com/something", text=""), ], @@ -265,7 +265,7 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]',)) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -337,7 +337,7 @@ class Base: restrict_css=("#subwrapper + a",), ) self.assertEqual( - [link for link in lx.extract_links(self.response)], + list(lx.extract_links(self.response)), [ Link(url="http://example.com/sample1.html", text=""), Link(url="http://example.com/sample2.html", text="sample 2"), @@ -705,7 +705,7 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -758,7 +758,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", text="Item 1", nofollow=False @@ -779,7 +779,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text="dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -791,7 +791,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Unique regex test lx = self.extractor_cls(restrict_text=r"of.*dog") self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", @@ -803,7 +803,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Multiple regex test lx = self.extractor_cls(restrict_text=[r"of.*dog", r"of.*cat"]) self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item1.html", @@ -834,7 +834,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( - [link for link in lx.extract_links(response)], + list(lx.extract_links(response)), [ Link( url="http://example.org/item2.html", diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 99cdf88d9..528efa142 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -565,37 +565,37 @@ class NoInputReprocessingFromDictTest(unittest.TestCase): """ def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title="foo")) + il = NoInputReprocessingDictLoader(item={"title": "foo"}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=["foo", "bar"])) + il = NoInputReprocessingDictLoader(item={"title": ["foo", "bar"]}) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="foo")) + self.assertEqual(il_loaded, {"title": "foo"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="foo") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "foo"} ) def test_avoid_reprocessing_without_initial_values_single(self): il = NoInputReprocessingDictLoader() il.add_value("title", "foo") il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) def test_avoid_reprocessing_without_initial_values_list(self): il = NoInputReprocessingDictLoader() il.add_value("title", ["foo", "bar"]) il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title="FOO")) + self.assertEqual(il_loaded, {"title": "FOO"}) self.assertEqual( - NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title="FOO") + NoInputReprocessingDictLoader(item=il_loaded).load_item(), {"title": "FOO"} ) diff --git a/tests/test_mail.py b/tests/test_mail.py index 2535e58db..ff1505397 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -91,7 +91,7 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(attach.get_payload(decode=True), b"content") def _catch_mail_sent(self, **kwargs): - self.catched_msg = dict(**kwargs) + self.catched_msg = {**kwargs} def test_send_utf8(self): subject = "sübjèçt" diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index be9811980..5a9a217ce 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -140,7 +140,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store - self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) + self.assertEqual(list(self.tmpmediastore.iterdir()), []) @defer.inlineCallbacks def test_download_media(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e7000e314..0babde4d9 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -221,7 +221,7 @@ class FilesPipelineTestCase(unittest.TestCase): file_path = CustomFilesPipeline.from_settings( Settings({"FILES_STORE": self.tempdir}) ).file_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual(file_path(request, item=item), "full/path-to-store-file") diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 2e2e06b89..18a2454b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -132,7 +132,7 @@ class ImagesPipelineTestCase(unittest.TestCase): thumb_path = CustomImagesPipeline.from_settings( Settings({"IMAGES_STORE": self.tempdir}) ).thumb_path - item = dict(path="path-to-store-file") + item = {"path": "path-to-store-file"} request = Request("http://example.com") self.assertEqual( thumb_path(request, "small", item=item), "thumb/small/path-to-store-file" @@ -433,14 +433,14 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): ] # This should match what is defined in ImagesPipeline. - default_pipeline_settings = dict( - MIN_WIDTH=0, - MIN_HEIGHT=0, - EXPIRES=90, - THUMBS={}, - IMAGES_URLS_FIELD="image_urls", - IMAGES_RESULT_FIELD="images", - ) + default_pipeline_settings = { + "MIN_WIDTH": 0, + "MIN_HEIGHT": 0, + "EXPIRES": 90, + "THUMBS": {}, + "IMAGES_URLS_FIELD": "image_urls", + "IMAGES_RESULT_FIELD": "images", + } def setUp(self): self.tempdir = mkdtemp() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index d477b59be..d4dde4a40 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -59,7 +59,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_to_download(request, self.info) is None def test_default_get_media_requests(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.get_media_requests(item, self.info) is None def test_default_media_downloaded(self): @@ -73,7 +73,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe.media_failed(fail, request, self.info) is fail def test_default_item_completed(self): - item = dict(name="name") + item = {"name": "name"} assert self.pipe.item_completed([], item, self.info) is item # Check that failures are logged by default @@ -98,7 +98,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase): @inlineCallbacks def test_default_process_item(self): - item = dict(name="name") + item = {"name": "name"} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item @@ -226,11 +226,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): rsp = Response("http://url1") req = Request( "http://url1", - meta=dict(response=rsp), + meta={"response": rsp}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp)]) self.assertEqual( @@ -250,11 +250,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): fail = Failure(Exception()) req = Request( "http://url1", - meta=dict(response=fail), + meta={"response": fail}, callback=self._callback, errback=self._errback, ) - item = dict(requests=req) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(False, fail)]) self.assertEqual( @@ -272,10 +272,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_mix_of_success_and_failure(self): self.pipe.LOG_FAILED_RESULTS = False rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) fail = Failure(Exception()) - req2 = Request("http://url2", meta=dict(response=fail)) - item = dict(requests=[req1, req2]) + req2 = Request("http://url2", meta={"response": fail}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled @@ -294,7 +294,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request("http://url") - item = dict(requests=req) # pass a single item + item = {"requests": req} # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item self.assertIn(self.fingerprint(req), self.info.downloaded) @@ -302,7 +302,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): # returns iterable of Requests req1 = Request("http://url1") req2 = Request("http://url2") - item = dict(requests=iter([req1, req2])) + item = {"requests": iter([req1, req2])} new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded @@ -311,17 +311,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_across_multiple_items(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) - item = dict(requests=req1) + req1 = Request("http://url1", meta={"response": rsp1}) + item = {"requests": req1} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1)]) # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=req2) + item = {"requests": req2} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(self.fingerprint(req1), self.fingerprint(req2)) @@ -330,11 +330,11 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): @inlineCallbacks def test_results_are_cached_for_requests_of_single_item(self): rsp1 = Response("http://url1") - req1 = Request("http://url1", meta=dict(response=rsp1)) + req1 = Request("http://url1", meta={"response": rsp1}) req2 = Request( - req1.url, meta=dict(response=Response("http://donot.download.me")) + req1.url, meta={"response": Response("http://donot.download.me")} ) - item = dict(requests=[req1, req2]) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertTrue(new_item is item) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @@ -359,16 +359,16 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def rsp2_func(): self.fail("it must cache rsp1 result and must not try to redownload") - req1 = Request("http://url", meta=dict(response=rsp1_func)) - req2 = Request(req1.url, meta=dict(response=rsp2_func)) - item = dict(requests=[req1, req2]) + req1 = Request("http://url", meta={"response": rsp1_func}) + req2 = Request(req1.url, meta={"response": rsp2_func}) + item = {"requests": [req1, req2]} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)]) @inlineCallbacks def test_use_media_to_download_result(self): - req = Request("http://url", meta=dict(result="ITSME", response=self.fail)) - item = dict(requests=req) + req = Request("http://url", meta={"result": "ITSME", "response": self.fail}) + item = {"requests": req} new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item["results"], [(True, "ITSME")]) self.assertEqual( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index f8465a5ff..37099dae6 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -45,15 +45,15 @@ class MockDownloader: class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict( - SCHEDULER_DEBUG=False, - SCHEDULER_DISK_QUEUE="scrapy.squeues.PickleLifoDiskQueue", - SCHEDULER_MEMORY_QUEUE="scrapy.squeues.LifoMemoryQueue", - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS="scrapy.dupefilters.BaseDupeFilter", - REQUEST_FINGERPRINTER_IMPLEMENTATION="2.7", - ) + settings = { + "SCHEDULER_DEBUG": False, + "SCHEDULER_DISK_QUEUE": "scrapy.squeues.PickleLifoDiskQueue", + "SCHEDULER_MEMORY_QUEUE": "scrapy.squeues.LifoMemoryQueue", + "SCHEDULER_PRIORITY_QUEUE": priority_queue_cls, + "JOBDIR": jobdir, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + } super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -338,10 +338,10 @@ class TestIntegrationWithDownloaderAwareInMemory(TestCase): class TestIncompatibility(unittest.TestCase): def _incompatible(self): - settings = dict( - SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", - CONCURRENT_REQUESTS_PER_IP=1, - ) + settings = { + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue", + "CONCURRENT_REQUESTS_PER_IP": 1, + } crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name="spider") diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ea45b7698..837f1c2c8 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,10 +16,10 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict( - name="foo", - allowed_domains=["scrapytest.org", "scrapy.org", "scrapy.test.org"], - ) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", "scrapy.org", "scrapy.test.org"], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -50,7 +50,7 @@ class TestOffsiteMiddleware(TestCase): class TestOffsiteMiddleware2(TestOffsiteMiddleware): def _get_spiderargs(self): - return dict(name="foo", allowed_domains=None) + return {"name": "foo", "allowed_domains": None} def test_process_spider_output(self): res = Response("http://scrapytest.org") @@ -61,13 +61,16 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spiderargs(self): - return dict(name="foo") + return {"name": "foo"} class TestOffsiteMiddleware4(TestOffsiteMiddleware3): def _get_spiderargs(self): bad_hostname = urlparse("http:////scrapytest.org").hostname - return dict(name="foo", allowed_domains=["scrapytest.org", None, bad_hostname]) + return { + "name": "foo", + "allowed_domains": ["scrapytest.org", None, bad_hostname], + } def test_process_spider_output(self): res = Response("http://scrapytest.org") diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index ee22e6675..ec377bb19 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -355,7 +355,7 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) - result = [row for row in csv] + result = list(csv) self.assertEqual( result, [ @@ -377,7 +377,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -394,7 +394,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual( - [row for row in csv1], + list(csv1), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -407,7 +407,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual( - [row for row in csv2], + list(csv2), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -422,7 +422,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, { @@ -441,7 +441,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter="\t") self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -458,7 +458,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode("utf-8") for h in headers]) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, @@ -475,7 +475,7 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual( - [row for row in csv], + list(csv), [ {"id": "1", "name": "alpha", "value": "foobar"}, {"id": "2", "name": "unicode", "value": "\xfan\xedc\xf3d\xe9\u203d"}, diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index cbe80e157..fc42c0d2f 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -16,7 +16,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): rmtree(self.tmp_path) def test_simple_render(self): - context = dict(project_name="proj", name="spi", classname="TheSpider") + context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" From 2169810414a700fcbfe33eafe1e85e46e7f62413 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 29 Feb 2024 06:41:14 -0300 Subject: [PATCH 21/45] fix: Proxy tests don't use custom certificate authority --- tests/test_proxy_connect.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 46d42e9f6..93f006c76 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -27,7 +27,7 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem" + cert_path = Path(__file__).parent.resolve() / "keys" self.proc = Popen( [ sys.executable, @@ -40,8 +40,8 @@ sys.exit(mitmdump()) "0", "--proxyauth", f"{self.auth_user}:{self.auth_pass}", - "--certs", - str(cert_path), + "--set", + f"confdir={cert_path}", "--ssl-insecure", ], stdout=PIPE, From 2bfd9a2257c79ae56955e95b46f2bc7b23e1eabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:11:42 +0100 Subject: [PATCH 22/45] bandit: allow-list false positives --- .bandit.yml | 11 +---------- scrapy/commands/bench.py | 6 ++++-- scrapy/commands/edit.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/core/downloader/__init__.py | 2 +- scrapy/exporters.py | 4 ++-- scrapy/extensions/httpcache.py | 6 +++--- scrapy/extensions/spiderstate.py | 4 ++-- scrapy/settings/default_settings.py | 2 +- scrapy/shell.py | 2 +- scrapy/squeues.py | 2 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/engine.py | 2 +- 13 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 6e8331c0f..4fcd75c57 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,19 +1,10 @@ skips: -- B101 # assert_used -- B105 # hardcoded_password_string -- B301 # pickle -- B307 # eval -- B311 # random +- B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B403 # import_pickle -- B404 # import_subprocess -- B406 # import_xml_sax - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults -- B603 # subprocess_without_shell_equals_true -- B605 # start_process_with_a_shell exclude_dirs: ['tests'] diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index e1ccdc451..aaf5a439f 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,4 +1,4 @@ -import subprocess +import subprocess # nosec import sys import time from urllib.parse import urlencode @@ -29,7 +29,9 @@ class _BenchServer: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] - self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv()) + self.proc = subprocess.Popen( + pargs, stdout=subprocess.PIPE, env=get_testenv() + ) # nosec self.proc.stdout.readline() def __exit__(self, exc_type, exc_value, traceback): diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 03a8ed5c7..e85d2c9ec 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -37,4 +37,4 @@ class Command(ScrapyCommand): sfile = sys.modules[spidercls.__module__].__file__ sfile = sfile.replace(".pyc", ".py") - self.exitcode = os.system(f'{editor} "{sfile}"') + self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 68cbe8ff6..567ebcdc0 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -113,7 +113,7 @@ class Command(ScrapyCommand): if template_file: self._genspider(module, name, url, opts.template, template_file) if opts.edit: - self.exitcode = os.system(f'scrapy edit "{name}"') + self.exitcode = os.system(f'scrapy edit "{name}"') # nosec def _genspider(self, module, name, url, template_name, template_file): """Generate the spider module, based on the given template""" diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c84525160..666282856 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -40,7 +40,7 @@ class Slot: def download_delay(self) -> float: if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # nosec return self.delay def close(self) -> None: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f85f1dad8..79fd4e56f 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats. import csv import io import marshal -import pickle +import pickle # nosec import pprint from collections.abc import Mapping -from xml.sax.saxutils import XMLGenerator +from xml.sax.saxutils import XMLGenerator # nosec from itemadapter import ItemAdapter, is_item diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 7e4f047a8..335728502 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,6 @@ import gzip import logging -import pickle +import pickle # nosec from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path @@ -274,7 +274,7 @@ class DbmCacheStorage: if 0 < self.expiration_secs < time() - float(ts): return # expired - return pickle.loads(db[f"{key}_data"]) + return pickle.loads(db[f"{key}_data"]) # nosec class FilesystemCacheStorage: @@ -352,7 +352,7 @@ class FilesystemCacheStorage: if 0 < self.expiration_secs < time() - mtime: return # expired with self._open(metapath, "rb") as f: - return pickle.load(f) + return pickle.load(f) # nosec def parse_cachecontrol(header): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 929a3be70..43359401b 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec from pathlib import Path from scrapy import signals @@ -31,7 +31,7 @@ class SpiderState: def spider_opened(self, spider): if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) + spider.state = pickle.load(f) # nosec else: spider.state = {} diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 49ab1b5ef..2b3d95a0e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private" FILES_STORE_GCS_ACL = "" FTP_USER = "anonymous" -FTP_PASSWORD = "guest" +FTP_PASSWORD = "guest" # nosec FTP_PASSIVE_MODE = True GCS_PROJECT_ID = None diff --git a/scrapy/shell.py b/scrapy/shell.py index 05909977a..63ea33892 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -50,7 +50,7 @@ class Shell: else: self.populate_vars() if self.code: - print(eval(self.code, globals(), self.vars)) + print(eval(self.code, globals(), self.vars)) # nosec else: """ Detect interactive shell setting in scrapy.cfg diff --git a/scrapy/squeues.py b/scrapy/squeues.py index f665ad88c..e20f60f06 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -import pickle +import pickle # nosec from os import PathLike from pathlib import Path from typing import Union diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 38884a9f0..f6f704d4b 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -14,7 +14,7 @@ class Root(Resource): def render(self, request): total = _getarg(request, b"total", 100, int) show = _getarg(request, b"show", 10, int) - nlist = [random.randint(1, total) for _ in range(show)] + nlist = [random.randint(1, total) for _ in range(show)] # nosec request.write(b"") args = request.args.copy() for nl in nlist: diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a5f2a8c6e..0b2722663 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: checks: List[Tuple[str, Any]] = [] for test in tests: try: - checks += [(test, eval(test))] + checks += [(test, eval(test))] # nosec except Exception as e: checks += [(test, f"{type(e).__name__} (exception)")] From 31cbbb57584fe2a7c42d30acf2aa4707039457b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 29 Feb 2024 11:31:39 +0100 Subject: [PATCH 23/45] bandit: ignore md5 usage for download slot names --- scrapy/pqueues.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index b62d2fe58..593667f1f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -20,7 +20,7 @@ def _path_safe(text): pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() + unique_slot = hashlib.md5(text.encode("utf8")).hexdigest() # nosec return "-".join([pathable_slot, unique_slot]) From 1311e7db05204fe2cae7d1c5caf8b0ffe9371cd0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 16:31:06 +0500 Subject: [PATCH 24/45] Regenerate the expired mitmproxy-ca.pem. --- tests/keys/mitmproxy-ca.pem | 93 ++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 48 deletions(-) diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem index cdef75f99..61a690cc8 100644 --- a/tests/keys/mitmproxy-ca.pem +++ b/tests/keys/mitmproxy-ca.pem @@ -1,50 +1,47 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB -/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT -NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ -u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH -JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB -eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls -90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ -epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8 -VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn -v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT -I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU -oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au -/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG -gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn -YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf -dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5 -9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC -QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu -bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW -CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR -OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu -PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK -eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx -BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/ -B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy -HzPh2taMnCA0h1fNLFFBkw88 ------END PRIVATE KEY----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAuq7ujTpHoSwQn9/hFJT837jU/T7xLuyXjkAEfL7uVDuPWSdF +AJy+QJsuL6INMKMlxLUb1RRzxQgAmtYN1dIEbTPplffCNbfYm3FXg0mZlxg1UBg9 +rE1bPwuz/B+M76S35EIiKQlpaCFErLQi5oyhw9FIBvYLZxxgfeDfDPiXQlBvtHix +n9TFqNoLNZkAX+auh2Wj2SSjM74pBQWsuVZLkF5CAwYuMQkpEplCV/QHNX3ZeNdQ +YNFvpA7CxENa3sTZyHpeTmWoOdJXgJFveAWL4ZhSvkSd0HDuPPJp4JckETNESa9B +qOwFfj36SM+5dRiCwiuzwAQ+oaFjisMXuRbVFQIDAQABAoIBABqFaJmCupNgnboA +xcq1QdmMuiGCNCRs8zj/ykNoopYv4fUR+aEVI4gtI5obxRDwVJjF+/7BCZNnyCI3 +H78NN5jGA7zM9nfINwsaRor9xUasZ0KKNxTH5pslz/uVBeIzvfY9GPpIfoOPGmEI +tF6Zgw+9JyTqBoOvCdxIOpfupxqB8TQ0z4UbFUuBiEkGuJ+o8C1rX8Wze0JUl0qG +BOwhQtaCn/yrm+dTXZ5XaelJY5mcwgFy22Jiynmm6TbLhyZlACd2Q/MGak7o1TJL +QgvvGMlcVrK7MZ3TJN+wzwWfwAAjXnT3Xvd5pD5yunZJoNe8YyFOCMlh0swNG5Zt +0tGeX78CgYEA+m6gYGKTNWFnqlqMZRfGTqiqVZeVQKftcLdA1dkscnffRP9bvKOW +9TbgzoGHiyZnjZBDFTUuy67El8RXIMsxYy3GYuGRdUSLS63Fr2af4pBQIYvW6OmG +UZlcAP6ZAhUzn409XGlXaac3F30hFeKdC62+V5ZMnfPlVhHRCoKaaqsCgYEAvtV4 +FuU5sFKyhKPPV3rzaNZtL0swvtBIuODH1oAWhPNySQvCu+45W0EOOAPPpsYP4wGX +G+otOSp4RLdlVXNhkh1rpJzeK77KZ5ZY+ShkuHD+uL/iRARwl/gh2Ve1aqUrm1LE +9ldchmQGvLalN9HalzeW//xHA3X9SF4Vo16Dvz8CgYABeZlUOABp9hLoO/RLvCIc +4H1wV543bUXGvi2RlN/gJLiZ7W8a41PGSfZ1AOpNdYJyoQDkJRYLeRILWsqwlMHL +tb9PYci7ihXP8kwRxmb2rKbsK6iuYoG6BU83akh4bKuLKwfLfYtYQfXfG4uQV29Z +XEKcvXPiEkethBlZGH/UVQKBgCa9Pvum3OcmYob6mgSwOOl3XgLTyLlzns+pEehB +aFDk+rZJZOaxnYMg2boVS/oXCvKSSBKqnzOTo4aPlEqceZonzspD7fYDbSNKKhWq +VYf7qDno+g3EuPagsH5mh7V2gjutub4oTegaNiPpD/Ec8Lrx1f1xQRk6wogGUW4w +qZ4RAoGBAPg1LezV8mlesF5mhj+KubYP4l1Zf9geAeQprjDbFsA0BEAS2KsWgmwR +Ye1fmek7jDjCPLQ4Amq030mLJuQGEM3cZPqjKX2sBZ8fQcgw7pWJWMvKMTBA4Aah +zQx1KXwHJANMWq/0QSFDq/LGJ2OYMlV2F0tH3P5Kp7ZASTyc78ux +-----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM -CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX -DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt -aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW -ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7 -ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg -z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI -ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE -6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/ -0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC -AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD -BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG -CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC -AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA -A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd -kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe -gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY -1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj -envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj -wiKjdLjCmJgjGAQjRw5VYJ8JI0XO +MIIDNTCCAh2gAwIBAgIUcGDiCmOuhfxMGFS/otcGGFkOSAEwDQYJKoZIhvcNAQEL +BQAwKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcN +MjQwMjI3MTMwNTQ4WhcNMzQwMjI2MTMwNTQ4WjAoMRIwEAYDVQQDDAltaXRtcHJv +eHkxEjAQBgNVBAoMCW1pdG1wcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBALqu7o06R6EsEJ/f4RSU/N+41P0+8S7sl45ABHy+7lQ7j1knRQCcvkCb +Li+iDTCjJcS1G9UUc8UIAJrWDdXSBG0z6ZX3wjW32JtxV4NJmZcYNVAYPaxNWz8L +s/wfjO+kt+RCIikJaWghRKy0IuaMocPRSAb2C2ccYH3g3wz4l0JQb7R4sZ/Uxaja +CzWZAF/mrodlo9kkozO+KQUFrLlWS5BeQgMGLjEJKRKZQlf0BzV92XjXUGDRb6QO +wsRDWt7E2ch6Xk5lqDnSV4CRb3gFi+GYUr5EndBw7jzyaeCXJBEzREmvQajsBX49 ++kjPuXUYgsIrs8AEPqGhY4rDF7kW1RUCAwEAAaNXMFUwDwYDVR0TAQH/BAUwAwEB +/zATBgNVHSUEDDAKBggrBgEFBQcDATAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FOjFT0G7itqsrCij2InhRSfB0sEkMA0GCSqGSIb3DQEBCwUAA4IBAQCVMa5/xlH4 +GUbrWNMdxr9LL7Dh+vK0wYCfAsc/kO2zCq8iVt/MaqVLel/bKcQhvE5RZHvyep13 +x7378OfCqqHkDDDNroWIvij84ZtMUaM53tF13G/ZGOlNsoLNynWs9IVVvqGKsH7o +/buJ1RNArI/0irF0UD7qrMmo1p6SYanZhqdh2PphNy9NS3FsfrfnuWvf+/TRp9Ts +L8058B0p/LIL0OB5trYFircC3iKSOuRl0ERD2ufgSqsSVEYm1mc6UIxv+d1iFD+Q +8CRUF88icQXrec1TCbhh0CfdDxz+FYSTnW0DR0L75coa/CBmRxAjnrkLoXRr3Y1d +sTjU4zDdBcBw -----END CERTIFICATE----- From 4cd94aa668c60f92b1d9f4e5cf27752e1fe9c9cd Mon Sep 17 00:00:00 2001 From: "Yuri H. Galvao" Date: Fri, 1 Mar 2024 04:07:38 -0600 Subject: [PATCH 25/45] Restore brotlipy support (#6261) --- scrapy/utils/_compression.py | 32 +++++++++++++++++++++++++++++++- tox.ini | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 5610595d3..14531df3f 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -1,10 +1,40 @@ import zlib from io import BytesIO +from warnings import warn + +from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli except ImportError: pass +else: + try: + brotli.Decompressor.process + except AttributeError: + + warn( + ( + "You have brotlipy installed, and Scrapy will use it, but " + "Scrapy support for brotlipy is deprecated and will stop " + "working in a future version of Scrapy. brotlipy itself is " + "deprecated, it has been superseded by brotlicffi (not " + "currently supported by Scrapy). Please, uninstall brotlipy " + "and install brotli instead. brotlipy has the same import " + "name as brotli, so keeping both installed is strongly " + "discouraged." + ), + ScrapyDeprecationWarning, + ) + + def _brotli_decompress(decompressor, data): + return decompressor.decompress(data) + + else: + + def _brotli_decompress(decompressor, data): + return decompressor.process(data) + try: import zstandard @@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes: decompressed_size = 0 while output_chunk: input_chunk = input_stream.read(_CHUNK_SIZE) - output_chunk = decompressor.process(input_chunk) + output_chunk = _brotli_decompress(decompressor, input_chunk) decompressed_size += len(output_chunk) if max_size and decompressed_size > max_size: raise _DecompressionMaxSizeExceeded( diff --git a/tox.ini b/tox.ini index 4ed9b3bd7..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -135,6 +135,7 @@ deps = google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 + brotlipy install_command = {[pinned]install_command} setenv = {[pinned]setenv} From aa1bf6907964f0281264052cabc28197c5d28107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2024 12:48:00 +0100 Subject: [PATCH 26/45] Mark hashlib usages as not intended for security (#6264) --- .bandit.yml | 1 - scrapy/pipelines/files.py | 28 ++++++++++++++++++++++------ scrapy/pipelines/images.py | 9 ++++----- scrapy/utils/misc.py | 10 +++++++++- scrapy/utils/request.py | 2 +- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index 4fcd75c57..db2fbb84c 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -2,7 +2,6 @@ skips: - B101 # assert_used, needed for mypy - B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B324 # hashlib "Use of weak SHA1 hash for security" - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 - B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1d7625299..d04218089 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -16,7 +16,7 @@ from ftplib import FTP from io import BytesIO from os import PathLike from pathlib import Path -from typing import DefaultDict, Optional, Set, Union +from typing import IO, DefaultDict, Optional, Set, Union from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -31,7 +31,6 @@ from scrapy.utils.boto import is_botocore_available from scrapy.utils.datatypes import CaseInsensitiveDict from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes from scrapy.utils.request import referer_str @@ -42,6 +41,23 @@ def _to_string(path: Union[str, PathLike]) -> str: return str(path) # convert a Path object to string +def _md5sum(file: IO) -> str: + """Calculate the md5 checksum of a file-like object without reading its + whole content in memory. + + >>> from io import BytesIO + >>> _md5sum(BytesIO(b'file content to hash')) + '784406af91dd5a54fbb9c84c2236595a' + """ + m = hashlib.md5() # nosec + while True: + d = file.read(8096) + if not d: + break + m.update(d) + return m.hexdigest() + + class FileException(Exception): """General media error exception""" @@ -70,7 +86,7 @@ class FSFilesStore: return {} with absolute_path.open("rb") as f: - checksum = md5sum(f) + checksum = _md5sum(f) return {"last_modified": last_modified, "checksum": checksum} @@ -299,7 +315,7 @@ class FTPFilesStore: ftp.set_pasv(False) file_path = f"{self.basedir}/{path}" last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() + m = hashlib.md5() # nosec ftp.retrbinary(f"RETR {file_path}", m.update) return {"last_modified": last_modified, "checksum": m.hexdigest()} # The file doesn't exist @@ -531,7 +547,7 @@ class FilesPipeline(MediaPipeline): def file_downloaded(self, response, request, info, *, item=None): path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) - checksum = md5sum(buf) + checksum = _md5sum(buf) buf.seek(0) self.store.persist_file(path, buf, info) return checksum @@ -542,7 +558,7 @@ class FilesPipeline(MediaPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 8169583f8..137aa7a9a 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -17,11 +17,10 @@ from itemadapter import ItemAdapter from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import FileException, FilesPipeline +from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.misc import md5sum from scrapy.utils.python import get_func_args, to_bytes @@ -128,7 +127,7 @@ class ImagesPipeline(FilesPipeline): for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) - checksum = md5sum(buf) + checksum = _md5sum(buf) width, height = image.size self.store.persist_file( path, @@ -228,9 +227,9 @@ class ImagesPipeline(FilesPipeline): return item def file_path(self, request, response=None, info=None, *, item=None): - image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"full/{image_guid}.jpg" def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None): - thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() # nosec return f"thumbs/{thumb_id}/{thumb_guid}.jpg" diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7b43760a8..7f83d06fb 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -113,7 +113,15 @@ def md5sum(file: IO) -> str: >>> md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ - m = hashlib.md5() + warnings.warn( + ( + "The scrapy.utils.misc.md5sum function is deprecated, and will be " + "removed in a future version of Scrapy." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + m = hashlib.md5() # nosec while True: d = file.read(8096) if not d: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index e99d1eeb5..1f07d58eb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -111,7 +111,7 @@ def fingerprint( "headers": headers, } fingerprint_json = json.dumps(fingerprint_data, sort_keys=True) - cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() + cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest() # nosec return cache[cache_key] From bf149356fc6e519e92fb55150a60b40b14e45ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 1 Mar 2024 16:02:03 +0100 Subject: [PATCH 27/45] Bandit: allow-list lxml usages (#6265) --- .bandit.yml | 2 -- scrapy/http/request/form.py | 17 ++++++----------- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/selector/unified.py | 3 ++- scrapy/utils/_compression.py | 1 - scrapy/utils/iterators.py | 5 +++-- scrapy/utils/sitemap.py | 4 ++-- scrapy/utils/versions.py | 2 +- 8 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index db2fbb84c..b7f1817e0 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,9 +1,7 @@ skips: - B101 # assert_used, needed for mypy -- B320 # xml_bad_etree - B321 # ftplib, https://github.com/scrapy/scrapy/issues/4180 - B402 # import_ftplib, https://github.com/scrapy/scrapy/issues/4180 -- B410 # import_lxml - B411 # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082 - B503 # ssl_with_bad_defaults exclude_dirs: ['tests'] diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0f80a0ab7..3206d79cd 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -10,21 +10,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit -from lxml.html import ( - FormElement, - HTMLParser, - InputElement, - MultipleSelectOptions, - SelectElement, - TextareaElement, -) -from parsel.selector import create_root_node +from lxml.html import FormElement # nosec +from lxml.html import InputElement # nosec +from lxml.html import MultipleSelectOptions # nosec +from lxml.html import SelectElement # nosec +from lxml.html import TextareaElement # nosec from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes -from scrapy.utils.response import get_base_url if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -120,7 +115,7 @@ def _get_form( formxpath: Optional[str], ) -> FormElement: """Find the wanted form element within the given response.""" - root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) + root = response.selector.root forms = root.xpath("//form") if not forms: raise ValueError(f"No element found in {response}") diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index d76db20ba..55bc0fc43 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -7,7 +7,7 @@ import operator from functools import partial from urllib.parse import urljoin, urlparse -from lxml import etree +from lxml import etree # nosec from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 75d5e9fbd..aa9581fcd 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -8,6 +8,7 @@ from parsel import Selector as _ParselSelector from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.utils.python import to_bytes +from scrapy.utils.response import get_base_url from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] @@ -88,7 +89,7 @@ class Selector(_ParselSelector, object_ref): if response is not None: text = response.text - kwargs.setdefault("base_url", response.url) + kwargs.setdefault("base_url", get_base_url(response)) self.response = response diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 14531df3f..7c40d0a02 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -12,7 +12,6 @@ else: try: brotli.Decompressor.process except AttributeError: - warn( ( "You have brotlipy installed, and Scrapy will use it, but " diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 93a2ba7a1..49493e9c6 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -18,7 +18,7 @@ from typing import ( ) from warnings import warn -from lxml import etree +from lxml import etree # nosec from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -26,7 +26,7 @@ from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode if TYPE_CHECKING: - from lxml._types import SupportsReadClose + from lxml._types import SupportsReadClose # nosec logger = logging.getLogger(__name__) @@ -101,6 +101,7 @@ def xmliter_lxml( cast("SupportsReadClose[bytes]", reader), encoding=reader.encoding, events=("end", "start-ns"), + resolve_entities=False, huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 8bf941eb2..7dcee3a2f 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -8,7 +8,7 @@ SitemapSpider, its API is subject to change without notice. from typing import Any, Dict, Generator, Iterator, Optional from urllib.parse import urljoin -import lxml.etree +import lxml.etree # nosec class Sitemap: @@ -19,7 +19,7 @@ class Sitemap: xmlp = lxml.etree.XMLParser( recover=True, remove_comments=True, resolve_entities=False ) - self._root = lxml.etree.fromstring(xmltext, parser=xmlp) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # nosec rt = self._root.tag self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 9b637bdb0..42e5e9be4 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -4,7 +4,7 @@ from typing import List, Tuple import cryptography import cssselect -import lxml.etree +import lxml.etree # nosec import parsel import twisted import w3lib From cab1016bb6f719b15043f65502c63fbaa191df36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:24:17 -0300 Subject: [PATCH 28/45] Add brotlicffi support --- scrapy/utils/_compression.py | 2 ++ ...st_downloadermiddleware_httpcompression.py | 35 +++++++++++++++---- tox.ini | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7c40d0a02..7896f4c01 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning try: import brotli +except ImportError: + import brotlicffi as brotli except ImportError: pass else: diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index ae5569d0a..7c36f748e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse("br") @@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_setting("br") @@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_spider_attr("br") @@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase): def test_compression_bomb_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_compression_bomb_request_meta("br") @@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_setting_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_setting("br") @@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_spider_attr_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_spider_attr("br") @@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase): def test_download_warnsize_request_meta_br(self): try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: raise SkipTest("no brotli") self._test_download_warnsize_request_meta("br") diff --git a/tox.ini b/tox.ini index 237aa489c..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 + brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 3421823dce94a693ee86915110d899d8da6f3e9f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 20:26:23 -0300 Subject: [PATCH 29/45] Nested try-except block --- scrapy/utils/_compression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 7896f4c01..477573588 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -5,9 +5,10 @@ from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning try: - import brotli -except ImportError: - import brotlicffi as brotli + try: + import brotli + except ImportError: + import brotlicffi as brotli except ImportError: pass else: From a52429ae08ec15d70f7f3e2079d32933bb639d6b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:12:25 -0300 Subject: [PATCH 30/45] Update disclaimer --- scrapy/utils/_compression.py | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 477573588..4b3fd342d 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,8 +20,8 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi (not " - "currently supported by Scrapy). Please, uninstall brotlipy " + "deprecated, it has been superseded by brotlicffi " + "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." diff --git a/tox.ini b/tox.ini index 6b804b78c..9cf3c92ad 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli + brotli; implementation_name != 'pypy' zstandard [testenv:extra-deps-pinned] From 16864ea602ebc3d1a764aaf6c101a5f20ff57bee Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:18:24 -0300 Subject: [PATCH 31/45] Remove PyPy condition --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9cf3c92ad..6b804b78c 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli; implementation_name != 'pypy' + brotli zstandard [testenv:extra-deps-pinned] From 532cd2eabd8b280e64a1087c49b8f5eb5f05530f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 21:30:20 -0300 Subject: [PATCH 32/45] Use brotlicffi for PyPy --- tests/requirements.txt | 3 +-- tox.ini | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 5b75674f5..ca5f6ddbd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,8 +11,7 @@ uvloop; platform_system != "Windows" bpython # optional for shell wrapper tests brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072 -brotli==1.0.9; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests +brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 6b804b78c..237aa489c 100644 --- a/tox.ini +++ b/tox.ini @@ -162,7 +162,6 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 - brotlicffi commands = pytest --durations=10 scrapy tests install_command = {[pinned]install_command} From 7f1fbdba3cc6f118cbf11285ed26e488f854aed1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 5 Mar 2024 22:11:11 -0300 Subject: [PATCH 33/45] Check brotlicffi for ACCEPTED_ENCODINGS --- scrapy/downloadermiddlewares/httpcompression.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aa3abe853..0e5e215ac 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -29,7 +29,10 @@ logger = getLogger(__name__) ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"] try: - import brotli # noqa: F401 + try: + import brotli # noqa: F401 + except ImportError: + import brotlicffi # noqa: F401 except ImportError: pass else: From 7be919138d84ec00feb80a78e13721dff998c10c Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 05:49:31 -0300 Subject: [PATCH 34/45] Update scrapy/utils/_compression.py Co-authored-by: Andrey Rakhmatullin --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4b3fd342d..349fd9ac0 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -20,7 +20,7 @@ else: "You have brotlipy installed, and Scrapy will use it, but " "Scrapy support for brotlipy is deprecated and will stop " "working in a future version of Scrapy. brotlipy itself is " - "deprecated, it has been superseded by brotlicffi " + "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " "and install brotli instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " From 99f7165c63a8a2dba72090f65ba1093d476d669a Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:02:01 -0300 Subject: [PATCH 35/45] Update scrapy/utils/_compression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/_compression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 349fd9ac0..84c255c28 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -22,7 +22,7 @@ else: "working in a future version of Scrapy. brotlipy itself is " "deprecated, it has been superseded by brotlicffi. " "Please, uninstall brotlipy " - "and install brotli instead. brotlipy has the same import " + "and install brotli or brotlicffi instead. brotlipy has the same import " "name as brotli, so keeping both installed is strongly " "discouraged." ), From 6ecc9e0a34be6317d1b35a3ca1fc13cb98129732 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 6 Mar 2024 17:21:08 +0500 Subject: [PATCH 36/45] Add typing for scrapy/commands (#6268) --- scrapy/commands/__init__.py | 48 +++++++----- scrapy/commands/bench.py | 18 +++-- scrapy/commands/check.py | 18 +++-- scrapy/commands/crawl.py | 14 +++- scrapy/commands/edit.py | 14 ++-- scrapy/commands/fetch.py | 23 +++--- scrapy/commands/genspider.py | 31 +++++--- scrapy/commands/list.py | 8 +- scrapy/commands/parse.py | 128 +++++++++++++++++++++++--------- scrapy/commands/runspider.py | 12 +-- scrapy/commands/settings.py | 13 ++-- scrapy/commands/shell.py | 19 ++--- scrapy/commands/startproject.py | 22 +++--- scrapy/commands/version.py | 13 ++-- scrapy/commands/view.py | 15 +++- scrapy/shell.py | 70 +++++++++++------ scrapy/utils/console.py | 48 ++++++++---- scrapy/utils/response.py | 26 +++---- 18 files changed, 355 insertions(+), 185 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 27993710e..9fe803d3c 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,61 +3,62 @@ Base class for Scrapy commands """ import argparse +import builtins import os from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Optional from twisted.python import failure -from scrapy.crawler import CrawlerProcess +from scrapy.crawler import Crawler, CrawlerProcess from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli class ScrapyCommand: - requires_project = False + requires_project: bool = False crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} - exitcode = 0 + exitcode: int = 0 def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline - def set_crawler(self, crawler): + def set_crawler(self, crawler: Crawler) -> None: if hasattr(self, "_crawler"): raise RuntimeError("crawler already set") - self._crawler = crawler + self._crawler: Crawler = crawler - def syntax(self): + def syntax(self) -> str: """ Command syntax (preferably one-line). Do not include command name. """ return "" - def short_desc(self): + def short_desc(self) -> str: """ A short description of the command """ return "" - def long_desc(self): + def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() - def help(self): + def help(self) -> str: """An extensive help for the command. It will be shown when using the "help" command. It can contain newlines since no post-formatting will be applied to its contents. """ return self.long_desc() - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: """ Populate option parse with options available for this command """ @@ -92,7 +93,7 @@ class ScrapyCommand: ) group.add_argument("--pdb", action="store_true", help="enable pdb on failure") - def process_options(self, args, opts): + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: @@ -129,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand): Common class used to share functionality between the crawl, parse and runspider commands """ - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-a", dest="spargs", @@ -162,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand): help="format to use for dumping items", ) - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: @@ -183,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): Help Formatter for scrapy command line help messages. """ - def __init__(self, prog, indent_increment=2, max_help_position=24, width=None): + def __init__( + self, + prog: str, + indent_increment: int = 2, + max_help_position: int = 24, + width: Optional[int] = None, + ): super().__init__( prog, indent_increment=indent_increment, @@ -191,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter): width=width, ) - def _join_parts(self, part_strings): - parts = self.format_part_strings(part_strings) + def _join_parts(self, part_strings: Iterable[str]) -> str: + # scrapy.commands.list shadows builtins.list + parts = self.format_part_strings(builtins.list(part_strings)) return super()._join_parts(parts) - def format_part_strings(self, part_strings): + def format_part_strings(self, part_strings: List[str]) -> List[str]: """ Underline and title case command line help message headers. """ diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index aaf5a439f..2e6bb5d86 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,10 +1,14 @@ +import argparse import subprocess # nosec import sys import time +from typing import Any, Iterable, List from urllib.parse import urlencode import scrapy +from scrapy import Request from scrapy.commands import ScrapyCommand +from scrapy.http import Response from scrapy.linkextractors import LinkExtractor @@ -15,26 +19,28 @@ class Command(ScrapyCommand): "CLOSESPIDER_TIMEOUT": 10, } - def short_desc(self): + def short_desc(self) -> str: return "Run quick benchmark test" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: with _BenchServer(): + assert self.crawler_process self.crawler_process.crawl(_BenchSpider, total=100000) self.crawler_process.start() class _BenchServer: - def __enter__(self): + def __enter__(self) -> None: from scrapy.utils.test import get_testenv pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"] self.proc = subprocess.Popen( pargs, stdout=subprocess.PIPE, env=get_testenv() ) # nosec + assert self.proc.stdout self.proc.stdout.readline() - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: self.proc.kill() self.proc.wait() time.sleep(0.2) @@ -49,11 +55,11 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - def start_requests(self): + def start_requests(self) -> Iterable[Request]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" return [scrapy.Request(url, dont_filter=True)] - def parse(self, response): + def parse(self, response: Response) -> Any: # type: ignore[override] for link in self.link_extractor.extract_links(response): yield scrapy.Request(link.url, callback=self.parse) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index de54ca4d3..22c8abf7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,5 +1,7 @@ +import argparse import time from collections import defaultdict +from typing import List from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ class TextTestResult(_TextTestResult): - def printSummary(self, start, stop): + def printSummary(self, start: float, stop: float) -> None: write = self.stream.write - writeln = self.stream.writeln + # _WritelnDecorator isn't implemented in typeshed yet + writeln = self.stream.writeln # type: ignore[attr-defined] run = self.testsRun plural = "s" if run != 1 else "" @@ -42,14 +45,14 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Check spider contracts" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -66,7 +69,7 @@ class Command(ScrapyCommand): help="print contract tests for all spiders", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # load contracts contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS")) conman = ContractsManager(load_object(c) for c in contracts) @@ -76,6 +79,7 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) + assert self.crawler_process spider_loader = self.crawler_process.spider_loader with set_environ(SCRAPY_CHECK="true"): diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 2f0f1c7b9..6e023af81 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,3 +1,8 @@ +import argparse +from typing import List, cast + +from twisted.python.failure import Failure + from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommand): requires_project = True - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a spider" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) < 1: raise UsageError() elif len(args) > 1: @@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand): ) spname = args[0] + assert self.crawler_process crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) if getattr(crawl_defer, "result", None) is not None and issubclass( - crawl_defer.result.type, Exception + cast(Failure, crawl_defer.result).type, Exception ): self.exitcode = 1 else: diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index e85d2c9ec..04012bee8 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,5 +1,7 @@ +import argparse import os import sys +from typing import List from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -9,32 +11,34 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "" - def short_desc(self): + def short_desc(self) -> str: return "Edit spider" - def long_desc(self): + def long_desc(self) -> str: return ( "Edit a spider using the editor defined in the EDITOR environment" " variable or else the EDITOR setting" ) - def _err(self, msg): + def _err(self, msg: str) -> None: sys.stderr.write(msg + os.linesep) self.exitcode = 1 - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() editor = self.settings["EDITOR"] + assert self.crawler_process try: spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: return self._err(f"Spider not found: {args[0]}") sfile = sys.modules[spidercls.__module__].__file__ + assert sfile sfile = sfile.replace(".pyc", ".py") self.exitcode = os.system(f'{editor} "{sfile}"') # nosec diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index cdb7ad4ae..1acf2d26f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,13 +1,13 @@ import sys -from argparse import Namespace -from typing import List, Type +from argparse import ArgumentParser, Namespace +from typing import Dict, List, Type from w3lib.url import is_url from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import DefaultSpider, spidercls_for_request @@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request class Command(ScrapyCommand): requires_project = False - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Fetch a URL using the Scrapy downloader" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and print its content" " to stdout. You may want to use --nolog to disable logging" ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument("--spider", dest="spider", help="use this spider") parser.add_argument( "--headers", @@ -44,20 +44,21 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def _print_headers(self, headers, prefix): + def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None: for key, values in headers.items(): for value in values: self._print_bytes(prefix + b" " + key + b": " + value) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: Namespace) -> None: if opts.headers: + assert response.request self._print_headers(response.request.headers, b">") print(">") self._print_headers(response.headers, b"<") else: self._print_bytes(response.body) - def _print_bytes(self, bytes_): + def _print_bytes(self, bytes_: bytes) -> None: sys.stdout.buffer.write(bytes_ + b"\n") def run(self, args: List[str], opts: Namespace) -> None: diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 567ebcdc0..2649fb23d 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,9 +1,10 @@ +import argparse import os import shutil import string from importlib import import_module from pathlib import Path -from typing import Optional, cast +from typing import List, Optional, Union, cast from urllib.parse import urlparse import scrapy @@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -def sanitize_module_name(module_name): +def sanitize_module_name(module_name: str) -> str: """Sanitize the given module name, by replacing dashes and points with underscores and prefixing it with a letter if it doesn't start with one @@ -23,7 +24,7 @@ def sanitize_module_name(module_name): return module_name -def extract_domain(url): +def extract_domain(url: str) -> str: """Extract domain name from URL string""" o = urlparse(url) if o.scheme == "" and o.netloc == "": @@ -31,7 +32,7 @@ def extract_domain(url): return o.netloc -def verify_url_scheme(url): +def verify_url_scheme(url: str) -> str: """Check url for scheme and insert https if none found.""" parsed = urlparse(url) if parsed.scheme == "" and parsed.netloc == "": @@ -43,14 +44,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Generate new spider using pre-defined templates" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-l", "--list", @@ -86,7 +87,7 @@ class Command(ScrapyCommand): help="If the spider already exists, overwrite it with the template", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.list: self._list_templates() return @@ -115,7 +116,14 @@ class Command(ScrapyCommand): if opts.edit: self.exitcode = os.system(f'scrapy edit "{name}"') # nosec - def _genspider(self, module, name, url, template_name, template_file): + def _genspider( + self, + module: str, + name: str, + url: str, + template_name: str, + template_file: Union[str, os.PathLike], + ) -> None: """Generate the spider module, based on the given template""" capitalized_module = "".join(s.capitalize() for s in module.split("_")) domain = extract_domain(url) @@ -130,6 +138,7 @@ class Command(ScrapyCommand): } if self.settings.get("NEWSPIDER_MODULE"): spiders_module = import_module(self.settings["NEWSPIDER_MODULE"]) + assert spiders_module.__file__ spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None @@ -152,7 +161,7 @@ class Command(ScrapyCommand): print('Use "scrapy genspider --list" to see all available templates.') return None - def _list_templates(self): + def _list_templates(self) -> None: print("Available templates:") for file in sorted(Path(self.templates_dir).iterdir()): if file.suffix == ".tmpl": diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2f5032360..dcc51a694 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + from scrapy.commands import ScrapyCommand @@ -5,9 +8,10 @@ class Command(ScrapyCommand): requires_project = True default_settings = {"LOG_ENABLED": False} - def short_desc(self): + def short_desc(self) -> str: return "List available spiders" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c9f8586d3..2453c0d39 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,16 +1,32 @@ +import argparse import functools import inspect import json import logging -from typing import Dict +from types import CoroutineType +from typing import ( + Any, + AsyncGenerator, + Callable, + Dict, + Iterable, + List, + Optional, + Tuple, + TypeVar, + Union, + overload, +) from itemadapter import ItemAdapter, is_item -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import Deferred, maybeDeferred +from twisted.python.failure import Failure from w3lib.url import is_url from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -from scrapy.http import Request +from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import aiter_errback, deferred_from_coro @@ -20,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Command(BaseRunSpiderCommand): requires_project = True spider = None - items: Dict[int, list] = {} - requests: Dict[int, list] = {} + items: Dict[int, List[Any]] = {} + requests: Dict[int, List[Request]] = {} first_response = None - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Parse URL (using its spider) and print the results" - def add_options(self, parser): - BaseRunSpiderCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--spider", dest="spider", @@ -106,7 +124,7 @@ class Command(BaseRunSpiderCommand): ) @property - def max_level(self): + def max_level(self) -> int: max_items, max_requests = 0, 0 if self.items: max_items = max(self.items) @@ -114,13 +132,21 @@ class Command(BaseRunSpiderCommand): max_requests = max(self.requests) return max(max_items, max_requests) - def handle_exception(self, _failure): + def handle_exception(self, _failure: Failure) -> None: logger.error( "An error is caught while iterating the async iterable", exc_info=failure_to_exc_info(_failure), ) - def iterate_spider_output(self, result): + @overload + def iterate_spider_output( + self, result: Union[AsyncGenerator, CoroutineType] + ) -> Deferred: ... + + @overload + def iterate_spider_output(self, result: _T) -> Iterable: ... + + def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -133,15 +159,15 @@ class Command(BaseRunSpiderCommand): return d return arg_to_iter(deferred_from_coro(result)) - def add_items(self, lvl, new_items): + def add_items(self, lvl: int, new_items: List[Any]) -> None: old_items = self.items.get(lvl, []) self.items[lvl] = old_items + new_items - def add_requests(self, lvl, new_reqs): + def add_requests(self, lvl: int, new_reqs: List[Request]) -> None: old_reqs = self.requests.get(lvl, []) self.requests[lvl] = old_reqs + new_reqs - def print_items(self, lvl=None, colour=True): + def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: items = [item for lst in self.items.values() for item in lst] else: @@ -150,7 +176,7 @@ class Command(BaseRunSpiderCommand): print("# Scraped Items ", "-" * 60) display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour) - def print_requests(self, lvl=None, colour=True): + def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None: if lvl is None: if self.requests: requests = self.requests[max(self.requests)] @@ -162,7 +188,7 @@ class Command(BaseRunSpiderCommand): print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) - def print_results(self, opts): + def print_results(self, opts: argparse.Namespace) -> None: colour = not opts.nocolour if opts.verbose: @@ -179,7 +205,14 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): + def _get_items_and_requests( + self, + spider_output: Iterable[Any], + opts: argparse.Namespace, + depth: int, + spider: Spider, + callback: Callable, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -188,14 +221,21 @@ class Command(BaseRunSpiderCommand): requests.append(x) return items, requests, opts, depth, spider, callback - def run_callback(self, response, callback, cb_kwargs=None): + def run_callback( + self, + response: Response, + callback: Callable, + cb_kwargs: Optional[Dict[str, Any]] = None, + ) -> Deferred: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d - def get_callback_from_rules(self, spider, response): + def get_callback_from_rules( + self, spider: Spider, response: Response + ) -> Union[Callable, str, None]: if getattr(spider, "rules", None): - for rule in spider.rules: + for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): return rule.callback or "parse" else: @@ -204,8 +244,10 @@ class Command(BaseRunSpiderCommand): "please specify a callback to use for parsing", {"spider": spider.name}, ) + return None - def set_spidercls(self, url, opts): + def set_spidercls(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: try: @@ -219,13 +261,14 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - def _start_requests(spider): + def _start_requests(spider: Spider) -> Iterable[Request]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: self.spidercls.start_requests = _start_requests - def start_parsing(self, url, opts): + def start_parsing(self, url: str, opts: argparse.Namespace) -> None: + assert self.crawler_process self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = list(self.crawler_process.crawlers)[0] self.crawler_process.start() @@ -233,7 +276,12 @@ class Command(BaseRunSpiderCommand): if not self.first_response: logger.error("No response downloaded for: %(url)s", {"url": url}) - def scraped_data(self, args): + def scraped_data( + self, + args: Tuple[ + List[Any], List[Request], argparse.Namespace, int, Spider, Callable + ], + ) -> List[Any]: items, requests, opts, depth, spider, callback = args if opts.pipelines: itemproc = self.pcrawler.engine.scraper.itemproc @@ -252,8 +300,14 @@ class Command(BaseRunSpiderCommand): return scraped_data - def _get_callback(self, *, spider, opts, response=None): - cb = None + def _get_callback( + self, + *, + spider: Spider, + opts: argparse.Namespace, + response: Optional[Response] = None, + ) -> Callable: + cb: Union[str, Callable, None] = None if response: cb = response.meta["_callback"] if not cb: @@ -270,6 +324,7 @@ class Command(BaseRunSpiderCommand): cb = "parse" if not callable(cb): + assert cb is not None cb_method = getattr(spider, cb, None) if callable(cb_method): cb = cb_method @@ -277,10 +332,13 @@ class Command(BaseRunSpiderCommand): raise ValueError( f"Cannot find callback {cb!r} in spider: {spider.name}" ) + assert callable(cb) return cb - def prepare_request(self, spider, request, opts): - def callback(response, **cb_kwargs): + def prepare_request( + self, spider: Spider, request: Request, opts: argparse.Namespace + ) -> Request: + def callback(response: Response, **cb_kwargs: Any) -> Deferred: # memorize first request if not self.first_response: self.first_response = response @@ -288,7 +346,7 @@ class Command(BaseRunSpiderCommand): cb = self._get_callback(spider=spider, opts=opts, response=response) # parse items and requests - depth = response.meta["_depth"] + depth: int = response.meta["_depth"] d = self.run_callback(response, cb, cb_kwargs) d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) @@ -311,13 +369,13 @@ class Command(BaseRunSpiderCommand): request.callback = callback return request - def process_options(self, args, opts): - BaseRunSpiderCommand.process_options(self, args, opts) + def process_options(self, args: List[str], opts: argparse.Namespace) -> None: + super().process_options(args, opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_request_meta(self, opts): + def process_request_meta(self, opts: argparse.Namespace) -> None: if opts.meta: try: opts.meta = json.loads(opts.meta) @@ -328,7 +386,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def process_request_cb_kwargs(self, opts): + def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: try: opts.cbkwargs = json.loads(opts.cbkwargs) @@ -339,7 +397,7 @@ class Command(BaseRunSpiderCommand): print_help=False, ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: # parse arguments if not len(args) == 1 or not is_url(args[0]): raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 58ed89a81..77850e7b5 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,9 +1,10 @@ +import argparse import sys from importlib import import_module from os import PathLike from pathlib import Path from types import ModuleType -from typing import Union +from typing import List, Union from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand): requires_project = False default_settings = {"SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options] " - def short_desc(self): + def short_desc(self) -> str: return "Run a self-contained spider (without creating a project)" - def long_desc(self): + def long_desc(self) -> str: return "Run the spider defined in the given file" - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError() filename = Path(args[0]) @@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand): raise UsageError(f"No spider found in file: {filename}\n") spidercls = spclasses.pop() + assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 318187204..dbda73b44 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,4 +1,6 @@ +import argparse import json +from typing import List from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -8,14 +10,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[options]" - def short_desc(self): + def short_desc(self) -> str: return "Get settings values" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--get", dest="get", metavar="SETTING", help="print raw setting value" ) @@ -44,7 +46,8 @@ class Command(ScrapyCommand): help="print setting value, interpreted as a list", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: + assert self.crawler_process settings = self.crawler_process.settings if opts.get: s = settings.get(opts.get) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index f72a23c6a..668c95a7b 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,9 +4,9 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from argparse import Namespace +from argparse import ArgumentParser, Namespace from threading import Thread -from typing import List, Type +from typing import Any, Dict, List, Type from scrapy import Spider from scrapy.commands import ScrapyCommand @@ -24,20 +24,20 @@ class Command(ScrapyCommand): "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", } - def syntax(self): + def syntax(self) -> str: return "[url|file]" - def short_desc(self): + def short_desc(self) -> str: return "Interactive scraping console" - def long_desc(self): + def long_desc(self) -> str: return ( "Interactive console for scraping the given url or file. " "Use ./file.html syntax or full path for local file." ) - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "-c", dest="code", @@ -52,7 +52,7 @@ class Command(ScrapyCommand): help="do not handle HTTP 3xx status codes and print response as-is", ) - def update_vars(self, vars): + def update_vars(self, vars: Dict[str, Any]) -> None: """You can use this function to update the Scrapy objects that will be available in the shell """ @@ -88,7 +88,8 @@ class Command(ScrapyCommand): shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) shell.start(url=url, redirect=not opts.no_redirect) - def _start_crawler_thread(self): + def _start_crawler_thread(self) -> None: + assert self.crawler_process t = Thread( target=self.crawler_process.start, kwargs={"stop_after_crawl": False, "install_signal_handlers": False}, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index fde609c6f..58c1aa28f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,3 +1,4 @@ +import argparse import os import re import string @@ -5,13 +6,14 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION +from typing import List, Tuple, Union import scrapy from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.template import render_templatefile, string_camelcase -TEMPLATES_TO_RENDER = ( +TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = ( ("scrapy.cfg",), ("${project_name}", "settings.py.tmpl"), ("${project_name}", "items.py.tmpl"), @@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = ( IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn") -def _make_writable(path): +def _make_writable(path: Union[str, os.PathLike]) -> None: current_permissions = os.stat(path).st_mode os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) @@ -31,14 +33,14 @@ class Command(ScrapyCommand): requires_project = False default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return " [project_dir]" - def short_desc(self): + def short_desc(self) -> str: return "Create new project" - def _is_valid_name(self, project_name): - def _module_exists(module_name): + def _is_valid_name(self, project_name: str) -> bool: + def _module_exists(module_name: str) -> bool: spec = find_spec(module_name) return spec is not None and spec.loader is not None @@ -53,7 +55,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src: Path, dst: Path): + def _copytree(self, src: Path, dst: Path) -> None: """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -84,7 +86,7 @@ class Command(ScrapyCommand): copystat(src, dst) _make_writable(dst) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if len(args) not in (1, 2): raise UsageError() @@ -105,7 +107,9 @@ class Command(ScrapyCommand): return self._copytree(Path(self.templates_dir), project_dir.resolve()) - move(project_dir / "module", project_dir / project_name) + # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case + # See https://bugs.python.org/issue32689 + move(project_dir / "module", project_dir / project_name) # type: ignore[arg-type] for paths in TEMPLATES_TO_RENDER: tplfile = Path( project_dir, diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 47582866b..f057e8544 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,3 +1,6 @@ +import argparse +from typing import List + import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions @@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True} - def syntax(self): + def syntax(self) -> str: return "[-v]" - def short_desc(self): + def short_desc(self) -> str: return "Print Scrapy version" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) parser.add_argument( "--verbose", "-v", @@ -22,7 +25,7 @@ class Command(ScrapyCommand): help="also display twisted/python/platform info (useful for bug reports)", ) - def run(self, args, opts): + def run(self, args: List[str], opts: argparse.Namespace) -> None: if opts.verbose: versions = scrapy_components_versions() width = max(len(n) for (n, _) in versions) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index ebdfa10a8..21679e3aa 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,21 +1,28 @@ import argparse +import logging from scrapy.commands import fetch +from scrapy.http import Response, TextResponse from scrapy.utils.response import open_in_browser +logger = logging.getLogger(__name__) + class Command(fetch.Command): - def short_desc(self): + def short_desc(self) -> str: return "Open URL in browser, as seen by Scrapy" - def long_desc(self): + def long_desc(self) -> str: return ( "Fetch a URL using the Scrapy downloader and show its contents in a browser" ) - def add_options(self, parser): + def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) parser.add_argument("--headers", help=argparse.SUPPRESS) - def _print_response(self, response, opts): + def _print_response(self, response: Response, opts: argparse.Namespace) -> None: + if not isinstance(response, TextResponse): + logger.error("Cannot view a non-text response.") + return open_in_browser(response) diff --git a/scrapy/shell.py b/scrapy/shell.py index 63ea33892..fac42e8a2 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst import os import signal +from typing import Any, Callable, Dict, Optional, Tuple, Union from itemadapter import is_item from twisted.internet import defer, threads @@ -26,18 +27,32 @@ from scrapy.utils.response import open_in_browser class Shell: - relevant_classes = (Crawler, Spider, Request, Response, Settings) + relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings) - def __init__(self, crawler, update_vars=None, code=None): - self.crawler = crawler - self.update_vars = update_vars or (lambda x: None) - self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) - self.spider = None - self.inthread = not threadable.isInIOThread() - self.code = code - self.vars = {} + def __init__( + self, + crawler: Crawler, + update_vars: Optional[Callable[[Dict[str, Any]], None]] = None, + code: Optional[str] = None, + ): + self.crawler: Crawler = crawler + self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or ( + lambda x: None + ) + self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"]) + self.spider: Optional[Spider] = None + self.inthread: bool = not threadable.isInIOThread() + self.code: Optional[str] = code + self.vars: Dict[str, Any] = {} - def start(self, url=None, request=None, response=None, spider=None, redirect=True): + def start( + self, + url: Optional[str] = None, + request: Optional[Request] = None, + response: Optional[Response] = None, + spider: Optional[Spider] = None, + redirect: bool = True, + ) -> None: # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: @@ -77,7 +92,7 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request, spider): + def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -85,10 +100,11 @@ class Shell: spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) + assert self.crawler.engine self.crawler.engine.crawl(request) return d - def _open_spider(self, request, spider): + def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider: if self.spider: return self.spider @@ -96,11 +112,18 @@ class Shell: spider = self.crawler.spider or self.crawler._create_spider() self.crawler.spider = spider + assert self.crawler.engine self.crawler.engine.open_spider(spider, close_if_idle=False) self.spider = spider return spider - def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): + def fetch( + self, + request_or_url: Union[Request, str], + spider: Optional[Spider] = None, + redirect: bool = True, + **kwargs: Any, + ) -> None: from twisted.internet import reactor if isinstance(request_or_url, Request): @@ -123,7 +146,12 @@ class Shell: pass self.populate_vars(response, request, spider) - def populate_vars(self, response=None, request=None, spider=None): + def populate_vars( + self, + response: Optional[Response] = None, + request: Optional[Request] = None, + spider: Optional[Spider] = None, + ) -> None: import scrapy self.vars["scrapy"] = scrapy @@ -141,10 +169,10 @@ class Shell: if not self.code: self.vars["banner"] = self.get_help() - def print_help(self): + def print_help(self) -> None: print(self.get_help()) - def get_help(self): + def get_help(self) -> str: b = [] b.append("Available Scrapy objects:") b.append( @@ -168,11 +196,11 @@ class Shell: return "\n".join(f"[s] {line}" for line in b) - def _is_relevant(self, value): + def _is_relevant(self, value: Any) -> bool: return isinstance(value, self.relevant_classes) or is_item(value) -def inspect_response(response, spider): +def inspect_response(response: Response, spider: Spider) -> None: """Open a shell to inspect the given response""" # Shell.start removes the SIGINT handler, so save it and re-add it after # the shell has closed @@ -181,7 +209,7 @@ def inspect_response(response, spider): signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request): +def _request_deferred(request: Request) -> defer.Deferred: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -195,12 +223,12 @@ def _request_deferred(request): request_callback = request.callback request_errback = request.errback - def _restore_callbacks(result): + def _restore_callbacks(result: Any) -> Any: request.callback = request_callback request.errback = request_errback return result - d = defer.Deferred() + d: defer.Deferred = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallbacks(request.callback, request.errback) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 100f040bb..bf1803115 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,17 +1,27 @@ from functools import wraps +from typing import Any, Callable, Dict, Iterable, Optional + +EmbedFuncT = Callable[..., None] +KnownShellsT = Dict[str, Callable[..., EmbedFuncT]] -def _embed_ipython_shell(namespace={}, banner=""): +def _embed_ipython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start an IPython Shell""" try: from IPython.terminal.embed import InteractiveShellEmbed from IPython.terminal.ipapp import load_default_config except ImportError: - from IPython.frontend.terminal.embed import InteractiveShellEmbed - from IPython.frontend.terminal.ipapp import load_default_config + from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] + InteractiveShellEmbed, + ) + from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] + load_default_config, + ) @wraps(_embed_ipython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: config = load_default_config() # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -26,30 +36,36 @@ def _embed_ipython_shell(namespace={}, banner=""): return wrapper -def _embed_bpython_shell(namespace={}, banner=""): +def _embed_bpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a bpython shell""" import bpython @wraps(_embed_bpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper -def _embed_ptpython_shell(namespace={}, banner=""): +def _embed_ptpython_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl @wraps(_embed_ptpython_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) return wrapper -def _embed_standard_shell(namespace={}, banner=""): +def _embed_standard_shell( + namespace: Dict[str, Any] = {}, banner: str = "" +) -> EmbedFuncT: """Start a standard python shell""" import code @@ -63,13 +79,13 @@ def _embed_standard_shell(namespace={}, banner=""): readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) - def wrapper(namespace=namespace, banner=""): + def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper -DEFAULT_PYTHON_SHELLS = { +DEFAULT_PYTHON_SHELLS: KnownShellsT = { "ptpython": _embed_ptpython_shell, "ipython": _embed_ipython_shell, "bpython": _embed_bpython_shell, @@ -77,7 +93,9 @@ DEFAULT_PYTHON_SHELLS = { } -def get_shell_embed_func(shells=None, known_shells=None): +def get_shell_embed_func( + shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None +) -> Any: """Return the first acceptable shell-embed function from a given list of shell names. """ @@ -95,7 +113,11 @@ def get_shell_embed_func(shells=None, known_shells=None): continue -def start_python_console(namespace=None, banner="", shells=None): +def start_python_console( + namespace: Optional[Dict[str, Any]] = None, + banner: str = "", + shells: Optional[Iterable[str]] = None, +) -> None: """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. """ diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 63a484b42..a0b06f75c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -3,24 +3,27 @@ This module provides some useful functions for working with scrapy.http.Response objects """ +from __future__ import annotations + import os import re import tempfile import webbrowser -from typing import Any, Callable, Iterable, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html -import scrapy -from scrapy.http.response import Response from scrapy.utils.python import to_bytes, to_unicode -_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() +if TYPE_CHECKING: + from scrapy.http import Response, TextResponse + +_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary() -def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: +def get_base_url(response: TextResponse) -> str: """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] @@ -30,13 +33,13 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str: return _baseurl_cache[response] -_metaref_cache: ( - "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" -) = WeakKeyDictionary() +_metaref_cache: WeakKeyDictionary[ + Response, Union[Tuple[None, None], Tuple[float, str]] +] = WeakKeyDictionary() def get_meta_refresh( - response: "scrapy.http.response.text.TextResponse", + response: TextResponse, ignore_tags: Iterable[str] = ("script", "noscript"), ) -> Union[Tuple[None, None], Tuple[float, str]]: """Parse the http-equiv refresh parameter from the given response""" @@ -68,10 +71,7 @@ def _remove_html_comments(body): def open_in_browser( - response: Union[ - "scrapy.http.response.html.HtmlResponse", - "scrapy.http.response.text.TextResponse", - ], + response: TextResponse, _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: """Open *response* in a local web browser, adjusting the `base tag`_ for From 8985a04bd1328cd6156a7c33a5db74ad8c81802f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Mar 2024 10:57:03 +0500 Subject: [PATCH 37/45] Full typing for scrapy/exporters.py. (#6275) --- scrapy/exporters.py | 150 ++++++++++++++++++-------------- scrapy/extensions/feedexport.py | 11 +-- 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 79fd4e56f..fb4998099 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats. """ import csv -import io import marshal import pickle # nosec import pprint -from collections.abc import Mapping +from io import BytesIO, TextIOWrapper +from json import JSONEncoder +from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union from xml.sax.saxutils import XMLGenerator # nosec +from xml.sax.xmlreader import AttributesImpl # nosec from itemadapter import ItemAdapter, is_item -from scrapy.item import Item +from scrapy.item import Field, Item from scrapy.utils.python import is_listlike, to_bytes, to_unicode from scrapy.utils.serialize import ScrapyJSONEncoder @@ -29,36 +31,42 @@ __all__ = [ class BaseItemExporter: - def __init__(self, *, dont_fail=False, **kwargs): - self._kwargs = kwargs + def __init__(self, *, dont_fail: bool = False, **kwargs: Any): + self._kwargs: Dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: """Configure the exporter by popping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options (useful for using with keyword arguments in subclasses ``__init__`` methods) """ - self.encoding = options.pop("encoding", None) - self.fields_to_export = options.pop("fields_to_export", None) - self.export_empty_fields = options.pop("export_empty_fields", False) - self.indent = options.pop("indent", None) + self.encoding: Optional[str] = options.pop("encoding", None) + self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = ( + options.pop("fields_to_export", None) + ) + self.export_empty_fields: bool = options.pop("export_empty_fields", False) + self.indent: Optional[int] = options.pop("indent", None) if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") - def export_item(self, item): + def export_item(self, item: Any) -> None: raise NotImplementedError - def serialize_field(self, field, name, value): - serializer = field.get("serializer", lambda x: x) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) - def start_exporting(self): + def start_exporting(self) -> None: pass - def finish_exporting(self): + def finish_exporting(self) -> None: pass - def _get_serialized_fields(self, item, default_value=None, include_empty=None): + def _get_serialized_fields( + self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None + ) -> Iterable[Tuple[str, Any]]: """Return the fields to export as an iterable of tuples (name, serialized_value) """ @@ -100,22 +108,22 @@ class BaseItemExporter: class JsonLinesItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file self._kwargs.setdefault("ensure_ascii", not self.encoding) - self.encoder = ScrapyJSONEncoder(**self._kwargs) + self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs) - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) + "\n" self.file.write(to_bytes(data, self.encoding)) class JsonItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(dont_fail=True, **kwargs) - self.file = file + self.file: BytesIO = file # there is a small difference between the behaviour or JsonItemExporter.indent # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent # the addition of newlines everywhere @@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter): self.encoder = ScrapyJSONEncoder(**self._kwargs) self.first_item = True - def _beautify_newline(self): + def _beautify_newline(self) -> None: if self.indent is not None: self.file.write(b"\n") - def _add_comma_after_first(self): + def _add_comma_after_first(self) -> None: if self.first_item: self.first_item = False else: self.file.write(b",") self._beautify_newline() - def start_exporting(self): + def start_exporting(self) -> None: self.file.write(b"[") self._beautify_newline() - def finish_exporting(self): + def finish_exporting(self) -> None: self._beautify_newline() self.file.write(b"]") - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) data = to_bytes(self.encoder.encode(itemdict), self.encoding) self._add_comma_after_first() @@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter): class XmlItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): self.item_element = kwargs.pop("item_element", "item") self.root_element = kwargs.pop("root_element", "items") super().__init__(**kwargs) @@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter): self.encoding = "utf-8" self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self, new_item=False): + def _beautify_newline(self, new_item: bool = False) -> None: if self.indent is not None and (self.indent > 0 or new_item): self.xg.characters("\n") - def _beautify_indent(self, depth=1): + def _beautify_indent(self, depth: int = 1) -> None: if self.indent: self.xg.characters(" " * self.indent * depth) - def start_exporting(self): + def start_exporting(self) -> None: self.xg.startDocument() - self.xg.startElement(self.root_element, {}) + self.xg.startElement(self.root_element, AttributesImpl({})) self._beautify_newline(new_item=True) - def export_item(self, item): + def export_item(self, item: Any) -> None: self._beautify_indent(depth=1) - self.xg.startElement(self.item_element, {}) + self.xg.startElement(self.item_element, AttributesImpl({})) self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=""): self._export_xml_field(name, value, depth=2) @@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(self.item_element) self._beautify_newline(new_item=True) - def finish_exporting(self): + def finish_exporting(self) -> None: self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value, depth): + def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None: self._beautify_indent(depth=depth) - self.xg.startElement(name, {}) + self.xg.startElement(name, AttributesImpl({})) if hasattr(serialized_value, "items"): self._beautify_newline() for subname, value in serialized_value.items(): @@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter): class CsvItemExporter(BaseItemExporter): def __init__( self, - file, - include_headers_line=True, - join_multivalued=",", - errors=None, - **kwargs, + file: BytesIO, + include_headers_line: bool = True, + join_multivalued: str = ",", + errors: Optional[str] = None, + **kwargs: Any, ): super().__init__(dont_fail=True, **kwargs) if not self.encoding: self.encoding = "utf-8" self.include_headers_line = include_headers_line - self.stream = io.TextIOWrapper( + self.stream = TextIOWrapper( file, line_buffering=False, write_through=True, @@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter): self._headers_not_written = True self._join_multivalued = join_multivalued - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._join_if_needed) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed) return serializer(value) - def _join_if_needed(self, value): + def _join_if_needed(self, value: Any) -> Any: if isinstance(value, (list, tuple)): try: return self._join_multivalued.join(value) @@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter): pass return value - def export_item(self, item): + def export_item(self, item: Any) -> None: if self._headers_not_written: self._headers_not_written = False self._write_headers_and_set_fields_to_export(item) @@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) - def finish_exporting(self): + def finish_exporting(self) -> None: self.stream.detach() # Avoid closing the wrapped file. - def _build_row(self, values): + def _build_row(self, values: Iterable[Any]) -> Iterable[Any]: for s in values: try: yield to_unicode(s, self.encoding) except TypeError: yield s - def _write_headers_and_set_fields_to_export(self, item): + def _write_headers_and_set_fields_to_export(self, item: Any) -> None: if self.include_headers_line: if not self.fields_to_export: # use declared field names, or keys if the item is a dict self.fields_to_export = ItemAdapter(item).field_names() + fields: Iterable[str] if isinstance(self.fields_to_export, Mapping): fields = self.fields_to_export.values() else: + assert self.fields_to_export fields = self.fields_to_export row = list(self._build_row(fields)) self.csv_writer.writerow(row) class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=4, **kwargs): + def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any): super().__init__(**kwargs) - self.file = file - self.protocol = protocol + self.file: BytesIO = file + self.protocol: int = protocol - def export_item(self, item): + def export_item(self, item: Any) -> None: d = dict(self._get_serialized_fields(item)) pickle.dump(d, self.file, self.protocol) @@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter): opened in binary mode, a :class:`~io.BytesIO` object, etc) """ - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: marshal.dump(dict(self._get_serialized_fields(item)), self.file) class PprintItemExporter(BaseItemExporter): - def __init__(self, file, **kwargs): + def __init__(self, file: BytesIO, **kwargs: Any): super().__init__(**kwargs) - self.file = file + self.file: BytesIO = file - def export_item(self, item): + def export_item(self, item: Any) -> None: itemdict = dict(self._get_serialized_fields(item)) self.file.write(to_bytes(pprint.pformat(itemdict) + "\n")) @@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter): .. _msgpack: https://pypi.org/project/msgpack/ """ - def _configure(self, options, dont_fail=False): + def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None: super()._configure(options, dont_fail) if not self.encoding: self.encoding = "utf-8" - def serialize_field(self, field, name, value): - serializer = field.get("serializer", self._serialize_value) + def serialize_field( + self, field: Union[Mapping[str, Any], Field], name: str, value: Any + ) -> Any: + serializer: Callable[[Any], Any] = field.get( + "serializer", self._serialize_value + ) return serializer(value) - def _serialize_value(self, value): + def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) if is_item(value): @@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter): return to_unicode(value, encoding=self.encoding) return value - def _serialize_item(self, item): + def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]: for key, value in ItemAdapter(item).items(): yield key, self._serialize_value(value) - def export_item(self, item): - result = dict(self._get_serialized_fields(item)) + def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]: # type: ignore[override] + result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item)) return result diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e5e363b52..3b0dd804e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,7 +11,7 @@ import warnings from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -21,6 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exporters import BaseItemExporter from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings @@ -324,12 +325,12 @@ class FeedSlot: filter, feed_options, spider, - exporters, + exporters: Dict[str, Type[BaseItemExporter]], settings, crawler, ): self.file = None - self.exporter = None + self.exporter: Optional[BaseItemExporter] = None self.storage = storage # feed params self.batch_id = batch_id @@ -341,7 +342,7 @@ class FeedSlot: # exporter params self.feed_options = feed_options self.spider = spider - self.exporters = exporters + self.exporters: Dict[str, Type[BaseItemExporter]] = exporters self.settings = settings self.crawler = crawler # flags @@ -373,7 +374,7 @@ class FeedSlot: def _get_instance(self, objcls, *args, **kwargs): return build_from_crawler(objcls, self.crawler, *args, **kwargs) - def _get_exporter(self, file, format, *args, **kwargs): + def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter: return self._get_instance(self.exporters[format], file, *args, **kwargs) def finish_exporting(self): From 421e08dd4a4d5ed2acf3cd0fca4c7bfa5a6d3eb2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 8 Mar 2024 10:59:56 +0500 Subject: [PATCH 38/45] Full typing for scrapy/extensions, part 1. (#6276) --- scrapy/extensions/closespider.py | 52 +++++++++++++++++++++----------- scrapy/extensions/corestats.py | 33 +++++++++++++------- scrapy/extensions/debug.py | 25 ++++++++++----- scrapy/extensions/logstats.py | 50 ++++++++++++++++++------------ scrapy/extensions/memdebug.py | 22 ++++++++++---- scrapy/extensions/memusage.py | 51 ++++++++++++++++++++----------- scrapy/mail.py | 12 ++++++-- scrapy/utils/engine.py | 13 ++++---- 8 files changed, 171 insertions(+), 87 deletions(-) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 4307b4170..812b3553c 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -4,20 +4,31 @@ conditions are met. See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging from collections import defaultdict +from typing import TYPE_CHECKING, Any, DefaultDict, Dict -from scrapy import signals +from twisted.python.failure import Failure + +from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class CloseSpider: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler - self.close_on = { + self.close_on: Dict[str, Any] = { "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"), "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"), "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"), @@ -28,7 +39,7 @@ class CloseSpider: if not any(self.close_on.values()): raise NotConfigured - self.counter = defaultdict(int) + self.counter: DefaultDict[str, int] = defaultdict(int) if self.close_on.get("errorcount"): crawler.signals.connect(self.error_count, signal=signals.spider_error) @@ -39,8 +50,8 @@ class CloseSpider: if self.close_on.get("itemcount"): crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) if self.close_on.get("timeout_no_item"): - self.timeout_no_item = self.close_on["timeout_no_item"] - self.items_in_period = 0 + self.timeout_no_item: int = self.close_on["timeout_no_item"] + self.items_in_period: int = 0 crawler.signals.connect( self.spider_opened_no_item, signal=signals.spider_opened ) @@ -50,22 +61,25 @@ class CloseSpider: crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def error_count(self, failure, response, spider): + def error_count(self, failure: Failure, response: Response, spider: Spider) -> None: self.counter["errorcount"] += 1 if self.counter["errorcount"] == self.close_on["errorcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_errorcount") - def page_count(self, response, request, spider): + def page_count(self, response: Response, request: Request, spider: Spider) -> None: self.counter["pagecount"] += 1 if self.counter["pagecount"] == self.close_on["pagecount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_pagecount") - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: from twisted.internet import reactor + assert self.crawler.engine self.task = reactor.callLater( self.close_on["timeout"], self.crawler.engine.close_spider, @@ -73,21 +87,22 @@ class CloseSpider: reason="closespider_timeout", ) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.counter["itemcount"] += 1 if self.counter["itemcount"] == self.close_on["itemcount"]: + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_itemcount") - def spider_closed(self, spider): - task = getattr(self, "task", False) + def spider_closed(self, spider: Spider) -> None: + task = getattr(self, "task", None) if task and task.active(): task.cancel() - task_no_item = getattr(self, "task_no_item", False) + task_no_item = getattr(self, "task_no_item", None) if task_no_item and task_no_item.running: task_no_item.stop() - def spider_opened_no_item(self, spider): + def spider_opened_no_item(self, spider: Spider) -> None: from twisted.internet import task self.task_no_item = task.LoopingCall(self._count_items_produced, spider) @@ -98,10 +113,10 @@ class CloseSpider: f"{self.timeout_no_item} seconds." ) - def item_scraped_no_item(self, item, spider): + def item_scraped_no_item(self, item: Any, spider: Spider) -> None: self.items_in_period += 1 - def _count_items_produced(self, spider): + def _count_items_produced(self, spider: Spider) -> None: if self.items_in_period >= 1: self.items_in_period = 0 else: @@ -109,4 +124,5 @@ class CloseSpider: f"Closing spider since no items were produced in the last " f"{self.timeout_no_item} seconds." ) + assert self.crawler.engine self.crawler.engine.close_spider(spider, "closespider_timeout_no_item") diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 717c249d9..f3ac19623 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -2,18 +2,28 @@ Extension for collecting core stats like items scraped and start/finish times """ -from datetime import datetime, timezone +from __future__ import annotations -from scrapy import signals +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Optional + +from scrapy import Spider, signals +from scrapy.crawler import Crawler +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class CoreStats: - def __init__(self, stats): - self.stats = stats - self.start_time = None + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats + self.start_time: Optional[datetime] = None @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) @@ -22,11 +32,12 @@ class CoreStats: crawler.signals.connect(o.response_received, signal=signals.response_received) return o - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: self.start_time = datetime.now(tz=timezone.utc) self.stats.set_value("start_time", self.start_time, spider=spider) - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: + assert self.start_time is not None finish_time = datetime.now(tz=timezone.utc) elapsed_time = finish_time - self.start_time elapsed_time_seconds = elapsed_time.total_seconds() @@ -36,13 +47,13 @@ class CoreStats: self.stats.set_value("finish_time", finish_time, spider=spider) self.stats.set_value("finish_reason", reason, spider=spider) - def item_scraped(self, item, spider): + def item_scraped(self, item: Any, spider: Spider) -> None: self.stats.inc_value("item_scraped_count", spider=spider) - def response_received(self, spider): + def response_received(self, spider: Spider) -> None: self.stats.inc_value("response_received_count", spider=spider) - def item_dropped(self, item, spider, exception): + def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None: reason = exception.__class__.__name__ self.stats.inc_value("item_dropped_count", spider=spider) self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider) diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 1b6c7777f..26726b662 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -4,22 +4,31 @@ Extensions for debugging Scrapy See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import signal import sys import threading import traceback from pdb import Pdb +from types import FrameType +from typing import TYPE_CHECKING, Optional +from scrapy.crawler import Crawler from scrapy.utils.engine import format_engine_status from scrapy.utils.trackref import format_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class StackTraceDump: - def __init__(self, crawler=None): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler try: signal.signal(signal.SIGUSR2, self.dump_stacktrace) signal.signal(signal.SIGQUIT, self.dump_stacktrace) @@ -28,10 +37,11 @@ class StackTraceDump: pass @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def dump_stacktrace(self, signum, frame): + def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None: + assert self.crawler.engine log_args = { "stackdumps": self._thread_stacks(), "enginestatus": format_engine_status(self.crawler.engine), @@ -44,7 +54,7 @@ class StackTraceDump: extra={"crawler": self.crawler}, ) - def _thread_stacks(self): + def _thread_stacks(self) -> str: id2name = dict((th.ident, th.name) for th in threading.enumerate()) dumps = "" for id_, frame in sys._current_frames().items(): @@ -55,12 +65,13 @@ class StackTraceDump: class Debugger: - def __init__(self): + def __init__(self) -> None: try: signal.signal(signal.SIGUSR2, self._enter_debugger) except AttributeError: # win32 platforms don't support SIGUSR signals pass - def _enter_debugger(self, signum, frame): + def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None: + assert frame Pdb().set_trace(frame.f_back) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 9f63e9c4b..2388afa75 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -1,9 +1,18 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Optional, Tuple, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) @@ -14,30 +23,31 @@ class LogStats: * IPM - Items per Minute """ - def __init__(self, stats, interval=60.0): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None + def __init__(self, stats: StatsCollector, interval: float = 60.0): + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured + assert crawler.stats o = cls(crawler.stats, interval) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.pagesprev = 0 - self.itemsprev = 0 + def spider_opened(self, spider: Spider) -> None: + self.pagesprev: int = 0 + self.itemsprev: int = 0 self.task = task.LoopingCall(self.log, spider) self.task.start(self.interval) - def log(self, spider): + def log(self, spider: Spider) -> None: self.calculate_stats() msg = ( @@ -52,14 +62,14 @@ class LogStats: } logger.info(msg, log_args, extra={"spider": spider}) - def calculate_stats(self): - self.items = self.stats.get_value("item_scraped_count", 0) - self.pages = self.stats.get_value("response_received_count", 0) - self.irate = (self.items - self.itemsprev) * self.multiplier - self.prate = (self.pages - self.pagesprev) * self.multiplier + def calculate_stats(self) -> None: + self.items: int = self.stats.get_value("item_scraped_count", 0) + self.pages: int = self.stats.get_value("response_received_count", 0) + self.irate: float = (self.items - self.itemsprev) * self.multiplier + self.prate: float = (self.pages - self.pagesprev) * self.multiplier self.pagesprev, self.itemsprev = self.pages, self.items - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: if self.task and self.task.running: self.task.stop() @@ -67,7 +77,9 @@ class LogStats: self.stats.set_value("responses_per_minute", rpm_final) self.stats.set_value("items_per_minute", ipm_final) - def calculate_final_stats(self, spider): + def calculate_final_stats( + self, spider: Spider + ) -> Union[Tuple[None, None], Tuple[float, float]]: start_time = self.stats.get_value("start_time") finished_time = self.stats.get_value("finished_time") diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 03ede0681..f304e1bf2 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -4,26 +4,36 @@ MemoryDebugger extension See documentation in docs/topics/extensions.rst """ -import gc +from __future__ import annotations -from scrapy import signals +import gc +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.trackref import live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MemoryDebugger: - def __init__(self, stats): - self.stats = stats + def __init__(self, stats: StatsCollector): + self.stats: StatsCollector = stats @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("MEMDEBUG_ENABLED"): raise NotConfigured + assert crawler.stats o = cls(crawler.stats) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: gc.collect() self.stats.set_value( "memdebug/gc_garbage_count", len(gc.garbage), spider=spider diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 4d4501c44..9de06b24d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -4,24 +4,32 @@ MemoryUsage extension See documentation in docs/topics/extensions.rst """ +from __future__ import annotations + import logging import socket import sys from importlib import import_module from pprint import pformat +from typing import TYPE_CHECKING, List from twisted.internet import task from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.utils.engine import get_engine_status +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) class MemoryUsage: - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("MEMUSAGE_ENABLED"): raise NotConfigured try: @@ -30,32 +38,33 @@ class MemoryUsage: except ImportError: raise NotConfigured - self.crawler = crawler - self.warned = False - self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") - self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 - self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 - self.check_interval = crawler.settings.getfloat( + self.crawler: Crawler = crawler + self.warned: bool = False + self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL") + self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 + self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 + self.check_interval: float = crawler.settings.getfloat( "MEMUSAGE_CHECK_INTERVAL_SECONDS" ) - self.mail = MailSender.from_settings(crawler.settings) + self.mail: MailSender = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def get_virtual_size(self): - size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss + def get_virtual_size(self) -> int: + size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss if sys.platform != "darwin": # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size - def engine_started(self): + def engine_started(self) -> None: + assert self.crawler.stats self.crawler.stats.set_value("memusage/startup", self.get_virtual_size()) - self.tasks = [] + self.tasks: List[task.LoopingCall] = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) tsk.start(self.check_interval, now=True) @@ -68,15 +77,18 @@ class MemoryUsage: self.tasks.append(tsk) tsk.start(self.check_interval, now=True) - def engine_stopped(self): + def engine_stopped(self) -> None: for tsk in self.tasks: if tsk.running: tsk.stop() - def update(self): + def update(self) -> None: + assert self.crawler.stats self.crawler.stats.max_value("memusage/max", self.get_virtual_size()) - def _check_limit(self): + def _check_limit(self) -> None: + assert self.crawler.engine + assert self.crawler.stats peak_mem_usage = self.get_virtual_size() if peak_mem_usage > self.limit: self.crawler.stats.set_value("memusage/limit_reached", 1) @@ -106,9 +118,10 @@ class MemoryUsage: {"virtualsize": peak_mem_usage / 1024 / 1024}, ) - def _check_warning(self): + def _check_warning(self) -> None: if self.warned: # warn only once return + assert self.crawler.stats if self.get_virtual_size() > self.warning: self.crawler.stats.set_value("memusage/warning_reached", 1) mem = self.warning / 1024 / 1024 @@ -126,8 +139,10 @@ class MemoryUsage: self.crawler.stats.set_value("memusage/warning_notified", 1) self.warned = True - def _send_report(self, rcpts, subject): + def _send_report(self, rcpts: List[str], subject: str) -> None: """send notification mail with some additional useful info""" + assert self.crawler.engine + assert self.crawler.stats stats = self.crawler.stats s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n" s += f"Maximum memory usage : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n" diff --git a/scrapy/mail.py b/scrapy/mail.py index 4b18b6003..dce33fcdf 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -4,6 +4,8 @@ Mail sending helpers See documentation in docs/topics/email.rst """ +from __future__ import annotations + import logging from email import encoders as Encoders from email.mime.base import MIMEBase @@ -12,14 +14,20 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from typing import TYPE_CHECKING from twisted import version as twisted_version from twisted.internet import defer, ssl from twisted.python.versions import Version +from scrapy.settings import BaseSettings from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -56,7 +64,7 @@ class MailSender: self.debug = debug @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings: BaseSettings) -> Self: return cls( smtphost=settings["MAIL_HOST"], mailfrom=settings["MAIL_FROM"], @@ -203,7 +211,7 @@ class MailSender: to_addrs, msg, d, - **factory_keywords + **factory_keywords, ) factory.noisy = False return factory diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 0b2722663..fdcf484d4 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,14 +1,15 @@ """Some debugging functions for working with the Scrapy engine""" +from __future__ import annotations + # used in global tests code from time import time # noqa: F401 -from typing import TYPE_CHECKING, Any, List, Tuple +from typing import Any, List, Tuple -if TYPE_CHECKING: - from scrapy.core.engine import ExecutionEngine +from scrapy.core.engine import ExecutionEngine -def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: +def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]: """Return a report of the current engine status""" tests = [ "time()-engine.start_time", @@ -37,7 +38,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]: return checks -def format_engine_status(engine: "ExecutionEngine") -> str: +def format_engine_status(engine: ExecutionEngine) -> str: checks = get_engine_status(engine) s = "Execution engine status\n\n" for test, result in checks: @@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str: return s -def print_engine_status(engine: "ExecutionEngine") -> None: +def print_engine_status(engine: ExecutionEngine) -> None: print(format_engine_status(engine)) From 6e84648c0717642b069249225857019a87de54b9 Mon Sep 17 00:00:00 2001 From: pengqiseven <134899215+pengqiseven@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:03:06 +0800 Subject: [PATCH 39/45] Fix some comments (#6285) Signed-off-by: pengqiseven Co-authored-by: pengqiseven --- sep/sep-018.rst | 2 +- tests/test_utils_python.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 9ac62c090..13ab501ed 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -140,7 +140,7 @@ Example: The data flow with Spider Middleware v2 is as follows: -1. When a response arrives from the engine, it it passed through all the spider +1. When a response arrives from the engine, it is passed through all the spider middlewares (in descending order). The result of each middleware ``process_response`` is kept and then returned along with the spider callback result diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 80d2e8da1..1d1d19146 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, ["x", "y"])) a.y = 1 - # differente attributes + # different attributes self.assertFalse(equal_attributes(a, b, ["x", "y"])) # test callable From 642af407049a5ce8e76b7999c68333670c8c8622 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 11 Mar 2024 14:09:09 +0500 Subject: [PATCH 40/45] Full typing for scrapy/extensions, part 2. (#6279) --- docs/topics/telnetconsole.rst | 4 +- scrapy/extensions/periodic_log.py | 89 +++++++++++++++++------------ scrapy/extensions/postprocessing.py | 4 +- scrapy/extensions/spiderstate.py | 26 ++++++--- scrapy/extensions/statsmailer.py | 31 +++++++--- scrapy/extensions/telnet.py | 37 +++++++----- scrapy/extensions/throttle.py | 49 ++++++++++------ scrapy/mail.py | 12 ++-- scrapy/utils/reactor.py | 8 +-- 9 files changed, 165 insertions(+), 95 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 832829b75..0e4a8fa6c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -172,8 +172,8 @@ TELNETCONSOLE_PORT Default: ``[6023, 6073]`` -The port range to use for the telnet console. If set to ``None`` or ``0``, a -dynamically assigned port is used. +The port range to use for the telnet console. If set to ``None``, a dynamically +assigned port is used. .. setting:: TELNETCONSOLE_HOST diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 2d557f123..9567f948a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,12 +1,22 @@ +from __future__ import annotations + import logging from datetime import datetime, timezone +from json import JSONEncoder +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from twisted.internet import task -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.statscollectors import StatsCollector from scrapy.utils.serialize import ScrapyJSONEncoder +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + logger = logging.getLogger(__name__) @@ -15,32 +25,34 @@ class PeriodicLog: def __init__( self, - stats, - interval=60.0, - ext_stats={}, - ext_delta={}, - ext_timing_enabled=False, + stats: StatsCollector, + interval: float = 60.0, + ext_stats: Dict[str, Any] = {}, + ext_delta: Dict[str, Any] = {}, + ext_timing_enabled: bool = False, ): - self.stats = stats - self.interval = interval - self.multiplier = 60.0 / self.interval - self.task = None - self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4) - self.ext_stats_enabled = bool(ext_stats) - self.ext_stats_include = ext_stats.get("include", []) - self.ext_stats_exclude = ext_stats.get("exclude", []) - self.ext_delta_enabled = bool(ext_delta) - self.ext_delta_include = ext_delta.get("include", []) - self.ext_delta_exclude = ext_delta.get("exclude", []) - self.ext_timing_enabled = ext_timing_enabled + self.stats: StatsCollector = stats + self.interval: float = interval + self.multiplier: float = 60.0 / self.interval + self.task: Optional[task.LoopingCall] = None + self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) + self.ext_stats_enabled: bool = bool(ext_stats) + self.ext_stats_include: List[str] = ext_stats.get("include", []) + self.ext_stats_exclude: List[str] = ext_stats.get("exclude", []) + self.ext_delta_enabled: bool = bool(ext_delta) + self.ext_delta_include: List[str] = ext_delta.get("include", []) + self.ext_delta_exclude: List[str] = ext_delta.get("exclude", []) + self.ext_timing_enabled: bool = ext_timing_enabled @classmethod - def from_crawler(cls, crawler): - interval = crawler.settings.getfloat("LOGSTATS_INTERVAL") + def from_crawler(cls, crawler: Crawler) -> Self: + interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") if not interval: raise NotConfigured try: - ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS") + ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_STATS" + ) except (TypeError, ValueError): ext_stats = ( {"enabled": True} @@ -48,7 +60,9 @@ class PeriodicLog: else None ) try: - ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA") + ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict( + "PERIODIC_LOG_DELTA" + ) except (TypeError, ValueError): ext_delta = ( {"enabled": True} @@ -56,11 +70,14 @@ class PeriodicLog: else None ) - ext_timing_enabled = crawler.settings.getbool( + ext_timing_enabled: bool = crawler.settings.getbool( "PERIODIC_LOG_TIMING_ENABLED", False ) if not (ext_stats or ext_delta or ext_timing_enabled): raise NotConfigured + assert crawler.stats + assert ext_stats is not None + assert ext_delta is not None o = cls( crawler.stats, interval, @@ -72,16 +89,16 @@ class PeriodicLog: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_opened(self, spider): - self.time_prev = datetime.now(tz=timezone.utc) - self.delta_prev = {} - self.stats_prev = {} + def spider_opened(self, spider: Spider) -> None: + self.time_prev: datetime = datetime.now(tz=timezone.utc) + self.delta_prev: Dict[str, Union[int, float]] = {} + self.stats_prev: Dict[str, Union[int, float]] = {} self.task = task.LoopingCall(self.log) self.task.start(self.interval) - def log(self): - data = {} + def log(self) -> None: + data: Dict[str, Any] = {} if self.ext_timing_enabled: data.update(self.log_timing()) if self.ext_delta_enabled: @@ -90,8 +107,8 @@ class PeriodicLog: data.update(self.log_crawler_stats()) logger.info(self.encoder.encode(data)) - def log_delta(self): - num_stats = { + def log_delta(self) -> Dict[str, Any]: + num_stats: Dict[str, Union[int, float]] = { k: v for k, v in self.stats._stats.items() if isinstance(v, (int, float)) @@ -101,7 +118,7 @@ class PeriodicLog: self.delta_prev = num_stats return {"delta": delta} - def log_timing(self): + def log_timing(self) -> Dict[str, Any]: now = datetime.now(tz=timezone.utc) time = { "log_interval": self.interval, @@ -113,7 +130,7 @@ class PeriodicLog: self.time_prev = now return {"time": time} - def log_crawler_stats(self): + def log_crawler_stats(self) -> Dict[str, Any]: stats = { k: v for k, v in self.stats._stats.items() @@ -121,7 +138,9 @@ class PeriodicLog: } return {"stats": stats} - def param_allowed(self, stat_name, include, exclude): + def param_allowed( + self, stat_name: str, include: List[str], exclude: List[str] + ) -> bool: if not include and not exclude: return True for p in exclude: @@ -134,7 +153,7 @@ class PeriodicLog: return True return False - def spider_closed(self, spider, reason): + def spider_closed(self, spider: Spider, reason: str) -> None: self.log() if self.task and self.task.running: self.task.stop() diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py index f8b59827b..7ffbd8bc3 100644 --- a/scrapy/extensions/postprocessing.py +++ b/scrapy/extensions/postprocessing.py @@ -6,7 +6,7 @@ from bz2 import BZ2File from gzip import GzipFile from io import IOBase from lzma import LZMAFile -from typing import Any, BinaryIO, Dict, List +from typing import Any, BinaryIO, Dict, List, cast from scrapy.utils.misc import load_object @@ -142,7 +142,7 @@ class PostProcessingManager(IOBase): :return: returns number of bytes written :rtype: int """ - return self.head_plugin.write(data) + return cast(int, self.head_plugin.write(data)) def tell(self) -> int: return self.file.tell() diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 43359401b..c6eb20277 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,19 +1,27 @@ +from __future__ import annotations + import pickle # nosec from pathlib import Path +from typing import TYPE_CHECKING, Optional -from scrapy import signals +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + class SpiderState: """Store and load spider state during a scraping job""" - def __init__(self, jobdir=None): - self.jobdir = jobdir + def __init__(self, jobdir: Optional[str] = None): + self.jobdir: Optional[str] = jobdir @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: jobdir = job_dir(crawler.settings) if not jobdir: raise NotConfigured @@ -23,18 +31,20 @@ class SpiderState: crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> None: if self.jobdir: with Path(self.statefn).open("wb") as f: + assert hasattr(spider, "state") # set in spider_opened pickle.dump(spider.state, f, protocol=4) - def spider_opened(self, spider): + def spider_opened(self, spider: Spider) -> None: if self.jobdir and Path(self.statefn).exists(): with Path(self.statefn).open("rb") as f: - spider.state = pickle.load(f) # nosec + spider.state = pickle.load(f) # type: ignore[attr-defined] # nosec else: - spider.state = {} + spider.state = {} # type: ignore[attr-defined] @property def statefn(self) -> str: + assert self.jobdir return str(Path(self.jobdir, "spider.state")) diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 58610c25e..20b8f910c 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy import signals +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +from twisted.internet.defer import Deferred + +from scrapy import Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender +from scrapy.statscollectors import StatsCollector + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class StatsMailer: - def __init__(self, stats, recipients, mail): - self.stats = stats - self.recipients = recipients - self.mail = mail + def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender): + self.stats: StatsCollector = stats + self.recipients: List[str] = recipients + self.mail: MailSender = mail @classmethod - def from_crawler(cls, crawler): - recipients = crawler.settings.getlist("STATSMAILER_RCPTS") + def from_crawler(cls, crawler: Crawler) -> Self: + recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail = MailSender.from_settings(crawler.settings) + mail: MailSender = MailSender.from_settings(crawler.settings) + assert crawler.stats o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider): + def spider_closed(self, spider: Spider) -> Optional[Deferred]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index c92b7f5fe..00c69434c 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -4,13 +4,17 @@ Scrapy Telnet Console extension See documentation in docs/topics/telnetconsole.rst """ +from __future__ import annotations + import binascii import logging import os import pprint import traceback +from typing import TYPE_CHECKING, Any, Dict, List from twisted.internet import protocol +from twisted.internet.tcp import Port try: from twisted.conch import manhole, telnet @@ -22,12 +26,16 @@ except (ImportError, SyntaxError): TWISTED_CONCH_AVAILABLE = False from scrapy import signals +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import defers from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.trackref import print_live_refs +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) # signal to update telnet variables @@ -36,7 +44,7 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self, crawler): + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured if not TWISTED_CONCH_AVAILABLE: @@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory): "TELNETCONSOLE_ENABLED setting is True but required twisted " "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK ) - self.crawler = crawler - self.noisy = False - self.portrange = [ + self.crawler: Crawler = crawler + self.noisy: bool = False + self.portrange: List[int] = [ int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT") ] - self.host = crawler.settings["TELNETCONSOLE_HOST"] - self.username = crawler.settings["TELNETCONSOLE_USERNAME"] - self.password = crawler.settings["TELNETCONSOLE_PASSWORD"] + self.host: str = crawler.settings["TELNETCONSOLE_HOST"] + self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"] + self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"] if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode("utf8") @@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory): self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def start_listening(self): - self.port = listen_tcp(self.portrange, self.host, self) + def start_listening(self) -> None: + self.port: Port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() logger.info( "Telnet console listening on %(host)s:%(port)d", @@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory): extra={"crawler": self.crawler}, ) - def stop_listening(self): + def stop_listening(self) -> None: self.port.stopListening() - def protocol(self): + def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] class Portal: """An implementation of IPortal""" @@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory): return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal()) - def _get_telnet_vars(self): + def _get_telnet_vars(self) -> Dict[str, Any]: # Note: if you add entries here also update topics/telnetconsole.rst - telnet_vars = { + assert self.crawler.engine + telnet_vars: Dict[str, Any] = { "engine": self.crawler.engine, "spider": self.crawler.engine.spider, "slot": self.crawler.engine.slot, diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index d217c7a69..bf4e6bb63 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -1,19 +1,29 @@ -import logging +from __future__ import annotations -from scrapy import signals +import logging +from typing import TYPE_CHECKING, Optional, Tuple + +from scrapy import Request, Spider, signals +from scrapy.core.downloader import Slot +from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.http import Response + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self logger = logging.getLogger(__name__) class AutoThrottle: - def __init__(self, crawler): - self.crawler = crawler + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured - self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") - self.target_concurrency = crawler.settings.getfloat( + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") + self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" ) if self.target_concurrency <= 0.0: @@ -27,27 +37,29 @@ class AutoThrottle: ) @classmethod - def from_crawler(cls, crawler): + def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def _spider_opened(self, spider): + def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) + spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] - def _min_delay(self, spider): + def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY")) - def _max_delay(self, spider): + def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider): + def _start_delay(self, spider: Spider) -> float: return max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) - def _response_downloaded(self, response, request, spider): + def _response_downloaded( + self, response: Response, request: Request, spider: Spider + ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") if latency is None or slot is None: @@ -74,11 +86,16 @@ class AutoThrottle: extra={"spider": spider}, ) - def _get_slot(self, request, spider): - key = request.meta.get("download_slot") + def _get_slot( + self, request: Request, spider: Spider + ) -> Tuple[Optional[str], Optional[Slot]]: + key: Optional[str] = request.meta.get("download_slot") + if key is None: + return None, None + assert self.crawler.engine return key, self.crawler.engine.downloader.slots.get(key) - def _adjust_delay(self, slot, latency, response): + def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: """Define delay adjustment policy""" # If a server needs `latency` seconds to respond then diff --git a/scrapy/mail.py b/scrapy/mail.py index dce33fcdf..7cb5ef454 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -14,10 +14,11 @@ from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted import version as twisted_version -from twisted.internet import defer, ssl +from twisted.internet import ssl +from twisted.internet.defer import Deferred from twisted.python.versions import Version from scrapy.settings import BaseSettings @@ -85,9 +86,10 @@ class MailSender: mimetype="text/plain", charset=None, _callback=None, - ): + ) -> Optional[Deferred]: from twisted.internet import reactor + msg: MIMEBase if attachs: msg = MIMEMultipart() else: @@ -134,7 +136,7 @@ class MailSender: "mailattachs": len(attachs), }, ) - return + return None dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) dfd.addCallbacks( @@ -178,7 +180,7 @@ class MailSender: from twisted.internet import reactor msg = BytesIO(msg) - d = defer.Deferred() + d = Deferred() factory = self._create_sender_factory(to_addrs, msg, d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ad3d1d8bc..6cde49bfe 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -2,17 +2,19 @@ import asyncio import sys from asyncio import AbstractEventLoop, AbstractEventLoopPolicy from contextlib import suppress -from typing import Any, Callable, Dict, Optional, Sequence, Type +from typing import Any, Callable, Dict, List, Optional, Sequence, Type from warnings import catch_warnings, filterwarnings, warn from twisted.internet import asyncioreactor, error from twisted.internet.base import DelayedCall +from twisted.internet.protocol import ServerFactory +from twisted.internet.tcp import Port from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import load_object -def listen_tcp(portrange, host, factory): +def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] """Like reactor.listenTCP but tries different ports in a range.""" from twisted.internet import reactor @@ -20,8 +22,6 @@ def listen_tcp(portrange, host, factory): raise ValueError(f"invalid portrange: {portrange}") if not portrange: return reactor.listenTCP(0, factory, interface=host) - if not hasattr(portrange, "__iter__"): - return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) for x in range(portrange[0], portrange[1] + 1): From ab5ea32ffd9cbea22d0fb10ece5258cea207dd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Mar 2024 23:26:19 +0100 Subject: [PATCH 41/45] Fix WindowsRunSpiderCommandTest skip outside Windows for older Twisted. --- tests/test_commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index febad21da..ff308c5ac 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -987,10 +987,14 @@ class MySpider(scrapy.Spider): self.assertIn("The value of FOO is 42", log) -@skipIf(platform.system() != "Windows", "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" + def setUp(self): + # https://github.com/scrapy/scrapy/issues/6286 + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) From 188d9a8bb363ab3ff37dbb6020354afbc72ec02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 11 Mar 2024 23:28:57 +0100 Subject: [PATCH 42/45] Remove unnecessary comment --- tests/test_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index ff308c5ac..ae8289ba7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,7 +991,6 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" def setUp(self): - # https://github.com/scrapy/scrapy/issues/6286 if platform.system() != "Windows": raise unittest.SkipTest("Windows required for .pyw files") From e72de11f55dc5f37d449385f20d2ce4c504914d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Mar 2024 09:29:10 +0100 Subject: [PATCH 43/45] Add super --- tests/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index ae8289ba7..b9d468c66 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -993,6 +993,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): def setUp(self): if platform.system() != "Windows": raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") From d7581c6b41e97fc09c011b089cf34ddd62f41876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Mar 2024 09:44:29 +0100 Subject: [PATCH 44/45] Allow disabling the AutoThrottle extension for a given slot (#6246) --- docs/topics/autothrottle.rst | 12 +++++++++++ docs/topics/settings.rst | 12 +++++++++-- scrapy/core/downloader/__init__.py | 19 ++++++++++++++---- scrapy/extensions/throttle.py | 2 +- tests/test_core_downloader.py | 3 ++- tests/test_downloaderslotssettings.py | 29 ++++++++++++++++++++++++++- tests/test_extension_throttle.py | 19 ++++++++++-------- 7 files changed, 79 insertions(+), 17 deletions(-) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 5370d77b3..8a13b8976 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -47,6 +47,18 @@ effect, but there are some important differences: AutoThrottle doesn't have these issues. +Disabling throttling on a downloader slot +========================================= + +It is possible to disable AutoThrottle for a specific download slot at run time +by setting its ``throttle`` attribute to ``False``, e.g. using +:setting:`DOWNLOAD_SLOTS`. + +Note, however, that AutoThrottle still determines the starting delay of every +slot by setting the ``download_delay`` attribute on the running spider. You +might want to set a custom value for the ``delay`` attribute of the slot, e.g. +using :setting:`DOWNLOAD_SLOTS`. + Throttling algorithm ==================== diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 439aedc18..2bd9cf1ed 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. setting:: DOWNLOAD_SLOTS DOWNLOAD_SLOTS ----------------- +-------------- Default: ``{}`` @@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } @@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis: - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` + There is no global setting for ``throttle``, whose default value is + ``None``. + .. setting:: DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 666282856..ecd3e8b56 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,7 +2,7 @@ import random from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast from twisted.internet import task from twisted.internet.defer import Deferred @@ -24,10 +24,18 @@ if TYPE_CHECKING: class Slot: """Downloader slot""" - def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + def __init__( + self, + concurrency: int, + delay: float, + randomize_delay: bool, + *, + throttle: Optional[bool] = None, + ): self.concurrency: int = concurrency self.delay: float = delay self.randomize_delay: bool = randomize_delay + self.throttle = throttle self.active: Set[Request] = set() self.queue: Deque[Tuple[Request, Deferred]] = deque() @@ -52,13 +60,15 @@ class Slot: return ( f"{cls_name}(concurrency={self.concurrency!r}, " f"delay={self.delay:.2f}, " - f"randomize_delay={self.randomize_delay!r})" + f"randomize_delay={self.randomize_delay!r}, " + f"throttle={self.throttle!r})" ) def __str__(self) -> str: return ( f"" @@ -127,7 +137,8 @@ class Downloader: slot_settings.get("delay", delay), ) randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) + throttle = slot_settings.get("throttle", None) + new_slot = Slot(conc, delay, randomize_delay, throttle=throttle) self.slots[key] = new_slot return key, self.slots[key] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index bf4e6bb63..217e61a81 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -62,7 +62,7 @@ class AutoThrottle: ) -> None: key, slot = self._get_slot(request, spider) latency = request.meta.get("download_latency") - if latency is None or slot is None: + if latency is None or slot is None or slot.throttle is False: return olddelay = slot.delay diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 9a6e9e4ff..81cff4947 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase): def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) self.assertEqual( - repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)" + repr(slot), + "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)", ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d4072d19..ea8c5b4f0 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -3,8 +3,10 @@ import time from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import Request +from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner -from scrapy.http import Request +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import MetaSpider @@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "concurrency": 1, "delay": 2, "randomize_delay": False, + "throttle": False, }, "books.toscrape.com": {"delay": 3, "randomize_delay": False}, }, @@ -70,3 +73,27 @@ class CrawlTestCase(TestCase): } self.assertTrue(max(list(error_delta.values())) < tolerance) + + +def test_params(): + params = { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + "throttle": False, + } + settings = { + "DOWNLOAD_SLOTS": { + "example.com": params, + }, + } + crawler = get_crawler(settings_dict=settings) + downloader = Downloader(crawler) + downloader._slot_gc_loop.stop() # Prevent an unclean reactor. + request = Request("https://example.com") + _, actual = downloader._get_slot(request, spider=None) + expected = Slot(**params) + for param in params: + assert getattr(expected, param) == getattr( + actual, param + ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py index dae4ea966..722a05c26 100644 --- a/tests/test_extension_throttle.py +++ b/tests/test_extension_throttle.py @@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected) @pytest.mark.parametrize( - ("meta", "slot"), + ("meta", "slot", "throttle"), ( - ({}, None), - ({"download_latency": 1.0}, None), - ({"download_slot": "foo"}, None), - ({"download_slot": "foo"}, "foo"), - ({"download_latency": 1.0, "download_slot": "foo"}, None), + ({}, None, None), + ({"download_latency": 1.0}, None, None), + ({"download_slot": "foo"}, None, None), + ({"download_slot": "foo"}, "foo", None), + ({"download_latency": 1.0, "download_slot": "foo"}, None, None), + ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False), ), ) -def test_skipped(meta, slot): +def test_skipped(meta, slot, throttle): crawler = get_crawler() at = build_from_crawler(AutoThrottle, crawler) spider = TestSpider() @@ -177,7 +178,9 @@ def test_skipped(meta, slot): crawler.engine.downloader = Mock() crawler.engine.downloader.slots = {} if slot is not None: - crawler.engine.downloader.slots[slot] = object() + _slot = Mock() + _slot.throttle = throttle + crawler.engine.downloader.slots[slot] = _slot at._adjust_delay = None # Raise exception if called. at._response_downloaded(None, request, spider) From 4460d3ed9631f8409c24f78f1abb36345967b5d5 Mon Sep 17 00:00:00 2001 From: Lucas Belo <144740771+lucas-belo@users.noreply.github.com> Date: Wed, 13 Mar 2024 03:22:48 -0300 Subject: [PATCH 45/45] Remove tests/requirements.txt and refactor extra deps (#6272) Co-authored-by: lucasbelo777 --- conftest.py | 18 ++++++------------ tests/requirements.txt | 17 ----------------- tox.ini | 32 ++++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/conftest.py b/conftest.py index 2bfa46f5a..2ab3dffd4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,6 @@ -import platform -import sys from pathlib import Path import pytest -from twisted import version as twisted_version -from twisted.python.versions import Version from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import install_reactor @@ -85,14 +81,12 @@ def only_not_asyncio(request, reactor_pytest): def requires_uvloop(request): if not request.node.get_closest_marker("requires_uvloop"): return - if sys.implementation.name == "pypy": - pytest.skip("uvloop does not support pypy properly") - if platform.system() == "Windows": - pytest.skip("uvloop does not support Windows") - if twisted_version == Version("twisted", 21, 2, 0): - pytest.skip("https://twistedmatrix.com/trac/ticket/10106") - if sys.version_info >= (3, 12): - pytest.skip("uvloop doesn't support Python 3.12 yet") + try: + import uvloop + + del uvloop + except ImportError: + pytest.skip("uvloop is not installed") def pytest_configure(config): diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index ca5f6ddbd..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Tests requirements -attrs -pexpect >= 4.8.0 -pyftpdlib >= 1.5.8 -pytest -pytest-cov==4.0.0 -pytest-xdist -sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 -testfixtures -uvloop; platform_system != "Windows" - -bpython # optional for shell wrapper tests -brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests -zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests -ipython -pywin32; sys_platform == "win32" diff --git a/tox.ini b/tox.ini index 237aa489c..b5effb527 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,23 @@ envlist = pre-commit,pylint,typing,py minversion = 1.7.0 +[test-requirements] +deps = + attrs + pexpect >= 4.8.0 + pyftpdlib >= 1.5.8 + pygments + pytest + pytest-cov==4.0.0 + pytest-xdist + sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 + testfixtures + pywin32; sys_platform == "win32" + [testenv] deps = - -rtests/requirements.txt + {[test-requirements]deps} + # mitmproxy does not support PyPy mitmproxy; implementation_name != 'pypy' passenv = @@ -81,7 +95,7 @@ deps = w3lib==1.17.0 zope.interface==5.1.0 lxml==4.4.1 - -rtests/requirements.txt + {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment @@ -124,8 +138,12 @@ deps = robotexclusionrulesparser Pillow Twisted[http2] - brotli - zstandard + uvloop; platform_system != "Windows" + bpython # optional for shell wrapper tests + brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests + zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + ipython [testenv:extra-deps-pinned] basepython = python3.8 @@ -136,6 +154,12 @@ deps = Pillow==7.1.0 robotexclusionrulesparser==1.6.2 brotlipy + uvloop==0.14.0; platform_system != "Windows" + bpython==0.7.1 + zstandard==0.1; implementation_name != 'pypy' + ipython==2.0.0 + brotli==0.5.2; implementation_name != 'pypy' + brotlicffi==0.8.0; implementation_name == 'pypy' install_command = {[pinned]install_command} setenv = {[pinned]setenv}