From a34b929a40c12933f75db4665b71348444a5d603 Mon Sep 17 00:00:00 2001 From: srki24 Date: Fri, 4 Nov 2022 18:00:17 +0100 Subject: [PATCH 01/65] issues/5043 Detaching the stream --- scrapy/exporters.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 76cbe4d4b..243ec4fe1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -247,6 +247,12 @@ class CsvItemExporter(BaseItemExporter): values = list(self._build_row(x for _, x in fields)) self.csv_writer.writerow(values) + def finish_exporting(self): + # Detaching stream in order to avoid file closing. + # The file will be closed with slot.storage.store + # https://github.com/scrapy/scrapy/issues/5043 + self.stream.detach() + def _build_row(self, values): for s in values: try: From 2f2bcb006d349eeeed10018362c780496be96550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 2 Feb 2023 05:55:59 +0100 Subject: [PATCH 02/65] Test stream detaching in CsvItemExporter --- scrapy/exporters.py | 5 +---- tests/test_exporters.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 243ec4fe1..42105690c 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -248,10 +248,7 @@ class CsvItemExporter(BaseItemExporter): self.csv_writer.writerow(values) def finish_exporting(self): - # Detaching stream in order to avoid file closing. - # The file will be closed with slot.storage.store - # https://github.com/scrapy/scrapy/issues/5043 - self.stream.detach() + self.stream.detach() # Avoid closing the wrapped file. def _build_row(self, values): for s in values: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 69ac928c3..bec8d2267 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -85,6 +85,10 @@ class BaseItemExporterTest(unittest.TestCase): if self.ie.__class__ is not BaseItemExporter: raise self.ie.finish_exporting() + # Delete the item exporter object, so that if it causes the output + # file handle be closed, which should not be the case, follow-up + # interactions with the output file handle will surface the issue. + del self.ie self._check_output() def test_export_item(self): @@ -230,6 +234,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i1) ie.export_item(i2) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. f.seek(0) self.assertEqual(self.item_class(**pickle.load(f)), i1) self.assertEqual(self.item_class(**pickle.load(f)), i2) @@ -241,6 +246,7 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertEqual(pickle.loads(fp.getvalue()), item) @@ -267,6 +273,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. fp.seek(0) self.assertEqual(marshal.load(fp), item) @@ -299,6 +306,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual(fp.getvalue(), expected) def test_header_export_all(self): @@ -330,6 +338,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertCsvEqual(output.getvalue(), b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') @@ -414,6 +423,7 @@ class XmlItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(item) ie.finish_exporting() + del ie # See the first “del self.ie” in this file for context. self.assertXmlEquivalent(fp.getvalue(), expected_value) def _check_output(self): @@ -520,6 +530,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, self._expected_nested) @@ -534,6 +545,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item['time'] = str(item['time']) self.assertEqual(exported, item) @@ -561,6 +573,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) @@ -577,6 +590,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) @@ -588,6 +602,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) @@ -597,6 +612,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.start_exporting() self.ie.export_item(item) self.ie.finish_exporting() + del self.ie # See the first “del self.ie” in this file for context. exported = json.loads(to_unicode(self.output.getvalue())) item['time'] = str(item['time']) self.assertEqual(exported, [item]) From 426f3ebb7b368084f6e77ccf8a121c85c7913049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 2 Feb 2023 05:58:32 +0100 Subject: [PATCH 03/65] =?UTF-8?q?Fix=20typo:=20causes=20it=20be=20closed?= =?UTF-8?q?=20=E2=86=92=20causes=20it=20to=20be=20closed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 34475b05d..95ff5a93c 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -92,7 +92,7 @@ class BaseItemExporterTest(unittest.TestCase): raise self.ie.finish_exporting() # Delete the item exporter object, so that if it causes the output - # file handle be closed, which should not be the case, follow-up + # file handle to be closed, which should not be the case, follow-up # interactions with the output file handle will surface the issue. del self.ie self._check_output() From 90ce6589eee58e8aca9c368a71907b30250df68d Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Thu, 30 Mar 2023 13:07:51 +0000 Subject: [PATCH 04/65] Add try/except to safe_url_string() Added a try catch condition to the safe_url_string() processing in the LxmlParserLinkExtractor class to avoid scrapers crashing unneccessarily --- scrapy/linkextractors/lxmlhtml.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index dd8dcdf7c..1ee81427c 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -88,7 +88,11 @@ class LxmlParserLinkExtractor: url = self.process_attr(attr_val) if url is None: continue - url = safe_url_string(url, encoding=response_encoding) + try: + url = safe_url_string(url, encoding=response_encoding) + except ValueError: + continue # Disregard badly formatted urls + # to fix relative links after process_value url = urljoin(response_url, url) link = Link( From 9ef00c5c0bc16c9b44c878952a2b54310dc25638 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:01:54 +0000 Subject: [PATCH 05/65] Add logging Lines Adds an error loggign line to the LinkExtractor to detail encountered bad links --- scrapy/linkextractors/lxmlhtml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1ee81427c..f7c6937b0 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -2,6 +2,7 @@ Link extractor based on lxml.html """ import operator +import logging from functools import partial from urllib.parse import urljoin, urlparse @@ -23,6 +24,8 @@ from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain +logger = logging.getLogger(__name__) + # from lxml/src/lxml/html/__init__.py XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" @@ -91,6 +94,7 @@ class LxmlParserLinkExtractor: try: url = safe_url_string(url, encoding=response_encoding) except ValueError: + logger.error(f"Skipping extraction of bad link {url}") continue # Disregard badly formatted urls # to fix relative links after process_value From 9cbcf7724df7de9a449659fbf68bc5d532c33499 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:07:43 +0000 Subject: [PATCH 06/65] Add test to make sure spider doesn't crash on bad --- tests/test_linkextractors.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index f663013ba..d992a5eae 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -815,3 +815,26 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() + + def test_skip_bad_links(self): + html = b""" + Why would you do this? + Good Link + Good Link 2 + """ + response = HtmlResponse("http://example.org/index.html", body=html) + self.assertEqual( + [link for link in lx.extract_links(response)], + [ + Link( + url="http://example.org/item2.html", + text="Good Link", + nofollow=False, + ), + Link( + url="http://example.org/item3.html", + text="Good Link 2", + nofollow=False, + ), + ], + ) From 7cb7cf1ad1aa3d75b494a5b069e5b76b60328daa Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:09:02 +0000 Subject: [PATCH 07/65] Add link extractor back to test --- tests/test_linkextractors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d992a5eae..3ad1abea5 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -823,6 +823,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Good Link 2 """ response = HtmlResponse("http://example.org/index.html", body=html) + lx = self.extractor_cls() self.assertEqual( [link for link in lx.extract_links(response)], [ From 00d93026c8b078d75e4fe43c344f6524f9b45f28 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 08:30:19 +0000 Subject: [PATCH 08/65] Fix bad test case --- tests/test_linkextractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 3ad1abea5..1ea364d80 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -818,11 +818,11 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_skip_bad_links(self): html = b""" - Why would you do this? + Why would you do this? Good Link Good Link 2 """ - response = HtmlResponse("http://example.org/index.html", body=html) + response = HtmlResponse("http://example.org/index.html", body=html, encoding='utf-8') lx = self.extractor_cls() self.assertEqual( [link for link in lx.extract_links(response)], From 4043560547faac0ee4cfadba4d0f02b4be1f72de Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 12:29:22 +0000 Subject: [PATCH 09/65] remove utf-8 encoding flag from test --- tests/test_linkextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 1ea364d80..3673e82cd 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -822,7 +822,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Good Link Good Link 2 """ - response = HtmlResponse("http://example.org/index.html", body=html, encoding='utf-8') + response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual( [link for link in lx.extract_links(response)], From c9a5934494cbb3fe0aa572b536ee222a3e212487 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 12:29:49 +0000 Subject: [PATCH 10/65] Reduce logging level of bad URL --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f7c6937b0..0d1b76531 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -94,7 +94,7 @@ class LxmlParserLinkExtractor: try: url = safe_url_string(url, encoding=response_encoding) except ValueError: - logger.error(f"Skipping extraction of bad link {url}") + logger.debug(f"Skipping extraction of bad link {url}") continue # Disregard badly formatted urls # to fix relative links after process_value From 608b7de582af891a37a3fab60423c847af648db8 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 14:38:06 +0000 Subject: [PATCH 11/65] Skip new test if python version less than 3.8 --- tests/test_linkextractors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 3673e82cd..78219f642 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,6 +1,7 @@ import pickle import re import unittest +import sys from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -816,6 +817,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() + @unittest.skipIf( + sys.version_info < (3, 8), + reason="Urllib3 is less strict in versions for python 3.7 so does not cause spider to crash", + ) def test_skip_bad_links(self): html = b""" Why would you do this? From 618e82dbe104c4b97cc4f7b37bf9130a76093734 Mon Sep 17 00:00:00 2001 From: Samuel Bartlett Date: Fri, 31 Mar 2023 15:12:47 +0000 Subject: [PATCH 12/65] Exclude test for python versionbs less than 3.8 --- tests/test_linkextractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 78219f642..784fdb658 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -819,7 +819,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @unittest.skipIf( sys.version_info < (3, 8), - reason="Urllib3 is less strict in versions for python 3.7 so does not cause spider to crash", + reason="some library for python 3.7 so is less strict so bad links like htis don't crash scrapy", ) def test_skip_bad_links(self): html = b""" From d47c732ae9ebda84c689563048919923ddb17383 Mon Sep 17 00:00:00 2001 From: Kartik Kumar <130273246+heppymxm@users.noreply.github.com> Date: Tue, 11 Apr 2023 21:55:42 +0530 Subject: [PATCH 13/65] base64-decode GCS checksums (#5891) --- scrapy/pipelines/files.py | 3 ++- tests/test_pipeline_files.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6e9f661e5..4b594ccb7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -3,6 +3,7 @@ Files Pipeline See documentation in topics/media-pipeline.rst """ +import base64 import functools import hashlib import logging @@ -228,7 +229,7 @@ class GCSFilesStore: def stat_file(self, path, info): def _onsuccess(blob): if blob: - checksum = blob.md5_hash + checksum = base64.b64decode(blob.md5_hash).hex() last_modified = time.mktime(blob.updated.timetuple()) return {"checksum": checksum, "last_modified": last_modified} return {} diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 9701e5d4e..c80666586 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -601,7 +601,7 @@ class TestGCSFilesStore(unittest.TestCase): s = yield store.stat_file(path, info=None) self.assertIn("last_modified", s) self.assertIn("checksum", s) - self.assertEqual(s["checksum"], "zc2oVgXkbQr2EQdSdw3OPA==") + self.assertEqual(s["checksum"], "cdcda85605e46d0af6110752770dce3c") u = urlparse(uri) content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) self.assertEqual(content, data) From 3f0c2fae5e18c448bd1791920500c976d44fc321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Apr 2023 09:28:28 +0200 Subject: [PATCH 14/65] Skip test_skip_bad_links based on the w3lib version --- scrapy/linkextractors/lxmlhtml.py | 6 +++--- tests/test_linkextractors.py | 16 +++++++++++----- tox.ini | 1 + 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 0d1b76531..23cbd0116 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,8 +1,8 @@ """ Link extractor based on lxml.html """ -import operator import logging +import operator from functools import partial from urllib.parse import urljoin, urlparse @@ -94,8 +94,8 @@ class LxmlParserLinkExtractor: try: url = safe_url_string(url, encoding=response_encoding) except ValueError: - logger.debug(f"Skipping extraction of bad link {url}") - continue # Disregard badly formatted urls + logger.debug(f"Skipping extraction of link with bad URL {url!r}") + continue # to fix relative links after process_value url = urljoin(response_url, url) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 784fdb658..e1ec19601 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,7 +1,10 @@ import pickle import re import unittest -import sys + +from packaging.version import Version +from pytest import mark +from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -817,13 +820,16 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() - @unittest.skipIf( - sys.version_info < (3, 8), - reason="some library for python 3.7 so is less strict so bad links like htis don't crash scrapy", + @mark.skipif( + Version(w3lib_version) < Version("2.0.0"), + reason=( + "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain " + "about an invalid port value." + ), ) def test_skip_bad_links(self): html = b""" - Why would you do this? + Why would you do this? Good Link Good Link 2 """ diff --git a/tox.ini b/tox.ini index 5a9d9cf29..873e7662b 100644 --- a/tox.ini +++ b/tox.ini @@ -101,6 +101,7 @@ install_command = python -I -m pip install {opts} {packages} [testenv:pinned] +basepython = python3.7 deps = {[pinned]deps} PyDispatcher==2.0.5 From c2a31974ffc06412a5ab8d87fe070c98cd9c856b Mon Sep 17 00:00:00 2001 From: Serhii A Date: Thu, 13 Apr 2023 12:44:20 +0300 Subject: [PATCH 15/65] Improve get_func_args (#5885) --- scrapy/utils/python.py | 52 +++++++++++++++++++------------------- tests/test_utils_python.py | 10 +++----- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fc50e0f12..818fa5d6b 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -174,33 +174,33 @@ def binary_is_text(data): def get_func_args(func, stripself=False): - """Return the argument name list of a callable""" - if inspect.isfunction(func): - spec = inspect.getfullargspec(func) - func_args = spec.args + spec.kwonlyargs - elif inspect.isclass(func): - return get_func_args(func.__init__, True) - elif inspect.ismethod(func): - return get_func_args(func.__func__, True) - elif inspect.ismethoddescriptor(func): - return [] - elif isinstance(func, partial): - return [ - x - for x in get_func_args(func.func)[len(func.args) :] - if not (func.keywords and x in func.keywords) - ] - elif hasattr(func, "__call__"): - if inspect.isroutine(func): - return [] - if getattr(func, "__name__", None) == "__call__": - return [] - return get_func_args(func.__call__, True) + """Return the argument name list of a callable object""" + if not callable(func): + raise TypeError(f"func must be callable, got '{type(func).__name__}'") + + args = [] + try: + sig = inspect.signature(func) + except ValueError: + return args + + if isinstance(func, partial): + partial_args = func.args + partial_kw = func.keywords + + for name, param in sig.parameters.items(): + if param.name in partial_args: + continue + if partial_kw and param.name in partial_kw: + continue + args.append(name) else: - raise TypeError(f"{type(func)} is not callable") - if stripself: - func_args.pop(0) - return func_args + for name in sig.parameters.keys(): + args.append(name) + + if stripself and args and args[0] == "self": + args = args[1:] + return args def get_spec(func): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 57f40c2e5..80d2e8da1 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -235,20 +235,16 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(partial_f3), ["c"]) self.assertEqual(get_func_args(cal), ["a", "b", "c"]) self.assertEqual(get_func_args(object), []) + self.assertEqual(get_func_args(str.split, stripself=True), ["sep", "maxsplit"]) + self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) if platform.python_implementation() == "CPython": - # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(str.split), []) - self.assertEqual(get_func_args(" ".join), []) + # doesn't work on CPython: https://bugs.python.org/issue42785 self.assertEqual(get_func_args(operator.itemgetter(2)), []) elif platform.python_implementation() == "PyPy": - self.assertEqual( - get_func_args(str.split, stripself=True), ["sep", "maxsplit"] - ) self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ["obj"] ) - self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"]) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) From 441ac196e4151765fa424af59f3938e72b8434c1 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Thu, 13 Apr 2023 12:46:59 -0300 Subject: [PATCH 16/65] Implement a request_to_curl function (#5892) --- scrapy/utils/request.py | 31 ++++++++++++++++++ tests/test_utils_request.py | 64 +++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 409ca2e52..6d8be991d 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -327,3 +327,34 @@ def _get_method(obj, name): return getattr(obj, name) except AttributeError: raise ValueError(f"Method {name!r} not found in: {obj}") + + +def request_to_curl(request: Request) -> str: + """ + Converts a :class:`~scrapy.Request` object to a curl command. + + :param :class:`~scrapy.Request`: Request object to be converted + :return: string containing the curl command + """ + method = request.method + + data = f"--data-raw '{request.body.decode('utf-8')}'" if request.body else "" + + headers = " ".join( + f"-H '{k.decode()}: {v[0].decode()}'" for k, v in request.headers.items() + ) + + url = request.url + cookies = "" + if request.cookies: + if isinstance(request.cookies, dict): + cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items()) + cookies = f"--cookie '{cookie}'" + elif isinstance(request.cookies, list): + cookie = "; ".join( + f"{list(c.keys())[0]}={list(c.values())[0]}" for c in request.cookies + ) + cookies = f"--cookie '{cookie}'" + + curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip() + return " ".join(curl_cmd.split()) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 6ca272de1..e6d1abe3f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,3 +1,4 @@ +import json import unittest import warnings from hashlib import sha1 @@ -18,6 +19,7 @@ from scrapy.utils.request import ( request_authenticate, request_fingerprint, request_httprepr, + request_to_curl, ) from scrapy.utils.test import get_crawler @@ -666,5 +668,67 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertEqual(fingerprint, settings["FINGERPRINT"]) +class RequestToCurlTest(unittest.TestCase): + def _test_request(self, request_object, expected_curl_command): + curl_command = request_to_curl(request_object) + self.assertEqual(curl_command, expected_curl_command) + + def test_get(self): + request_object = Request("https://www.example.com") + expected_curl_command = "curl -X GET https://www.example.com" + self._test_request(request_object, expected_curl_command) + + def test_post(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + 'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\'' + ) + self._test_request(request_object, expected_curl_command) + + def test_headers(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + ' --data-raw \'{"foo": "bar"}\'' + " -H 'Content-Type: application/json' -H 'Accept: application/json'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_dict(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies={"foo": "bar"}, + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + def test_cookies_list(self): + request_object = Request( + "https://www.httpbin.org/post", + method="POST", + cookies=[{"foo": "bar"}], + body=json.dumps({"foo": "bar"}), + ) + expected_curl_command = ( + "curl -X POST https://www.httpbin.org/post" + " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" + ) + self._test_request(request_object, expected_curl_command) + + if __name__ == "__main__": unittest.main() From e1f66620ec7341c55f3eb7f44088224b5f68c1ad Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Apr 2023 18:13:21 +0400 Subject: [PATCH 17/65] Fix typo on tutorial.rst (#5893) (#5895) Co-authored-by: Seth Herr --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 064ce05f8..04d73d95a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -329,7 +329,7 @@ the :meth:`~scrapy.selector.SelectorList.re` method to extract using >>> response.css("title::text").re(r"(\w+) to (\w+)") ['Quotes', 'Scrape'] -In order to find the proper CSS selectors to use, you might find useful opening +In order to find the proper CSS selectors to use, you might find it useful to open the response page from the shell in your web browser using ``view(response)``. You can use your browser's developer tools to inspect the HTML and come up with a selector (see :ref:`topics-developer-tools`). From 02f3e8d413ccdd6a3f0b5828a9cd94e1fb3662b1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Apr 2023 10:37:52 +0400 Subject: [PATCH 18/65] Typing for scrapy/core/downloader (#5896) --- scrapy/core/downloader/__init__.py | 95 ++++++++++++--------- scrapy/core/downloader/contextfactory.py | 56 +++++++----- scrapy/core/downloader/handlers/__init__.py | 31 ++++--- scrapy/core/downloader/middleware.py | 29 ++++--- scrapy/core/downloader/tls.py | 26 +++--- scrapy/core/downloader/webclient.py | 41 +++++---- scrapy/utils/python.py | 36 ++++++-- scrapy/utils/ssl.py | 22 +++-- 8 files changed, 211 insertions(+), 125 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index dde76a547..426056dc8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,45 +2,52 @@ import random from collections import deque from datetime import datetime from time import time +from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast -from twisted.internet import defer, task +from twisted.internet import task from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.http import Response from scrapy.resolver import dnscache +from scrapy.settings import BaseSettings +from scrapy.signalmanager import SignalManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class Slot: """Downloader slot""" - def __init__(self, concurrency, delay, randomize_delay): - self.concurrency = concurrency - self.delay = delay - self.randomize_delay = randomize_delay + def __init__(self, concurrency: int, delay: float, randomize_delay: bool): + self.concurrency: int = concurrency + self.delay: float = delay + self.randomize_delay: bool = randomize_delay - self.active = set() - self.queue = deque() - self.transferring = set() - self.lastseen = 0 + self.active: Set[Request] = set() + self.queue: Deque[Tuple[Request, Deferred]] = deque() + self.transferring: Set[Request] = set() + self.lastseen: float = 0 self.latercall = None - def free_transfer_slots(self): + def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) - def download_delay(self): + def download_delay(self) -> float: if self.randomize_delay: return random.uniform(0.5 * self.delay, 1.5 * self.delay) return self.delay - def close(self): + def close(self) -> None: if self.latercall and self.latercall.active(): self.latercall.cancel() - def __repr__(self): + def __repr__(self) -> str: cls_name = self.__class__.__name__ return ( f"{cls_name}(concurrency={self.concurrency!r}, " @@ -48,7 +55,7 @@ class Slot: f"randomize_delay={self.randomize_delay!r})" ) - def __str__(self): + def __str__(self) -> str: return ( f" Tuple[int, float]: + delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay @@ -72,23 +81,29 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader: DOWNLOAD_SLOT = "download_slot" - def __init__(self, crawler): - self.settings = crawler.settings - self.signals = crawler.signals - self.slots = {} - self.active = set() - self.handlers = DownloadHandlers(crawler) - self.total_concurrency = self.settings.getint("CONCURRENT_REQUESTS") - self.domain_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") - self.ip_concurrency = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") - self.randomize_delay = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") - self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) - self._slot_gc_loop = task.LoopingCall(self._slot_gc) + def __init__(self, crawler: "Crawler"): + self.settings: BaseSettings = crawler.settings + self.signals: SignalManager = crawler.signals + self.slots: Dict[str, Slot] = {} + self.active: Set[Request] = set() + self.handlers: DownloadHandlers = DownloadHandlers(crawler) + self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") + self.domain_concurrency: int = self.settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") + self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") + self.middleware: DownloaderMiddlewareManager = ( + DownloaderMiddlewareManager.from_crawler(crawler) + ) + self._slot_gc_loop: task.LoopingCall = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {}) + self.per_slot_settings: Dict[str, Dict[str, Any]] = self.settings.getdict( + "DOWNLOAD_SLOTS", {} + ) def fetch(self, request: Request, spider: Spider) -> Deferred: - def _deactivate(response): + def _deactivate(response: Response) -> Response: self.active.remove(request) return response @@ -99,7 +114,7 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - def _get_slot(self, request, spider): + def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]: key = self._get_slot_key(request, spider) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) @@ -117,9 +132,9 @@ class Downloader: return key, self.slots[key] - def _get_slot_key(self, request, spider): + def _get_slot_key(self, request: Request, spider: Spider) -> str: if self.DOWNLOAD_SLOT in request.meta: - return request.meta[self.DOWNLOAD_SLOT] + return cast(str, request.meta[self.DOWNLOAD_SLOT]) key = urlparse_cached(request).hostname or "" if self.ip_concurrency: @@ -127,11 +142,11 @@ class Downloader: return key - def _enqueue_request(self, request, spider): + def _enqueue_request(self, request: Request, spider: Spider) -> Deferred: key, slot = self._get_slot(request, spider) request.meta[self.DOWNLOAD_SLOT] = key - def _deactivate(response): + def _deactivate(response: Response) -> Response: slot.active.remove(request) return response @@ -139,12 +154,12 @@ class Downloader: self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=spider ) - deferred = defer.Deferred().addBoth(_deactivate) + deferred = Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) return deferred - def _process_queue(self, spider, slot): + def _process_queue(self, spider: Spider, slot: Slot) -> None: from twisted.internet import reactor if slot.latercall and slot.latercall.active(): @@ -172,7 +187,7 @@ class Downloader: self._process_queue(spider, slot) break - def _download(self, slot, request, spider): + def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred: # The order is very important for the following deferreds. Do not change! # 1. Create the download deferred @@ -180,7 +195,7 @@ class Downloader: # 2. Notify response_downloaded listeners about the recent download # before querying queue for next request - def _downloaded(response): + def _downloaded(response: Response) -> Response: self.signals.send_catch_log( signal=signals.response_downloaded, response=response, @@ -197,7 +212,7 @@ class Downloader: # middleware itself) slot.transferring.add(request) - def finish_transferring(_): + def finish_transferring(_: Any) -> Any: slot.transferring.remove(request) self._process_queue(spider, slot) self.signals.send_catch_log( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 53ae78918..909cc273f 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,4 +1,5 @@ import warnings +from typing import TYPE_CHECKING, Any, List, Optional from OpenSSL import SSL from twisted.internet._sslverify import _setAcceptableProtocols @@ -18,8 +19,12 @@ from scrapy.core.downloader.tls import ( ScrapyClientTLSOptions, openssl_methods, ) +from scrapy.settings import BaseSettings from scrapy.utils.misc import create_instance, load_object +if TYPE_CHECKING: + from twisted.internet._sslverify import ClientTLSOptions + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): @@ -35,25 +40,34 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def __init__( self, - method=SSL.SSLv23_METHOD, - tls_verbose_logging=False, - tls_ciphers=None, - *args, - **kwargs, + method: int = SSL.SSLv23_METHOD, + tls_verbose_logging: bool = False, + tls_ciphers: Optional[str] = None, + *args: Any, + **kwargs: Any, ): super().__init__(*args, **kwargs) - self._ssl_method = method - self.tls_verbose_logging = tls_verbose_logging + self._ssl_method: int = method + self.tls_verbose_logging: bool = tls_verbose_logging + self.tls_ciphers: AcceptableCiphers if tls_ciphers: self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS @classmethod - def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): - tls_verbose_logging = settings.getbool("DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING") - tls_ciphers = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] - return cls( + def from_settings( + cls, + settings: BaseSettings, + method: int = SSL.SSLv23_METHOD, + *args: Any, + **kwargs: Any, + ): + tls_verbose_logging: bool = settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + tls_ciphers: Optional[str] = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + return cls( # type: ignore[misc] method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, @@ -61,7 +75,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): **kwargs, ) - def getCertificateOptions(self): + def getCertificateOptions(self) -> CertificateOptions: # setting verify=True will require you to provide CAs # to verify against; in other words: it's not that simple @@ -82,12 +96,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() - def getContext(self, hostname=None, port=None): + def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: ctx = self.getCertificateOptions().getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": return ScrapyClientTLSOptions( hostname.decode("ascii"), self.getContext(), @@ -114,7 +128,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -133,13 +147,15 @@ class AcceptableProtocolsContextFactory: negotiation. """ - def __init__(self, context_factory, acceptable_protocols): + def __init__(self, context_factory: Any, acceptable_protocols: List[bytes]): verifyObject(IPolicyForHTTPS, context_factory) - self._wrapped_context_factory = context_factory - self._acceptable_protocols = acceptable_protocols + self._wrapped_context_factory: Any = context_factory + self._acceptable_protocols: List[bytes] = acceptable_protocols - def creatorForNetloc(self, hostname, port): - options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions": + options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc( + hostname, port + ) _setAcceptableProtocols(options._ctx, self._acceptable_protocols) return options diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 39155efe9..6a211aafa 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,25 +1,32 @@ """Download handlers for different schemes""" import logging +from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast from twisted.internet import defer +from twisted.internet.defer import Deferred -from scrapy import signals +from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, NotSupported from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values +if TYPE_CHECKING: + from scrapy.crawler import Crawler + logger = logging.getLogger(__name__) class DownloadHandlers: - def __init__(self, crawler): - self._crawler = crawler - self._schemes = {} # stores acceptable schemes on instancing - self._handlers = {} # stores instanced handlers for schemes - self._notconfigured = {} # remembers failed handlers - handlers = without_none_values( + def __init__(self, crawler: "Crawler"): + self._crawler: "Crawler" = crawler + self._schemes: Dict[ + str, Union[str, Callable] + ] = {} # stores acceptable schemes on instancing + self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes + self._notconfigured: Dict[str, str] = {} # remembers failed handlers + handlers: Dict[str, Union[str, Callable]] = without_none_values( crawler.settings.getwithbase("DOWNLOAD_HANDLERS") ) for scheme, clspath in handlers.items(): @@ -28,7 +35,7 @@ class DownloadHandlers: crawler.signals.connect(self._close, signals.engine_stopped) - def _get_handler(self, scheme): + def _get_handler(self, scheme: str) -> Any: """Lazy-load the downloadhandler for a scheme only on the first request for that scheme. """ @@ -42,7 +49,7 @@ class DownloadHandlers: return self._load_handler(scheme) - def _load_handler(self, scheme, skip_lazy=False): + def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any: path = self._schemes[scheme] try: dhcls = load_object(path) @@ -69,17 +76,17 @@ class DownloadHandlers: self._handlers[scheme] = dh return dh - def download_request(self, request, spider): + def download_request(self, request: Request, spider: Spider) -> Deferred: scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: raise NotSupported( f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}" ) - return handler.download_request(request, spider) + return cast(Deferred, handler.download_request(request, spider)) @defer.inlineCallbacks - def _close(self, *_a, **_kw): + def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]: for dh in self._handlers.values(): if hasattr(dh, "close"): yield dh.close() diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 56df48b2e..dca13c01e 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,15 +3,16 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -from typing import Callable, Union, cast +from typing import Any, Callable, Generator, List, Union, cast -from twisted.internet import defer +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager +from scrapy.settings import BaseSettings from scrapy.utils.conf import build_component_list from scrapy.utils.defer import deferred_from_coro, mustbe_deferred @@ -20,10 +21,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" @classmethod - def _get_mwlist_from_settings(cls, settings): + def _get_mwlist_from_settings(cls, settings: BaseSettings) -> List[Any]: return build_component_list(settings.getwithbase("DOWNLOADER_MIDDLEWARES")) - def _add_middleware(self, mw): + def _add_middleware(self, mw: Any) -> None: if hasattr(mw, "process_request"): self.methods["process_request"].append(mw.process_request) if hasattr(mw, "process_response"): @@ -31,9 +32,11 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, "process_exception"): self.methods["process_exception"].appendleft(mw.process_exception) - def download(self, download_func: Callable, request: Request, spider: Spider): - @defer.inlineCallbacks - def process_request(request: Request): + def download( + self, download_func: Callable, request: Request, spider: Spider + ) -> Deferred: + @inlineCallbacks + def process_request(request: Request) -> Generator[Deferred, Any, Any]: for method in self.methods["process_request"]: method = cast(Callable, method) response = yield deferred_from_coro( @@ -50,8 +53,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return (yield download_func(request=request, spider=spider)) - @defer.inlineCallbacks - def process_response(response: Union[Response, Request]): + @inlineCallbacks + def process_response( + response: Union[Response, Request] + ) -> Generator[Deferred, Any, Union[Response, Request]]: if response is None: raise TypeError("Received None in process_response") elif isinstance(response, Request): @@ -71,8 +76,10 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response return response - @defer.inlineCallbacks - def process_exception(failure: Failure): + @inlineCallbacks + def process_exception( + failure: Failure, + ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]: exception = failure.value for method in self.methods["process_exception"]: method = cast(Callable, method) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 025575fe1..33cea7263 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,4 +1,5 @@ import logging +from typing import Any, Dict from OpenSSL import SSL from service_identity.exceptions import CertificateError @@ -20,7 +21,7 @@ METHOD_TLSv11 = "TLSv1.1" METHOD_TLSv12 = "TLSv1.2" -openssl_methods = { +openssl_methods: Dict[str, int] = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only @@ -39,11 +40,13 @@ class ScrapyClientTLSOptions(ClientTLSOptions): logging warnings. Also, HTTPS connection parameters logging is added. """ - def __init__(self, hostname, ctx, verbose_logging=False): + def __init__(self, hostname: str, ctx: SSL.Context, verbose_logging: bool = False): super().__init__(hostname, ctx) - self.verbose_logging = verbose_logging + self.verbose_logging: bool = verbose_logging - def _identityVerifyingInfoCallback(self, connection, where, ret): + def _identityVerifyingInfoCallback( + self, connection: SSL.Connection, where: int, ret: Any + ) -> None: if where & SSL.SSL_CB_HANDSHAKE_START: connection.set_tlsext_host_name(self._hostnameBytes) elif where & SSL.SSL_CB_HANDSHAKE_DONE: @@ -55,11 +58,12 @@ class ScrapyClientTLSOptions(ClientTLSOptions): connection.get_cipher_name(), ) server_cert = connection.get_peer_certificate() - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) + if server_cert: + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) key_info = get_temp_key_info(connection._ssl) if key_info: logger.debug("SSL temp key: %s", key_info) @@ -82,4 +86,6 @@ class ScrapyClientTLSOptions(ClientTLSOptions): ) -DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString("DEFAULT") +DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString( + "DEFAULT" +) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3d103652b..bb1f73805 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,22 +1,25 @@ import re from time import time -from urllib.parse import urldefrag, urlparse, urlunparse +from typing import Optional, Tuple +from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from twisted.internet import defer from twisted.internet.protocol import ClientFactory from twisted.web.http import HTTPClient +from scrapy import Request from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode -def _parsed_url_args(parsed): +def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]: # Assume parsed is urlparse-d from Request.url, # which was passed via safe_url_string and is ascii-only. - path = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) - path = to_bytes(path, encoding="ascii") + path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, "")) + path = to_bytes(path_str, encoding="ascii") + assert parsed.hostname is not None host = to_bytes(parsed.hostname, encoding="ascii") port = parsed.port scheme = to_bytes(parsed.scheme, encoding="ascii") @@ -26,7 +29,7 @@ def _parsed_url_args(parsed): return scheme, netloc, host, port, path -def _parse(url): +def _parse(url: str) -> Tuple[bytes, bytes, bytes, int, bytes]: """Return tuple of (scheme, netloc, host, port, path), all in bytes except for port which is int. Assume url is from Request.url, which was passed via safe_url_string @@ -132,17 +135,19 @@ class ScrapyHTTPClientFactory(ClientFactory): self.scheme, _, self.host, self.port, _ = _parse(proxy) self.path = self.url - def __init__(self, request, timeout=180): - self._url = urldefrag(request.url)[0] + def __init__(self, request: Request, timeout: float = 180): + self._url: str = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url, encoding="ascii") - self.method = to_bytes(request.method, encoding="ascii") - self.body = request.body or None - self.headers = Headers(request.headers) - self.response_headers = None - self.timeout = request.meta.get("download_timeout") or timeout - self.start_time = time() - self.deferred = defer.Deferred().addCallback(self._build_response, request) + self.url: bytes = to_bytes(self._url, encoding="ascii") + self.method: bytes = to_bytes(request.method, encoding="ascii") + self.body: Optional[bytes] = request.body or None + self.headers: Headers = Headers(request.headers) + self.response_headers: Optional[Headers] = None + self.timeout: float = request.meta.get("download_timeout") or timeout + self.start_time: float = time() + self.deferred: defer.Deferred = defer.Deferred().addCallback( + self._build_response, request + ) # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected # to have _disconnectedDeferred. See Twisted r32329. @@ -150,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory): # needed to add the callback _waitForDisconnect. # Specifically this avoids the AttributeError exception when # clientConnectionFailed method is called. - self._disconnectedDeferred = defer.Deferred() + self._disconnectedDeferred: defer.Deferred = defer.Deferred() self._set_connection_attributes(request) @@ -166,8 +171,8 @@ class ScrapyHTTPClientFactory(ClientFactory): elif self.method == b"POST": self.headers["Content-Length"] = 0 - def __repr__(self): - return f"<{self.__class__.__name__}: {self.url}>" + def __repr__(self) -> str: + return f"<{self.__class__.__name__}: {self._url}>" def _cancelTimeout(self, result, timeoutCall): if timeoutCall.active(): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 818fa5d6b..27816c0df 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,7 +8,16 @@ import sys import weakref from functools import partial, wraps from itertools import chain -from typing import Any, AsyncGenerator, AsyncIterable, Iterable, Union +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Iterable, + Mapping, + Optional, + Union, + overload, +) from scrapy.utils.asyncgen import as_async_generator @@ -82,7 +91,9 @@ def unique(list_, key=lambda x: x): return result -def to_unicode(text, encoding=None, errors="strict"): +def to_unicode( + text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" +) -> str: """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" if isinstance(text, str): @@ -97,7 +108,9 @@ def to_unicode(text, encoding=None, errors="strict"): return text.decode(encoding, errors) -def to_bytes(text, encoding=None, errors="strict"): +def to_bytes( + text: Union[str, bytes], encoding: Optional[str] = None, errors: str = "strict" +) -> bytes: """Return the binary representation of ``text``. If ``text`` is already a bytes object, return it as-is.""" if isinstance(text, bytes): @@ -160,11 +173,12 @@ def memoizemethod_noargs(method): return new_method -_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} -_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} +_BINARYCHARS = { + i for i in range(32) if to_bytes(chr(i)) not in {b"\0", b"\t", b"\n", b"\r"} +} -def binary_is_text(data): +def binary_is_text(data: bytes) -> bool: """Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. """ @@ -258,6 +272,16 @@ def equal_attributes(obj1, obj2, attributes): return True +@overload +def without_none_values(iterable: Mapping) -> dict: + ... + + +@overload +def without_none_values(iterable: Iterable) -> Iterable: + ... + + def without_none_values(iterable): """Return a copy of ``iterable`` with all ``None`` entries removed. diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 3ddceea35..03ae4ba9e 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,24 +1,28 @@ +from typing import Any, Optional, cast + import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL +import OpenSSL.version +from OpenSSL.crypto import X509Name from scrapy.utils.python import to_unicode -def ffi_buf_to_string(buf): +def ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) -def x509name_to_string(x509name): +def x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ - result_buffer = pyOpenSSLutil.ffi.new("char[]", 512) + result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( - x509name._name, result_buffer, len(result_buffer) + x509name._name, result_buffer, len(result_buffer) # type: ignore[attr-defined] ) return ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object): +def get_temp_key_info(ssl_object: Any) -> Optional[str]: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): # removed in cryptography 40.0.0 @@ -53,8 +57,10 @@ def get_temp_key_info(ssl_object): return ", ".join(key_info) -def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION).decode( - "ascii", errors="replace" +def get_openssl_version() -> str: + # https://github.com/python/typeshed/issues/10024 + system_openssl_bytes = cast( + bytes, OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) ) + system_openssl = system_openssl_bytes.decode("ascii", errors="replace") return f"{OpenSSL.version.__version__} ({system_openssl})" From f5447f3b4ca2c91a07bfdd5acad9661061b8bbd7 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:31:51 -0500 Subject: [PATCH 19/65] fix: Request.from_curl() with prefixed string literals --- scrapy/utils/curl.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a2243ae2e..5e095f933 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,4 +1,5 @@ import argparse +import re import warnings from http.cookies import SimpleCookie from shlex import split @@ -7,6 +8,15 @@ from urllib.parse import urlparse from w3lib.http import basic_auth_header +class DataAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + value = str(values).encode("utf-8").decode("utf-8") + if items := re.findall(r"{.+}", value): + value = items[0] + + setattr(namespace, self.dest, value) + + class CurlParser(argparse.ArgumentParser): def error(self, message): error_msg = f"There was an error parsing the curl command: {message}" @@ -17,7 +27,7 @@ curl_parser = CurlParser() curl_parser.add_argument("url") curl_parser.add_argument("-H", "--header", dest="headers", action="append") curl_parser.add_argument("-X", "--request", dest="method") -curl_parser.add_argument("-d", "--data", "--data-raw", dest="data") +curl_parser.add_argument("-d", "--data", "--data-raw", dest="data", action=DataAction) curl_parser.add_argument("-u", "--user", dest="auth") From 88c58a8c9ca6f8149779bfc92d00499a613e5fb3 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 21:49:05 -0500 Subject: [PATCH 20/65] feat: added test_post_data_raw_with_string_prefix --- tests/test_utils_curl.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index fd4612eba..1816db29b 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -154,6 +154,15 @@ class CurlToRequestKwargsTest(unittest.TestCase): } self._test_command(curl_command, expected_result) + def test_post_data_raw_with_string_prefix(self): + curl_command = "curl 'https://www.example.org/' --data-raw $'{\"$filters\":\"Filter\u0021\"}'" + expected_result = { + "method": "POST", + "url": "https://www.example.org/", + "body": '{"$filters":"Filter!"}', + } + self._test_command(curl_command, expected_result) + def test_explicit_get_with_data(self): curl_command = "curl httpbin.org/anything -X GET --data asdf" expected_result = { From 69f96b9e96b1399f68cc61db0fe2d2b6cff5484d Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 22:04:34 -0500 Subject: [PATCH 21/65] fix: regex --- scrapy/utils/curl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 5e095f933..b873d2bdf 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -11,7 +11,7 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): value = str(values).encode("utf-8").decode("utf-8") - if items := re.findall(r"{.+}", value): + if items := re.findall(r"\$(.+)", value): value = items[0] setattr(namespace, self.dest, value) From 3209eac14f430f9cba522c12a615111bcabaecd5 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 22:35:15 -0500 Subject: [PATCH 22/65] fix: checks --- scrapy/utils/curl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index b873d2bdf..ecfa292ea 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -11,9 +11,8 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): value = str(values).encode("utf-8").decode("utf-8") - if items := re.findall(r"\$(.+)", value): - value = items[0] - + items = re.findall(r"\$(.+)", value) + value = items[0] if items else value setattr(namespace, self.dest, value) From 7e1814faf836757933afec4c7c394f43f34c3567 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 18 Apr 2023 23:36:51 -0500 Subject: [PATCH 23/65] fix: regex --- scrapy/utils/curl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index ecfa292ea..9c98e4cb8 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -11,8 +11,7 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): value = str(values).encode("utf-8").decode("utf-8") - items = re.findall(r"\$(.+)", value) - value = items[0] if items else value + value = value[1::] if re.match(r"^\$(.+)", value) else value setattr(namespace, self.dest, value) From ef61fb5698c93178a57a9d4d67760def9a3039d9 Mon Sep 17 00:00:00 2001 From: tstauder <55719290+tstauder@users.noreply.github.com> Date: Wed, 19 Apr 2023 02:33:32 -0400 Subject: [PATCH 24/65] Fix flaky tests involving feed export batches (#5898) Co-authored-by: jmannoop --- tests/test_feedexport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3124d9d67..83de0e77e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2542,7 +2542,7 @@ class BatchDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(batch_time)s replaced with the current date. + So %(batch_id)d replaced with the current id. """ items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2552,7 +2552,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = { "FEEDS": { self._random_temp_filename() - / "%(batch_time)s": { + / "%(batch_id)d": { "format": "json", }, }, @@ -2615,7 +2615,7 @@ class BatchDeliveriesTest(FeedExportTestBase): return super().open(*args, **kwargs) key = "export.csv" - uri = f"s3://{bucket}/{key}/%(batch_time)s.json" + uri = f"s3://{bucket}/{key}/%(batch_id)d.json" batch_item_count = 1 settings = { "AWS_ACCESS_KEY_ID": "access_key", From b7ecec18099ace6ba77161302cca85c3e58a2ae7 Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Wed, 19 Apr 2023 01:04:03 -0600 Subject: [PATCH 25/65] Update scrapy/utils/curl.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/curl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 9c98e4cb8..790c26b1a 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -10,8 +10,9 @@ from w3lib.http import basic_auth_header class DataAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): - value = str(values).encode("utf-8").decode("utf-8") - value = value[1::] if re.match(r"^\$(.+)", value) else value + value = str(values) + if value.startswith("$"): + value = value[1:] setattr(namespace, self.dest, value) From f69ba43f8e5b52af51119c682509167f5cf5517f Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Wed, 19 Apr 2023 02:06:00 -0500 Subject: [PATCH 26/65] fix: import re --- scrapy/utils/curl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 790c26b1a..f5dbbd64e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,5 +1,4 @@ import argparse -import re import warnings from http.cookies import SimpleCookie from shlex import split From 87c8c5199902b448e90708e779470494f365c647 Mon Sep 17 00:00:00 2001 From: DeflateAwning <11021263+DeflateAwning@users.noreply.github.com> Date: Thu, 20 Apr 2023 00:23:02 -0600 Subject: [PATCH 27/65] Fix a typo in the FAQ (#5904) --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 031f4b942..20dd814df 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -231,8 +231,8 @@ Can I return (Twisted) deferreds from signal handlers? Some signals support returning deferreds from their handlers, others don't. See the :ref:`topics-signals-ref` to know which ones. -What does the response status code 999 means? ---------------------------------------------- +What does the response status code 999 mean? +-------------------------------------------- 999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or From 5a37af146f4f036168ac562918fca43adb4ac65f Mon Sep 17 00:00:00 2001 From: Jalil SA Date: Fri, 21 Apr 2023 01:29:57 -0600 Subject: [PATCH 28/65] Update documentation expectations for Parsel 1.8.0 (#5902) --- docs/intro/tutorial.rst | 8 ++++---- pytest.ini | 1 - tox.ini | 11 ++++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 04d73d95a..19a76fc16 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -252,7 +252,7 @@ object: .. code-block:: pycon >>> response.css("title") - [] + [] The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of @@ -348,7 +348,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: .. code-block:: pycon >>> response.xpath("//title") - [] + [] >>> response.xpath("//title/text()").get() 'Quotes to Scrape' @@ -410,8 +410,8 @@ We get a list of selectors for the quote HTML elements with: .. code-block:: pycon >>> response.css("div.quote") - [, - , + [, + , ...] Each of the selectors returned by the query above allows us to run further diff --git a/pytest.ini b/pytest.ini index f5fbf2529..866f0c950 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,7 +5,6 @@ python_files=test_*.py __init__.py python_classes= addopts = --assert=plain - --doctest-modules --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst diff --git a/tox.ini b/tox.ini index 06b52f3dc..9d81ec3e7 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -99,6 +99,8 @@ setenv = _SCRAPY_PINNED=true install_command = python -I -m pip install {opts} {packages} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} [testenv:pinned] basepython = python3.7 @@ -108,6 +110,7 @@ deps = install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:windows-pinned] basepython = python3 @@ -117,6 +120,7 @@ deps = install_command = {[pinned]install_command} setenv = {[pinned]setenv} +commands = {[pinned]commands} [testenv:extra-deps] basepython = python3 @@ -137,7 +141,7 @@ commands = [testenv:asyncio-pinned] deps = {[testenv:pinned]deps} -commands = {[testenv:asyncio]commands} +commands = {[pinned]commands} --reactor=asyncio install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -152,7 +156,8 @@ basepython = {[testenv:pypy3]basepython} deps = {[pinned]deps} PyPyDispatcher==2.1.0 -commands = {[testenv:pypy3]commands} +commands = + pytest --durations=10 scrapy tests install_command = {[pinned]install_command} setenv = {[pinned]setenv} From 67bfb304cdeb19a0b72b0a09542582f718cf07d6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 21 Apr 2023 18:52:58 +0400 Subject: [PATCH 29/65] Release notes for the current master. --- docs/news.rst | 105 ++++++++++++++++++++++++++++++++++++++- docs/topics/settings.rst | 2 +- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 9b9eeac71..6cf366449 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,108 @@ Release notes ============= +.. _release-2.9.0: + +Scrapy 2.9.0 (YYYY-MM-DD) +------------------------- + +Highlights: + +- Per-domain request settings. +- Compatibility with new cryptography_ and new parsel_. +- TBD + +New features +~~~~~~~~~~~~ + +- Settings correponding to :setting:`DOWNLOAD_DELAY`, + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis + via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + +- Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a + curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) + +- Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be + :class:`pathlib.Path` instances. (:issue:`5801`) + +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The ``scrapy parse`` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The ``scrapy genspider`` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + +Bug fixes +~~~~~~~~~ + +- Fixed an error when using feed postprocessing with S3 storage. + (:issue:`5500`, :issue:`5581`) + +- Added the missing :meth:`scrapy.settings.BaseSettings.setdefault` method. + (:issue:`5811`, :issue:`5821`) + +- Fixed an error when using cryptography_ 40.0.0+ and + :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` is enabled. + (:issue:`5857`, :issue:`5858`) + +- The checksums returned by :class:`~scrapy.pipelines.files.FilesPipeline` + for files on Google Cloud Storage are no longer Base64-encoded. + (:issue:`5874`, :issue:`5891`) + +- Fixed an error breaking user handling of send failures in + :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + +Documentation +~~~~~~~~~~~~~ + +- Expanded contributing docs. (:issue:`5109`, :issue:`5851`) + +- Added blacken-docs_ to pre-commit and reformatted the docs with it. + (:issue:`5813`, :issue:`5816`) + +- Fixed a JS issue. (:issue:`5875`, :issue:`5877`) + +- Fixed ``make htmlview``. (:issue:`5878`, :issue:`5879`) + +- Fixed typos and other small errors. (:issue:`5827`, :issue:`5839`, + :issue:`5883`, :issue:`5890`, :issue:`5895`, :issue:`5904`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) + +- Tests for most of the examples in the docs are now run as a part of CI, + found problems were fixed. (:issue:`5816`, :issue:`5826`) + +- Removed usage of deprecated Python classes. (:issue:`5849`) + +- Silenced ``include-ignored`` warnings from coverage. (:issue:`5820`) + +- Fixed a random failure of the ``test_feedexport.test_batch_path_differ`` + test. (:issue:`5855`, :issue:`5898`) + +- Updated docstrings to match output produced by parsel_ 1.8.1 so that they + don't cause test failures. (:issue:`5902`) + +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`) + +.. _blacken-docs: https://github.com/adamchainz/blacken-docs + .. _release-2.8.0: Scrapy 2.8.0 (2023-02-02) @@ -4207,8 +4309,6 @@ Relocations + Note: telnet is not enabled on Python 3 (https://github.com/scrapy/scrapy/pull/1524#issuecomment-146985595) -.. _parsel: https://github.com/scrapy/parsel - Bugfixes ~~~~~~~~ @@ -5638,6 +5738,7 @@ First release of Scrapy. .. _LevelDB: https://github.com/google/leveldb .. _lxml: https://lxml.de/ .. _marshal: https://docs.python.org/2/library/marshal.html +.. _parsel: https://github.com/scrapy/parsel .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4412b5c1c..3e06d84f9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,7 +783,7 @@ DOWNLOAD_SLOTS Default: ``{}`` -Allows to define concurrency/delay parameters on per slot(domain) basis: +Allows to define concurrency/delay parameters on per slot (domain) basis: .. code-block:: python From 8c8fb67057609d79a31e76587fc66b162d746906 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Apr 2023 11:34:34 +0400 Subject: [PATCH 30/65] Update tool versions (#5908) --- .bandit.yml | 1 + .pre-commit-config.yaml | 6 +++--- tox.ini | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.bandit.yml b/.bandit.yml index c8e84cc2e..2aae8a0aa 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,5 +1,6 @@ skips: - B101 +- B113 # https://github.com/PyCQA/bandit/issues/1010 - B105 - B301 - B303 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b90233e5..faf8808f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.5 hooks: - id: bandit args: [-r, -c, .bandit.yml] @@ -9,7 +9,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/psf/black.git - rev: 23.1.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/pycqa/isort @@ -21,4 +21,4 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.1.0 + - black==23.3.0 diff --git a/tox.ini b/tox.ini index 9d81ec3e7..af8f1f57a 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==2.16.0 + pylint==2.17.2 commands = pylint conftest.py docs extras scrapy setup.py tests From 9af596a6b806a0cd0ba7f0d3bcff9ea6e3a19519 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 25 Apr 2023 10:24:14 -0500 Subject: [PATCH 31/65] feat: Add support for the Parsel JMESPath --- scrapy/http/response/__init__.py | 6 ++++++ scrapy/http/response/text.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 4213d491d..a82ed834a 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -142,6 +142,12 @@ class Response(object_ref): """ raise NotSupported("Response content isn't text") + def jmespath(self, *a, **kw): + """Shortcut method implemented only by responses whose content + is text (subclasses of TextResponse). + """ + raise NotSupported("Response content isn't text") + def xpath(self, *a, **kw): """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 73bb811de..360d6334e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -139,6 +139,9 @@ class TextResponse(Response): self._cached_selector = Selector(self) return self._cached_selector + def jmespath(self, query, **kwargs): + return self.selector.jmespath(query, **kwargs) + def xpath(self, query, **kwargs): return self.selector.xpath(query, **kwargs) From b50c032ee9a75d1c9b42f1126637fdc655b141a8 Mon Sep 17 00:00:00 2001 From: guillermo-bondonno <95530227+guillermo-bondonno@users.noreply.github.com> Date: Wed, 26 Apr 2023 03:20:37 -0300 Subject: [PATCH 32/65] Add feed_slot_closed and feed_exporter_closed signals (#5876) --- docs/topics/signals.rst | 27 ++++++++++ scrapy/extensions/feedexport.py | 43 +++++++++++---- scrapy/signals.py | 2 + tests/test_feedexport.py | 92 ++++++++++++++++++++++++++++++--- 4 files changed, 148 insertions(+), 16 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3400a205a..9bfd1761c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -307,6 +307,33 @@ spider_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object +feed_slot_closed +~~~~~~~~~~~~~~~~ + +.. signal:: feed_slot_closed +.. function:: feed_slot_closed(slot) + + Sent when a :ref:`feed exports ` slot is closed. + + This signal supports returning deferreds from its handlers. + + :param slot: the slot closed + :type slot: scrapy.extensions.feedexport.FeedSlot + + +feed_exporter_closed +~~~~~~~~~~~~~~~~~~~~ + +.. signal:: feed_exporter_closed +.. function:: feed_exporter_closed() + + Sent when the :ref:`feed exports ` extension is closed, + during the handling of the :signal:`spider_closed` signal by the extension, + after all feed exporting has been handled. + + This signal supports returning deferreds from its handlers. + + Request signals --------------- diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index da1a88299..bcf0b779a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,10 +11,11 @@ import warnings from datetime import datetime from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, Any, Callable, Optional, Tuple, Union +from typing import IO, Any, Callable, List, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads +from twisted.internet.defer import DeferredList from w3lib.url import file_uri_to_path from zope.interface import Interface, implementer @@ -23,6 +24,8 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object @@ -271,7 +274,7 @@ class FTPFeedStorage(BlockingFeedStorage): ) -class _FeedSlot: +class FeedSlot: def __init__( self, file, @@ -309,7 +312,15 @@ class _FeedSlot: self._exporting = False +_FeedSlot = create_deprecated_class( + name="_FeedSlot", + new_class=FeedSlot, +) + + class FeedExporter: + _pending_deferreds: List[defer.Deferred] = [] + @classmethod def from_crawler(cls, crawler): exporter = cls(crawler) @@ -375,12 +386,18 @@ class FeedExporter: ) ) - def close_spider(self, spider): - deferred_list = [] + async def close_spider(self, spider): for slot in self.slots: - d = self._close_slot(slot, spider) - deferred_list.append(d) - return defer.DeferredList(deferred_list) if deferred_list else None + self._close_slot(slot, spider) + + # Await all deferreds + if self._pending_deferreds: + await maybe_deferred_to_future(DeferredList(self._pending_deferreds)) + + # Send FEED_EXPORTER_CLOSED signal + await maybe_deferred_to_future( + self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) + ) def _close_slot(self, slot, spider): def get_file(slot_): @@ -404,6 +421,14 @@ class FeedExporter: d.addErrback( self._handle_store_error, logmsg, spider, type(slot.storage).__name__ ) + self._pending_deferreds.append(d) + d.addCallback( + lambda _: self.crawler.signals.send_catch_log_deferred( + signals.feed_slot_closed, slot=slot + ) + ) + d.addBoth(lambda _: self._pending_deferreds.remove(d)) + return d def _handle_store_error(self, f, logmsg, spider, slot_type): @@ -444,7 +469,7 @@ class FeedExporter: indent=feed_options["indent"], **feed_options["item_export_kwargs"], ) - slot = _FeedSlot( + slot = FeedSlot( file=file, exporter=exporter, storage=storage, @@ -579,7 +604,7 @@ class FeedExporter: self, spider: Spider, uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]], - slot: Optional[_FeedSlot] = None, + slot: Optional[FeedSlot] = None, ) -> dict: params = {} for k in dir(spider): diff --git a/scrapy/signals.py b/scrapy/signals.py index 8cf2a4d93..0090f1c8b 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -22,6 +22,8 @@ bytes_received = object() item_scraped = object() item_dropped = object() item_error = object() +feed_slot_closed = object() +feed_exporter_closed = object() # for backward compatibility stats_spider_opened = spider_opened diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 83de0e77e..b1059099a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -32,18 +32,19 @@ from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy +from scrapy import signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( BlockingFeedStorage, FeedExporter, + FeedSlot, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, IFeedStorage, S3FeedStorage, StdoutFeedStorage, - _FeedSlot, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -660,8 +661,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return result -class InstrumentedFeedSlot(_FeedSlot): - """Instrumented _FeedSlot subclass for keeping track of calls to +class InstrumentedFeedSlot(FeedSlot): + """Instrumented FeedSlot subclass for keeping track of calls to start_exporting and finish_exporting.""" def start_exporting(self): @@ -964,7 +965,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -982,7 +983,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -1003,7 +1004,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -1022,7 +1023,7 @@ class FeedExportTest(FeedExportTestBase): listener = IsExportingListener() InstrumentedFeedSlot.subscribe__listener(listener) - with mock.patch("scrapy.extensions.feedexport._FeedSlot", InstrumentedFeedSlot): + with mock.patch("scrapy.extensions.feedexport.FeedSlot", InstrumentedFeedSlot): _ = yield self.exported_data(items, settings) self.assertFalse(listener.start_without_finish) self.assertFalse(listener.finish_without_start) @@ -2651,6 +2652,83 @@ class BatchDeliveriesTest(FeedExportTestBase): stub.assert_no_pending_responses() +# Test that the FeedExporer sends the feed_exporter_closed and feed_slot_closed signals +class FeedExporterSignalsTest(unittest.TestCase): + items = [ + {"foo": "bar1", "egg": "spam1"}, + {"foo": "bar2", "egg": "spam2", "baz": "quux2"}, + {"foo": "bar3", "baz": "quux3"}, + ] + + with tempfile.NamedTemporaryFile(suffix="json") as tmp: + settings = { + "FEEDS": { + f"file:///{tmp.name}": { + "format": "json", + }, + }, + } + + def feed_exporter_closed_signal_handler(self): + self.feed_exporter_closed_received = True + + def feed_slot_closed_signal_handler(self, slot): + self.feed_slot_closed_received = True + + def feed_exporter_closed_signal_handler_deferred(self): + d = defer.Deferred() + d.addCallback(lambda _: setattr(self, "feed_exporter_closed_received", True)) + d.callback(None) + return d + + def feed_slot_closed_signal_handler_deferred(self, slot): + d = defer.Deferred() + d.addCallback(lambda _: setattr(self, "feed_slot_closed_received", True)) + d.callback(None) + return d + + def run_signaled_feed_exporter( + self, feed_exporter_signal_handler, feed_slot_signal_handler + ): + crawler = get_crawler(settings_dict=self.settings) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") + spider.crawler = crawler + crawler.signals.connect( + feed_exporter_signal_handler, + signal=signals.feed_exporter_closed, + ) + crawler.signals.connect( + feed_slot_signal_handler, signal=signals.feed_slot_closed + ) + feed_exporter.open_spider(spider) + for item in self.items: + feed_exporter.item_scraped(item, spider) + defer.ensureDeferred(feed_exporter.close_spider(spider)) + + def test_feed_exporter_signals_sent(self): + self.feed_exporter_closed_received = False + self.feed_slot_closed_received = False + + self.run_signaled_feed_exporter( + self.feed_exporter_closed_signal_handler, + self.feed_slot_closed_signal_handler, + ) + self.assertTrue(self.feed_slot_closed_received) + self.assertTrue(self.feed_exporter_closed_received) + + def test_feed_exporter_signals_sent_deferred(self): + self.feed_exporter_closed_received = False + self.feed_slot_closed_received = False + + self.run_signaled_feed_exporter( + self.feed_exporter_closed_signal_handler_deferred, + self.feed_slot_closed_signal_handler_deferred, + ) + self.assertTrue(self.feed_slot_closed_received) + self.assertTrue(self.feed_exporter_closed_received) + + class FeedExportInitTest(unittest.TestCase): def test_unsupported_storage(self): settings = { From 865c36bdbbd7af0e5dd9c5c333d2285e88dfbcfd Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 28 Apr 2023 08:56:11 -0600 Subject: [PATCH 33/65] update docs --- docs/topics/request-response.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 99c7915df..407df32d2 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1281,6 +1281,12 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: + .. method:: TextResponse.jmespath(query) + + A shortcut to ``TextResponse.selector.jmespath(query)``:: + + response.jmespath('object.[*]') + .. method:: TextResponse.xpath(query) A shortcut to ``TextResponse.selector.xpath(query)``:: From 3d29f20fc2021efb80d96389dc02c5a2cac9ef62 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 28 Apr 2023 23:54:09 -0600 Subject: [PATCH 34/65] added tests for jmespath --- tests/test_selector.py | 138 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index febae46ac..b0deb5c99 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -108,3 +108,141 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, "received both response and text"): Selector(TextResponse(url="http://example.com", body=b""), text="") + + +class JMESPathTestCase(unittest.TestCase): + def test_json_has_html(self) -> None: + """Sometimes the information is returned in a json wrapper""" + body = """ + { + "content": [ + { + "name": "A", + "value": "a" + }, + { + "name": { + "age": 18 + }, + "value": "b" + }, + { + "name": "C", + "value": "c" + }, + { + "name": "D", + "value": "
d
" + } + ], + "html": "
def
" + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.jmespath("html").get(), + "
def
", + ) + self.assertEqual( + resp.jmespath("html").xpath("//div/a/text()").getall(), + ["a", "b", "d"], + ) + self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) + self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + + def test_html_has_json(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + + def test_jmestpath_with_re(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), + None, + ) + + self.assertEqual( + resp.xpath("//div/content") + .jmespath("user[*].age.to_string(@)") + .re(r"(\d+)"), + ["18", "32", "22", "25"], + ) From 578606779d0f127a759f8b1623c1e4be341a17db Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Sat, 29 Apr 2023 00:52:39 -0600 Subject: [PATCH 35/65] update tests --- tests/test_http_response.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dbc9f1fef..cefdb1709 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -32,6 +32,9 @@ class BaseResponseTest(unittest.TestCase): isinstance(self.response_class("http://example.com/"), self.response_class) ) self.assertRaises(TypeError, self.response_class, b"http://example.com") + self.assertRaises( + TypeError, self.response_class, url="http://example.com", body={} + ) # body can be str or None self.assertTrue( isinstance( @@ -192,6 +195,7 @@ class BaseResponseTest(unittest.TestCase): self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") self.assertRaisesRegex(NotSupported, msg, r.css, "body") self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") + self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") else: r.text r.css("body") From 8acde511a902515c56f7452f950221657953b92e Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 2 May 2023 12:11:23 -0300 Subject: [PATCH 36/65] fix: non-UTF-8 content-type headers --- scrapy/http/response/text.py | 6 ++++-- scrapy/responsetypes.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 73bb811de..d580a7876 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -100,11 +100,13 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): content_type = self.headers.get(b"Content-Type", b"") - return http_content_type_encoding(to_unicode(content_type)) + return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_unicode(self.headers.get(b"Content-Type", b"")) + content_type = to_unicode( + self.headers.get(b"Content-Type", b""), encoding="latin-1" + ) benc, ubody = html_to_unicode( content_type, self.body, diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index f01e9096c..58884f21a 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -51,7 +51,9 @@ class ResponseTypes: header""" if content_encoding: return Response - mimetype = to_unicode(content_type).split(";")[0].strip().lower() + mimetype = ( + to_unicode(content_type, encoding="latin-1").split(";")[0].strip().lower() + ) return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): From 7b49aa1b019672d4f2ab7a8d75465381cb5705a4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 2 May 2023 12:53:04 -0300 Subject: [PATCH 37/65] chore: add tests --- tests/test_http_response.py | 11 +++++++++++ tests/test_responsetypes.py | 1 + 2 files changed, 12 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dbc9f1fef..a05b702aa 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -448,6 +448,13 @@ class TextResponseTest(BaseResponseTest): body=codecs.BOM_UTF8 + b"\xc2\xa3", headers={"Content-type": ["text/html; charset=cp1251"]}, ) + r9 = self.response_class( + "http://www.example.com", + body=b"\x80", + headers={ + "Content-type": [b"application/x-download; filename=\x80dummy.txt"] + }, + ) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -458,9 +465,12 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r9._headers_encoding(), None) self.assertEqual(r8._declared_encoding(), "utf-8") + self.assertEqual(r9._declared_encoding(), None) self._assert_response_encoding(r5, "utf-8") self._assert_response_encoding(r8, "utf-8") + self._assert_response_encoding(r9, "cp1252") assert ( r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != "ascii" @@ -470,6 +480,7 @@ class TextResponseTest(BaseResponseTest): self._assert_response_values(r3, "iso-8859-1", "\xa3") self._assert_response_values(r6, "gb18030", "\u2015") self._assert_response_values(r7, "gb18030", "\u2015") + self._assert_response_values(r9, "cp1252", "€") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies self.assertRaises( diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 859960518..6e1ed82f0 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -42,6 +42,7 @@ class ResponseTypesTest(unittest.TestCase): ("application/octet-stream", Response), ("application/x-json; encoding=UTF8;charset=UTF-8", TextResponse), ("application/json-amazonui-streaming;charset=UTF-8", TextResponse), + (b"application/x-download; filename=\x80dummy.txt", Response), ] for source, cls in mappings: retcls = responsetypes.from_content_type(source) From 1eb44604853e24f7b526853d5e95414949fd88c6 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 18:49:05 -0600 Subject: [PATCH 38/65] fix: jmespath --- scrapy/http/response/text.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 360d6334e..dd042a2bd 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -140,7 +140,12 @@ class TextResponse(Response): return self._cached_selector def jmespath(self, query, **kwargs): - return self.selector.jmespath(query, **kwargs) + if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + raise AttributeError( + "Please install parsel >= 1.8.1 to get jmespath support" + ) + + return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined] def xpath(self, query, **kwargs): return self.selector.xpath(query, **kwargs) From a604dfae5c12a55760d0b44d7da06b022cce3615 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:19:00 -0600 Subject: [PATCH 39/65] update tests --- tests/test_selector.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index b0deb5c99..274d63d8d 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,10 +1,16 @@ import weakref +import packaging.version as version +import parsel +import pytest from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector +PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) +PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") + class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): @@ -111,8 +117,12 @@ class SelectorTestCase(unittest.TestCase): class JMESPathTestCase(unittest.TestCase): + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_json_has_html(self) -> None: """Sometimes the information is returned in a json wrapper""" + body = """ { "content": [ @@ -150,6 +160,9 @@ class JMESPathTestCase(unittest.TestCase): self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_html_has_json(self) -> None: body = """
@@ -191,6 +204,9 @@ class JMESPathTestCase(unittest.TestCase): ) self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_jmestpath_with_re(self) -> None: body = """
@@ -246,3 +262,14 @@ class JMESPathTestCase(unittest.TestCase): .re(r"(\d+)"), ["18", "32", "22", "25"], ) + + @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") + def test_jmespath_not_available(my_json_page) -> None: + body = """ + { + "website": {"name": "Example"} + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + with pytest.raises(AttributeError): + resp.jmespath("website.name").get() From 4bb99fd2f3a957958ed9ea8fa418b6127a6bebee Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:26:20 -0600 Subject: [PATCH 40/65] fix: pylint --- tests/test_selector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 274d63d8d..311c09aba 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,8 +1,8 @@ import weakref -import packaging.version as version import parsel import pytest +from packaging import version from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse @@ -11,6 +11,9 @@ from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") +print(PARSEL_VERSION) +print(PARSEL_18_PLUS) + class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): From a038faf11c2bc47921f57954a2405279d427f890 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:40:04 -0600 Subject: [PATCH 41/65] fix: tests/tes_selector.py --- tests/test_selector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 311c09aba..85527bba9 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -11,9 +11,6 @@ from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") -print(PARSEL_VERSION) -print(PARSEL_18_PLUS) - class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): From d907f9e09284367d555c89d2bea862a310f60a19 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 3 May 2023 22:12:21 -0300 Subject: [PATCH 42/65] fix: Handle Parsel > 1.7.0 warning --- scrapy/selector/unified.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index cff97104a..208dd807f 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -4,6 +4,11 @@ XPath selectors based on lxml from parsel import Selector as _ParselSelector +try: + from parsel.selector import _NOT_SET +except ImportError: + _NOT_SET = None + from scrapy.http import HtmlResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref @@ -63,7 +68,7 @@ class Selector(_ParselSelector, object_ref): __slots__ = ["response"] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=None, **kwargs): + def __init__(self, response=None, text=None, type=None, root=_NOT_SET, **kwargs): if response is not None and text is not None: raise ValueError( f"{self.__class__.__name__}.__init__() received " From 7317ff11014c4bf20d4e35193cc96a7151d1d5b0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 4 May 2023 05:55:25 -0300 Subject: [PATCH 43/65] refactor: use kwargs strategy --- scrapy/selector/unified.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 208dd807f..caff79e9c 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -4,17 +4,14 @@ XPath selectors based on lxml from parsel import Selector as _ParselSelector -try: - from parsel.selector import _NOT_SET -except ImportError: - _NOT_SET = None - from scrapy.http import HtmlResponse, XmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref __all__ = ["Selector", "SelectorList"] +_NOT_SET = object() + def _st(response, st): if st is None: @@ -85,4 +82,8 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault("base_url", response.url) self.response = response - super().__init__(text=text, type=st, root=root, **kwargs) + + if root is not _NOT_SET: + kwargs["root"] = root + + super().__init__(text=text, type=st, **kwargs) From cba891a66c2223745d7b8484550ed54ca1e155ec Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 15:04:33 +0400 Subject: [PATCH 44/65] Enable doc tests for selectors.rst, fix issues. --- docs/topics/selectors.rst | 81 ++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c25c75d17..4a64d530b 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -48,6 +48,8 @@ Constructing selectors .. highlight:: python +.. skip: start + Response objects expose a :class:`~scrapy.Selector` instance on ``.selector`` attribute: @@ -66,6 +68,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: >>> response.css("span::text").get() 'good' +.. skip: end + Scrapy selectors are instances of :class:`~scrapy.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or markup as a string (in ``text`` argument). @@ -93,7 +97,7 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of >>> from scrapy.selector import Selector >>> from scrapy.http import HtmlResponse - >>> response = HtmlResponse(url="http://example.com", body=body) + >>> response = HtmlResponse(url="http://example.com", body=body, encoding="utf-8") >>> Selector(response=response).xpath("//span/text()").get() 'good' @@ -103,6 +107,13 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of Using selectors --------------- +.. invisible-code-block: python + + html_response = response = load_response( + "https://docs.scrapy.org/en/latest/_static/selectors-sample1.html", + "../_static/selectors-sample1.html", + ) + To explain how to use the selectors we'll use the ``Scrapy shell`` (which provides interactive testing) and an example page located in the Scrapy documentation server: @@ -135,7 +146,7 @@ page, let's construct an XPath for selecting the text inside the title tag: .. code-block:: pycon >>> response.xpath("//title/text()") - [] + [] To actually extract the textual data, you must call the selector ``.get()`` or ``.getall()`` methods, as follows: @@ -363,11 +374,11 @@ too. Here's an example: >>> links = response.xpath('//a[contains(@href, "image")]') >>> links.getall() - ['Name: My image 1
', - 'Name: My image 2
', - 'Name: My image 3
', - 'Name: My image 4
', - 'Name: My image 5
'] + ['Name: My image 1
image1', + 'Name: My image 2
image2', + 'Name: My image 3
image3', + 'Name: My image 4
image4', + 'Name: My image 5
image5'] >>> for index, link in enumerate(links): ... href_xpath = link.xpath("@href").get() @@ -447,11 +458,11 @@ Here's an example used to extract image names from the :ref:`HTML code .. code-block:: pycon >>> response.xpath('//a[contains(@href, "image")]/text()').re(r"Name:\s*(.*)") - ['My image 1', - 'My image 2', - 'My image 3', - 'My image 4', - 'My image 5'] + ['My image 1 ', + 'My image 2 ', + 'My image 3 ', + 'My image 4 ', + 'My image 5 '] There's an additional helper reciprocating ``.get()`` (and its alias ``.extract_first()``) for ``.re()``, named ``.re_first()``. @@ -460,7 +471,7 @@ Use it to extract just the first matching string: .. code-block:: pycon >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r"Name:\s*(.*)") - 'My image 1' + 'My image 1 ' .. _old-extraction-api: @@ -761,6 +772,8 @@ on `XPath variables`_. Removing namespaces ------------------- +.. skip: start + When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more simple/convenient XPaths. You can use the @@ -808,8 +821,8 @@ nodes can be accessed directly by their names: >>> response.selector.remove_namespaces() >>> response.xpath("//link") - [, - , + [, + , ... If you wonder why the namespace removal procedure isn't always called by default @@ -824,6 +837,7 @@ of relevance, are: case some element names clash between namespaces. These cases are very rare though. +.. skip: end Using EXSLT extensions ---------------------- @@ -881,6 +895,8 @@ extracting text elements for example. Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops: +.. skip: next + .. code-block:: pycon >>> doc = """ @@ -977,26 +993,35 @@ Scrapy selectors also provide a sorely missed XPath extension function ``has-class`` that returns ``True`` for nodes that have all of the specified HTML classes. -.. highlight:: html +For the following HTML: -For the following HTML:: +.. code-block:: pycon -

First

-

Second

-

Third

-

Fourth

- -.. highlight:: python + >>> from scrapy.http import HtmlResponse + >>> response = HtmlResponse( + ... url="http://example.com", + ... body=""" + ... + ... + ...

First

+ ...

Second

+ ...

Third

+ ...

Fourth

+ ... + ... + ... """, + ... encoding="utf-8", + ... ) You can use it like this: .. code-block:: pycon >>> response.xpath('//p[has-class("foo")]') - [, - ] + [, + ] >>> response.xpath('//p[has-class("foo", "bar-baz")]') - [] + [] >>> response.xpath('//p[has-class("foo", "bar")]') [] @@ -1132,6 +1157,8 @@ a :class:`~scrapy.http.HtmlResponse` object like this: Selector examples on XML response --------------------------------- +.. skip: start + Here are some examples to illustrate concepts for :class:`Selector` objects instantiated with an :class:`~scrapy.http.XmlResponse` object: @@ -1154,4 +1181,6 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object: sel.register_namespace("g", "http://base.google.com/ns/1.0") sel.xpath("//g:price").getall() +.. skip: end + .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799 From d1d6465ef4ea8987efb08e2f9abfd65b36719e04 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 17:19:01 +0400 Subject: [PATCH 45/65] Address feedback. --- docs/news.rst | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 6cf366449..5f189760d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,7 +10,7 @@ Scrapy 2.9.0 (YYYY-MM-DD) Highlights: -- Per-domain request settings. +- Per-domain download settings. - Compatibility with new cryptography_ and new parsel_. - TBD @@ -19,7 +19,7 @@ New features - Settings correponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per domain basis + :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) - Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a @@ -28,26 +28,6 @@ New features - Values of :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can now be :class:`pathlib.Path` instances. (:issue:`5801`) -- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed - string values for the curl ``--data-raw`` argument, which are produced by - browsers for data that includes certain symbols. (:issue:`5899`, - :issue:`5901`) - -- The ``scrapy parse`` command now also works with async generator callbacks. - (:issue:`5819`, :issue:`5824`) - -- The ``scrapy genspider`` command now properly works with HTTPS URLs. - (:issue:`3553`, :issue:`5808`) - -- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) - -- :class:`LinkExtractor ` - now skips certain malformed URLs instead of raising an exception. - (:issue:`5881`) - -- :func:`scrapy.utils.python.get_func_args` now supports more types of - callables. (:issue:`5872`, :issue:`5885`) - Bug fixes ~~~~~~~~~ @@ -65,6 +45,26 @@ Bug fixes for files on Google Cloud Storage are no longer Base64-encoded. (:issue:`5874`, :issue:`5891`) +- :func:`scrapy.utils.request.request_from_curl` now supports $-prefixed + string values for the curl ``--data-raw`` argument, which are produced by + browsers for data that includes certain symbols. (:issue:`5899`, + :issue:`5901`) + +- The :command:`parse` command now also works with async generator callbacks. + (:issue:`5819`, :issue:`5824`) + +- The :command:`genspider` command now properly works with HTTPS URLs. + (:issue:`3553`, :issue:`5808`) + +- Improved handling of asyncio loops. (:issue:`5831`, :issue:`5832`) + +- :class:`LinkExtractor ` + now skips certain malformed URLs instead of raising an exception. + (:issue:`5881`) + +- :func:`scrapy.utils.python.get_func_args` now supports more types of + callables. (:issue:`5872`, :issue:`5885`) + - Fixed an error breaking user handling of send failures in :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) From 636559f1cc652e839ef42522e7168e3ff9d77921 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 4 May 2023 17:55:07 +0400 Subject: [PATCH 46/65] Add newer changes. --- docs/news.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 5f189760d..cbbb376e5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,8 @@ Highlights: - Per-domain download settings. - Compatibility with new cryptography_ and new parsel_. -- TBD +- JMESPath selectors from the new parsel_. +- Bug fixes. New features ~~~~~~~~~~~~ @@ -22,6 +23,12 @@ New features :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) +- Added :meth:`TextResponse.jmespath`, a shortcut for JMESPath selectors + available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) + +- Added :signal:`feed_slot_closed` and :signal:`feed_exporter_closed` + signals. (:issue:`5876`) + - Added :func:`scrapy.utils.request.request_to_curl`, a function to produce a curl command from a :class:`~scrapy.Request` object. (:issue:`5892`) @@ -31,6 +38,8 @@ New features Bug fixes ~~~~~~~~~ +- Fixed a warning with Parsel 1.8.1+. (:issue:`5903`, :issue:`5918`) + - Fixed an error when using feed postprocessing with S3 storage. (:issue:`5500`, :issue:`5581`) @@ -65,6 +74,9 @@ Bug fixes - :func:`scrapy.utils.python.get_func_args` now supports more types of callables. (:issue:`5872`, :issue:`5885`) +- Fixed an error when processing non-UTF8 values of ``Content-Type`` headers. + (:issue:`5914`, :issue:`5917`) + - Fixed an error breaking user handling of send failures in :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) @@ -89,7 +101,7 @@ Quality assurance - Extended typing hints. (:issue:`5805`, :issue:`5889`, :issue:`5896`) - Tests for most of the examples in the docs are now run as a part of CI, - found problems were fixed. (:issue:`5816`, :issue:`5826`) + found problems were fixed. (:issue:`5816`, :issue:`5826`, :issue:`5919`) - Removed usage of deprecated Python classes. (:issue:`5849`) @@ -99,9 +111,10 @@ Quality assurance test. (:issue:`5855`, :issue:`5898`) - Updated docstrings to match output produced by parsel_ 1.8.1 so that they - don't cause test failures. (:issue:`5902`) + don't cause test failures. (:issue:`5902`, :issue:`5919`) -- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`) +- Other CI and pre-commit improvements. (:issue:`5802`, :issue:`5823`, + :issue:`5908`) .. _blacken-docs: https://github.com/adamchainz/blacken-docs From caa66fa15ae5d353434e5bb1d0c1cc2bdf857b6c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 May 2023 13:27:01 +0400 Subject: [PATCH 47/65] Mention deprecating _FeedSlot. --- docs/news.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cbbb376e5..4e198c893 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -15,6 +15,13 @@ Highlights: - JMESPath selectors from the new parsel_. - Bug fixes. +Deprecations +~~~~~~~~~~~~ + +- :class:`scrapy.extensions.feedexport._FeedSlot` is renamed to + :class:`scrapy.extensions.feedexport.FeedSlot` and the old name is + deprecated. (:issue:`5876`) + New features ~~~~~~~~~~~~ From 52c072640aa61884de05214cb1bdda07c2a87bef Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 8 May 2023 14:30:06 +0400 Subject: [PATCH 48/65] =?UTF-8?q?Bump=20version:=202.8.0=20=E2=86=92=202.9?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4cfba674d..a00b7cfb3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.8.0 +current_version = 2.9.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 4e198c893..c7ad11862 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.9.0: -Scrapy 2.9.0 (YYYY-MM-DD) +Scrapy 2.9.0 (2023-05-08) ------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 834f26295..c8e38b614 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.8.0 +2.9.0 From c327a92e971411e50e49dded772a73b293f9f0a9 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 18:04:18 +0500 Subject: [PATCH 49/65] add additional requests examples. --- docs/topics/asyncio.rst | 47 +++++++++++++++++++++++++++++++++++++++++ scrapy/utils/defer.py | 10 +++++---- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7713b1af1..7aa83f505 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,6 +98,53 @@ Futures. Scrapy provides two helpers for this: into your own code. +Async additional requests +===================== + +The spider below shows a single use-case of scraping page and gathering price from a separate url:: + + + class SingleRequestSpider(scrapy.Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + 'h1': response.css('h1').get(), + 'price': additional_response.css('#price').get(), + } + + +Spider with gathering batch requests:: + + class BatchRequestsSpider(scrapy.Spider): + name = "batch" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + scrapy.Request("https://example.com/price1"), + scrapy.Request("https://example.com/price2"), + ] + coroutines = [] + for r in additional_requests: + deffered = self.crawler.engine.download(r) + coroutines.append(maybe_deferred_to_future(deffered)) + + responses = await asyncio.gather( + *coroutines, return_exceptions=True + ) + yield { + 'h1': response.css('h1::text').get(), + 'price': responses[0].css('.price_color::text').get(), + 'price2': responses[1].css('.price_color::text').get(), + } + + + .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d25ebbdf4..a46274fef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - additional_response = await deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await deferred_to_future(deferred) """ return d.asFuture(_get_asyncio_event_loop()) @@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: class MySpider(Spider): ... async def parse(self, response): - d = treq.get('https://example.com/additional') - extra_response = await maybe_deferred_to_future(d) + additional_request = scrapy.Request('https://example.org/price') + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) """ if not is_asyncio_reactor_installed(): return d From a75231a1ecd9a08b31ea3c1d5b59e457ac85ccf2 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 18:57:43 +0500 Subject: [PATCH 50/65] fix underline. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7aa83f505..d46527cfc 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this: Async additional requests -===================== +========================= The spider below shows a single use-case of scraping page and gathering price from a separate url:: From d32c6782347c97086e804da0da01f45622743198 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 19:02:34 +0500 Subject: [PATCH 51/65] Update description. --- docs/topics/asyncio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index d46527cfc..d439e0ab8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -101,7 +101,7 @@ Futures. Scrapy provides two helpers for this: Async additional requests ========================= -The spider below shows a single use-case of scraping page and gathering price from a separate url:: +The spider below shows a single use-case of scraping a page and gathering a price from a separate URL:: class SingleRequestSpider(scrapy.Spider): @@ -118,7 +118,7 @@ The spider below shows a single use-case of scraping page and gathering price fr } -Spider with gathering batch requests:: +The spider gathering batch requests:: class BatchRequestsSpider(scrapy.Spider): name = "batch" From 99b0ece165ff27b70a6d7375a86bcc67da111df7 Mon Sep 17 00:00:00 2001 From: bulat Date: Tue, 9 May 2023 20:27:46 +0500 Subject: [PATCH 52/65] remove extra line. --- docs/topics/asyncio.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index d439e0ab8..0dab0ac5e 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -144,7 +144,6 @@ The spider gathering batch requests:: } - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement From b1f4017788877ba2139f8621ecb5e821c62c111d Mon Sep 17 00:00:00 2001 From: bulat Date: Wed, 10 May 2023 15:34:58 +0500 Subject: [PATCH 53/65] Refactor batch sample. --- docs/topics/asyncio.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 0dab0ac5e..dc83148f5 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -126,8 +126,8 @@ The spider gathering batch requests:: async def parse(self, response, **kwargs): additional_requests = [ - scrapy.Request("https://example.com/price1"), - scrapy.Request("https://example.com/price2"), + scrapy.Request("https://example.com/price"), + scrapy.Request("https://example.com/color"), ] coroutines = [] for r in additional_requests: @@ -139,8 +139,8 @@ The spider gathering batch requests:: ) yield { 'h1': response.css('h1::text').get(), - 'price': responses[0].css('.price_color::text').get(), - 'price2': responses[1].css('.price_color::text').get(), + 'price': responses[0].css('.price::text').get(), + 'color': responses[1].css('color::text').get(), } From 87d10161cd413353eba2abf8ebdc2a8656927c43 Mon Sep 17 00:00:00 2001 From: bulat Date: Wed, 10 May 2023 15:35:48 +0500 Subject: [PATCH 54/65] Add selector as class. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index dc83148f5..f00ba0ff8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -140,7 +140,7 @@ The spider gathering batch requests:: yield { 'h1': response.css('h1::text').get(), 'price': responses[0].css('.price::text').get(), - 'color': responses[1].css('color::text').get(), + 'color': responses[1].css('.color::text').get(), } From 57f3140daaa0166f924fcca42d3f3d3ef178bf92 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Wed, 10 May 2023 18:31:54 +0500 Subject: [PATCH 55/65] Update docs/topics/asyncio.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index f00ba0ff8..f9efef108 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -101,8 +101,10 @@ Futures. Scrapy provides two helpers for this: Async additional requests ========================= -The spider below shows a single use-case of scraping a page and gathering a price from a separate URL:: +The spider below shows how to send a request and await its response all from +within a spider callback: +.. code-block:: python class SingleRequestSpider(scrapy.Spider): name = "single" From 26374e21f81eb53f5a21e1cc68a65e54fbb62cb6 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Wed, 10 May 2023 18:32:36 +0500 Subject: [PATCH 56/65] Update docs/topics/asyncio.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index f9efef108..7fe78585a 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -119,8 +119,9 @@ within a spider callback: 'price': additional_response.css('#price').get(), } +You can also send multiple requests in parallel: -The spider gathering batch requests:: +.. code-block:: python class BatchRequestsSpider(scrapy.Spider): name = "batch" From 85103b493289011161731e4fafec4320cd85e0af Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin Date: Thu, 11 May 2023 12:53:43 +0500 Subject: [PATCH 57/65] add proper example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/asyncio.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7fe78585a..eeec76157 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -123,6 +123,8 @@ You can also send multiple requests in parallel: .. code-block:: python + from twisted.internet.defer import DeferredList + class BatchRequestsSpider(scrapy.Spider): name = "batch" start_urls = ["https://example.com/product"] @@ -132,14 +134,11 @@ You can also send multiple requests in parallel: scrapy.Request("https://example.com/price"), scrapy.Request("https://example.com/color"), ] - coroutines = [] + deferreds = [] for r in additional_requests: - deffered = self.crawler.engine.download(r) - coroutines.append(maybe_deferred_to_future(deffered)) - - responses = await asyncio.gather( - *coroutines, return_exceptions=True - ) + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) yield { 'h1': response.css('h1::text').get(), 'price': responses[0].css('.price::text').get(), From 6194db133518b07b92bfdae35332c69e95f5c415 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 12:54:01 +0500 Subject: [PATCH 58/65] Update title. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 7fe78585a..1b670aaab 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,7 +98,7 @@ Futures. Scrapy provides two helpers for this: into your own code. -Async additional requests +Inline requests ========================= The spider below shows how to send a request and await its response all from From fc2d1b217130ebf605636049ea773196a50303a0 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 12:56:29 +0500 Subject: [PATCH 59/65] make example reachable. --- docs/topics/asyncio.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 12bf548df..fcf44c0cb 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,8 +125,8 @@ You can also send multiple requests in parallel: from twisted.internet.defer import DeferredList - class BatchRequestsSpider(scrapy.Spider): - name = "batch" + class MultipleRequestsSpider(scrapy.Spider): + name = "multiple" start_urls = ["https://example.com/product"] async def parse(self, response, **kwargs): @@ -141,8 +141,8 @@ You can also send multiple requests in parallel: responses = await maybe_deferred_to_future(DeferredList(deferreds)) yield { 'h1': response.css('h1::text').get(), - 'price': responses[0].css('.price::text').get(), - 'color': responses[1].css('.color::text').get(), + 'price': responses[0][1].css('.price::text').get(), + 'price2': responses[1][1].css('.color::text').get(), } From b62c1263de4b026e8529416d5dede1795d47f7ad Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:15:30 +0500 Subject: [PATCH 60/65] add import to the example. --- docs/topics/asyncio.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index fcf44c0cb..2ad784335 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -106,6 +106,8 @@ within a spider callback: .. code-block:: python + from scrapy.utils.defer import maybe_deferred_to_future + class SingleRequestSpider(scrapy.Spider): name = "single" start_urls = ["https://example.org/product"] From 4878cc7ef04ae40279d80fec5d5234d17569ce11 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:19:40 +0500 Subject: [PATCH 61/65] Add proper imports. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 2ad784335..a3f45a84b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,7 +125,7 @@ You can also send multiple requests in parallel: .. code-block:: python - from twisted.internet.defer import DeferredList + from scrapy.utils.defer import DeferredList class MultipleRequestsSpider(scrapy.Spider): name = "multiple" From 8de2064ba33d6e0b8e0a22a6b5f6928a35eb44b7 Mon Sep 17 00:00:00 2001 From: bulat Date: Thu, 11 May 2023 13:22:33 +0500 Subject: [PATCH 62/65] add import. --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index a3f45a84b..5e0063be0 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -125,7 +125,7 @@ You can also send multiple requests in parallel: .. code-block:: python - from scrapy.utils.defer import DeferredList + from scrapy.utils.defer import DeferredList, maybe_deferred_to_future class MultipleRequestsSpider(scrapy.Spider): name = "multiple" From 5adada5d19e9e275462330b070b746305115112e Mon Sep 17 00:00:00 2001 From: isabela_catanante Date: Fri, 12 May 2023 12:55:24 +0200 Subject: [PATCH 63/65] Improve the overwrite feed option documentation --- docs/topics/feed-exports.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..b4ac93b1d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -175,6 +175,12 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -209,6 +215,12 @@ You can also define a custom ACL and custom endpoint for exported feeds using th - :setting:`FEED_STORAGE_S3_ACL` - :setting:`AWS_ENDPOINT_URL` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. @@ -236,6 +248,12 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +storage backend is: ``True``. + +.. caution:: The value ``True`` in ``overwrite`` will cause you to lose the + previous version of your data. + This storage backend uses :ref:`delayed file delivery `. .. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python @@ -488,6 +506,8 @@ as a fallback value if that key is not provided for a specific feed definition: - :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported `_) + - :ref:`topics-feed-storage-gcs`: ``True`` (appending is not supported) + - :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported) .. versionadded:: 2.4.0 From e4cf8fc121fc89d70949a9159bfe67cbd0429e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 15 May 2023 18:51:58 +0200 Subject: [PATCH 64/65] Update asyncio.rst --- docs/topics/asyncio.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 5e0063be0..efb93c844 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -99,7 +99,7 @@ Futures. Scrapy provides two helpers for this: Inline requests -========================= +=============== The spider below shows how to send a request and await its response all from within a spider callback: From d362699fa3855c7fd6e11204ccd8668128e38a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 16 May 2023 13:39:02 +0200 Subject: [PATCH 65/65] Move inline request examples to the coroutines documentation --- docs/topics/asyncio.rst | 50 --------------------------------- docs/topics/coroutines.rst | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index efb93c844..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this: into your own code. -Inline requests -=============== - -The spider below shows how to send a request and await its response all from -within a spider callback: - -.. code-block:: python - - from scrapy.utils.defer import maybe_deferred_to_future - - class SingleRequestSpider(scrapy.Spider): - name = "single" - start_urls = ["https://example.org/product"] - - async def parse(self, response, **kwargs): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) - yield { - 'h1': response.css('h1').get(), - 'price': additional_response.css('#price').get(), - } - -You can also send multiple requests in parallel: - -.. code-block:: python - - from scrapy.utils.defer import DeferredList, maybe_deferred_to_future - - class MultipleRequestsSpider(scrapy.Spider): - name = "multiple" - start_urls = ["https://example.com/product"] - - async def parse(self, response, **kwargs): - additional_requests = [ - scrapy.Request("https://example.com/price"), - scrapy.Request("https://example.com/color"), - ] - deferreds = [] - for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) - yield { - 'h1': response.css('h1::text').get(), - 'price': responses[0][1].css('.price::text').get(), - 'price2': responses[1][1].css('.color::text').get(), - } - - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3916bd295..a65bab3ca 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -134,6 +134,63 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs +.. _inline-requests: + +Inline requests +=============== + +The spider below shows how to send a request and await its response all from +within a spider callback: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + + + class SingleRequestSpider(Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = Request("https://example.org/price") + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + "h1": response.css("h1").get(), + "price": additional_response.css("#price").get(), + } + +You can also send multiple requests in parallel: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + from twisted.internet.defer import DeferredList + + + class MultipleRequestsSpider(Spider): + name = "multiple" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + Request("https://example.com/price"), + Request("https://example.com/color"), + ] + deferreds = [] + for r in additional_requests: + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) + yield { + "h1": response.css("h1::text").get(), + "price": responses[0][1].css(".price::text").get(), + "price2": responses[1][1].css(".color::text").get(), + } + + .. _sync-async-spider-middleware: Mixing synchronous and asynchronous spider middlewares