From 25e616fa04bf4199b9017553b3e57699c40b0683 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 15 Mar 2019 18:09:47 +0530 Subject: [PATCH 01/78] do not degrade JPEG files. --- scrapy/pipelines/images.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a1457c7e9..add606a2e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -127,7 +127,7 @@ class ImagesPipeline(FilesPipeline): raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - image, buf = self.convert_image(orig_image) + image, buf = self.convert_image(orig_image, BytesIO(response.body)) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): @@ -135,7 +135,7 @@ class ImagesPipeline(FilesPipeline): thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None): + def convert_image(self, image, response_body, size=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -152,6 +152,9 @@ class ImagesPipeline(FilesPipeline): image = image.copy() image.thumbnail(size, Image.ANTIALIAS) + if not size and image.format == 'JPEG': + return image, response_body + buf = BytesIO() image.save(buf, 'JPEG') return image, buf From 07487dd487f7a10fbf0693e378b7b66909a67d6b Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 15 Mar 2019 23:29:53 +0530 Subject: [PATCH 02/78] make tests work with new convert_image --- tests/test_pipeline_images.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index a7c652959..eb3347442 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -79,28 +79,28 @@ class ImagesPipelineTestCase(unittest.TestCase): SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) + im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, _ = self.pipeline.convert_image(im) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -406,7 +406,7 @@ def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) - return Image.open(buf) + return Image.open(buf), buf if __name__ == "__main__": From ca882d8d9f094244d9f1d3476fb72fab9c230765 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Wed, 27 Mar 2019 19:10:44 +0530 Subject: [PATCH 03/78] include test --- tests/test_pipeline_images.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index eb3347442..efa96e146 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -80,9 +80,10 @@ class ImagesPipelineTestCase(unittest.TestCase): # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, buf = self.pipeline.convert_image(im, buf) + converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + self.assertEqual(converted_buf.read(), buf.read()) # check that thumbnail keep image ratio thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) From 398639a0bfe749eee07e51249f04b2c6c93eab73 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 8 Apr 2019 12:27:36 +0530 Subject: [PATCH 04/78] fix test --- scrapy/pipelines/images.py | 5 +++-- tests/test_pipeline_images.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index add606a2e..746244dab 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -132,7 +132,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - thumb_image, thumb_buf = self.convert_image(image, size) + thumb_image, thumb_buf = self.convert_image(image, buf, size) yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, response_body, size=None): @@ -153,7 +153,8 @@ class ImagesPipeline(FilesPipeline): image.thumbnail(size, Image.ANTIALIAS) if not size and image.format == 'JPEG': - return image, response_body + buf = BytesIO(response_body.read()) + return image, buf buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index efa96e146..dde0fa030 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -83,6 +83,47 @@ class ImagesPipelineTestCase(unittest.TestCase): converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that we don't convert JPEGs again + buf = io.BytesIO((b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00' + b'\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c' + b'\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c ' + b'$.\' ",#\x1c\x1c(7),01444\x1f\'9=82<.342\xff\xdb\x00C\x01\t\t\t\x0c\x0b\x0c' + b'\x18\r\r\x182!\x1c!222222222222222222222222222222222222222222222222' + b'22\xff\xc0\x00\x11\x08\x00\x14\x00\x14\x03\x01"\x00\x02\x11\x01\x03\x11' + b'\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' + b'\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00' + b'\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81' + b"\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456" + b'789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a' + b'\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8' + b'\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6' + b'\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3' + b'\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' + b'\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01' + b'\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' + b'\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00' + b'\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08' + b"\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&'" + b'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88' + b'\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6' + b'\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4' + b'\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2' + b'\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' + b'\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbb' + b'\xe2\x1b\xcb\x88\xe4\x90\t\x0e1\xd2\xb9\xab\x1dF\xe6\xda\xe0\xb4l\xd9' + b'5>\xaf\xac\xc7y}$k\x92\x03u\xaaL\xeb\x18VL\x9fZ+\xb9\xb9EBV]Ow\x0f\x8a\xc0' + b'\xac\x14\xbd\xa2NG_\xa5x\x8esg\xfb\xd9\x8e\xed\xc79\xa2\xa1\xd0\xf4\x8bk' + b'\x9d5e\xdeN\xe6=\xa8\xae\xa9V\xc2\xb6|\xdf2\xeep:w\xfc\x84d\x04g\x9e\xf5\xd3' + b'L\xa9\x0c\x1f*/>\xa2\x8a+\xca\x93z\x1c\x15]\xa9\xc6\xc4Q\xea\x97V\xca' + b'c\x85\xc2\xaes\x80(\xa2\x8a\xe3\x92W<\xdb\x9f\xff\xd9')) + im = Image.open(buf) + buf.seek(0) + converted, converted_buf = self.pipeline.convert_image(im, buf) + converted_buf.seek(0) + buf.seek(0) + self.assertEqual(im.format, "JPEG") self.assertEqual(converted_buf.read(), buf.read()) # check that thumbnail keep image ratio From c6769d6887b5c311c9083a5ac0349e81d1a2aea7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 13 Apr 2019 10:10:08 +0530 Subject: [PATCH 05/78] make suggested changes --- scrapy/pipelines/images.py | 6 ++--- tests/test_pipeline_images.py | 42 +---------------------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 746244dab..3450ee721 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -151,10 +151,8 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - - if not size and image.format == 'JPEG': - buf = BytesIO(response_body.read()) - return image, buf + elif image.format == 'JPEG': + return image, response_body buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dde0fa030..b419039b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -83,48 +83,8 @@ class ImagesPipelineTestCase(unittest.TestCase): converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - # check that we don't convert JPEGs again - buf = io.BytesIO((b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00' - b'\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c' - b'\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c ' - b'$.\' ",#\x1c\x1c(7),01444\x1f\'9=82<.342\xff\xdb\x00C\x01\t\t\t\x0c\x0b\x0c' - b'\x18\r\r\x182!\x1c!222222222222222222222222222222222222222222222222' - b'22\xff\xc0\x00\x11\x08\x00\x14\x00\x14\x03\x01"\x00\x02\x11\x01\x03\x11' - b'\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' - b'\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00' - b'\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81' - b"\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456" - b'789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a' - b'\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8' - b'\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6' - b'\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3' - b'\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' - b'\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01' - b'\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' - b'\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00' - b'\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08' - b"\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&'" - b'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88' - b'\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6' - b'\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4' - b'\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2' - b'\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' - b'\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbb' - b'\xe2\x1b\xcb\x88\xe4\x90\t\x0e1\xd2\xb9\xab\x1dF\xe6\xda\xe0\xb4l\xd9' - b'5>\xaf\xac\xc7y}$k\x92\x03u\xaaL\xeb\x18VL\x9fZ+\xb9\xb9EBV]Ow\x0f\x8a\xc0' - b'\xac\x14\xbd\xa2NG_\xa5x\x8esg\xfb\xd9\x8e\xed\xc79\xa2\xa1\xd0\xf4\x8bk' - b'\x9d5e\xdeN\xe6=\xa8\xae\xa9V\xc2\xb6|\xdf2\xeep:w\xfc\x84d\x04g\x9e\xf5\xd3' - b'L\xa9\x0c\x1f*/>\xa2\x8a+\xca\x93z\x1c\x15]\xa9\xc6\xc4Q\xea\x97V\xca' - b'c\x85\xc2\xaes\x80(\xa2\x8a\xe3\x92W<\xdb\x9f\xff\xd9')) - im = Image.open(buf) - buf.seek(0) - converted, converted_buf = self.pipeline.convert_image(im, buf) - converted_buf.seek(0) - buf.seek(0) - self.assertEqual(im.format, "JPEG") - self.assertEqual(converted_buf.read(), buf.read()) + self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) From 6039b66f42fe2c6e7708febb70939952cedaedd0 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 13 Apr 2019 10:17:26 +0530 Subject: [PATCH 06/78] aesthetic changes --- tests/test_pipeline_images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index b419039b3..0a2153ddb 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -87,14 +87,14 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) + thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25)) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, buf = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, buf = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) From 2a6bcdb413da0a4202fe905279d46d581368769d Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 29 Apr 2019 21:21:02 +0530 Subject: [PATCH 07/78] makes fix backward compatible --- scrapy/pipelines/images.py | 45 ++++++++++++++++++++++++++++++----- tests/test_pipeline_images.py | 8 +++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 3450ee721..ca8ac7b83 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -126,16 +126,42 @@ class ImagesPipeline(FilesPipeline): if width < self.min_width or height < self.min_height: raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) + + def _is_convert_image_overriden(): + import inspect + if six.PY2: + convert_image_signature = inspect.getargspec(self.convert_image) + elif six.PY3: + convert_image_signature = inspect.getfullargspec(self.convert_image) + if 'response_body' not in convert_image_signature.args: + return True + return False - image, buf = self.convert_image(orig_image, BytesIO(response.body)) + def _warn(): + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' + 'overriden method does not accept response_body attribute.', + category=ScrapyDeprecationWarning, stacklevel=1) + + convert_image_overriden = _is_convert_image_overriden() + if convert_image_overriden: + _warn() + image, buf = self.convert_image(orig_image) + else: + image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - thumb_image, thumb_buf = self.convert_image(image, buf, size) + if convert_image_overriden: + _warn() + thumb_image, thumb_buf = self.convert_image(image, size) + else: + thumb_image, thumb_buf = self.convert_image(image, size, buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, response_body, size=None): + def convert_image(self, image, size=None, response_body=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -151,9 +177,16 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - elif image.format == 'JPEG': - return image, response_body - + else: + if not response_body: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + 'method called without response_body attribute.', + category=ScrapyDeprecationWarning, stacklevel=1) + elif image.format == 'JPEG': + return image, response_body + buf = BytesIO() image.save(buf, 'JPEG') return image, buf diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0a2153ddb..3fe71589d 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -80,21 +80,21 @@ class ImagesPipelineTestCase(unittest.TestCase): # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, converted_buf = self.pipeline.convert_image(im, buf) + converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25)) + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, _ = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) From 33925a77610293c44a169efd1c274239a75ba968 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 7 May 2019 15:44:21 +0530 Subject: [PATCH 08/78] test for deprecation warning --- scrapy/pipelines/images.py | 20 +++++------ tests/test_pipeline_images.py | 63 ++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index ca8ac7b83..9776817bc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline): from scrapy.exceptions import ScrapyDeprecationWarning import warnings warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' - 'overriden method does not accept response_body attribute.', + 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) convert_image_overriden = _is_convert_image_overriden() @@ -162,6 +162,13 @@ class ImagesPipeline(FilesPipeline): yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None, response_body=None): + if not response_body: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + 'method called without response_body argument.', + category=ScrapyDeprecationWarning, stacklevel=1) + if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -177,15 +184,8 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - else: - if not response_body: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' - 'method called without response_body attribute.', - category=ScrapyDeprecationWarning, stacklevel=1) - elif image.format == 'JPEG': - return image, response_body + elif response_body and image.format == 'JPEG': + return image, response_body buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3fe71589d..651691862 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -76,37 +76,48 @@ class ImagesPipelineTestCase(unittest.TestCase): 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') def test_convert_image(self): - SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # tests for old API + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + SIZE = (100, 100) + # straigh forward case: RGB and JPEG + COLOUR = (0, 127, 255) + im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, converted_buf = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # ensure that we recieved deprecation warnings + self.assertTrue(len(w) >= 4) + + # tests for new API + # check that we don't convert JPEGs again COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) - self.assertEqual(thumbnail.mode, 'RGB') - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') - converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): return self.image_key(url) From 881bade2c1b2d5842f7d0bdf2648455996f814b7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 7 May 2019 16:12:26 +0530 Subject: [PATCH 09/78] tests for new API --- tests/test_pipeline_images.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 651691862..1dfca5c11 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -79,12 +79,11 @@ class ImagesPipelineTestCase(unittest.TestCase): # tests for old API with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, converted_buf = self.pipeline.convert_image(im) + im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) @@ -95,14 +94,14 @@ class ImagesPipelineTestCase(unittest.TestCase): # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') @@ -112,12 +111,36 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertTrue(len(w) >= 4) # tests for new API - # check that we don't convert JPEGs again + SIZE = (100, 100) + # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): return self.image_key(url) From 653ac3eebe96bdfff8128cdb1e500728215486a2 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 16 May 2019 06:20:39 +0000 Subject: [PATCH 10/78] makes suggested changes --- scrapy/pipelines/images.py | 1 - tests/test_pipeline_images.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9776817bc..b95383b3b 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -155,7 +155,6 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) if convert_image_overriden: - _warn() thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1dfca5c11..ba79dd6bd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -108,7 +108,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - self.assertTrue(len(w) >= 4) + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a incompatible way' in str(warning.message)]) == 4) # tests for new API SIZE = (100, 100) From 2994b624e03ec9a2cba2c024eacf549329c056de Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 17 May 2019 12:14:43 +0000 Subject: [PATCH 11/78] makes suggested changes --- scrapy/pipelines/images.py | 21 +++++---------------- tests/test_pipeline_images.py | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index b95383b3b..fd34c9df6 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,7 +15,7 @@ except ImportError: from PIL import Image from scrapy.utils.misc import md5sum -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, get_func_args from scrapy.http import Request from scrapy.settings import Settings from scrapy.exceptions import DropItem @@ -127,25 +127,14 @@ class ImagesPipeline(FilesPipeline): raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - def _is_convert_image_overriden(): - import inspect - if six.PY2: - convert_image_signature = inspect.getargspec(self.convert_image) - elif six.PY3: - convert_image_signature = inspect.getfullargspec(self.convert_image) - if 'response_body' not in convert_image_signature.args: - return True - return False - def _warn(): from scrapy.exceptions import ScrapyDeprecationWarning import warnings - warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' + warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) - convert_image_overriden = _is_convert_image_overriden() - if convert_image_overriden: + if 'response_body' not in get_func_args(self.convert_image): _warn() image, buf = self.convert_image(orig_image) else: @@ -154,7 +143,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if convert_image_overriden: + if 'response_body' not in get_func_args(self.convert_image): thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) @@ -164,7 +153,7 @@ class ImagesPipeline(FilesPipeline): if not response_body: from scrapy.exceptions import ScrapyDeprecationWarning import warnings - warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ba79dd6bd..ec0c87264 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -108,7 +108,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a incompatible way' in str(warning.message)]) == 4) + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) # tests for new API SIZE = (100, 100) From c8e28ec194b3730e7954a18526ac77cb44138feb Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 23 May 2019 15:04:21 +0530 Subject: [PATCH 12/78] makes suggested changes --- scrapy/pipelines/images.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fd34c9df6..a3f5f9292 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -134,7 +134,8 @@ class ImagesPipeline(FilesPipeline): 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) - if 'response_body' not in get_func_args(self.convert_image): + convert_image_overriden = 'response_body' not in get_func_args(self.convert_image) + if convert_image_overriden: _warn() image, buf = self.convert_image(orig_image) else: @@ -143,7 +144,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if 'response_body' not in get_func_args(self.convert_image): + if convert_image_overriden: thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) From 90fdefcbca89d0ef2cc81955b9fd8b8af1dff392 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 25 May 2019 19:11:48 +0530 Subject: [PATCH 13/78] cache if convert_image has deprecated signature --- scrapy/pipelines/images.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a3f5f9292..f709c5057 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -84,6 +84,8 @@ class ImagesPipeline(FilesPipeline): resolve('IMAGES_THUMBS'), self.THUMBS ) + self._deprecated_convert_image = None + @classmethod def from_settings(cls, settings): s3store = cls.STORE_SCHEMES['s3'] @@ -126,17 +128,17 @@ class ImagesPipeline(FilesPipeline): if width < self.min_width or height < self.min_height: raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - - def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) - convert_image_overriden = 'response_body' not in get_func_args(self.convert_image) - if convert_image_overriden: - _warn() + if self._deprecated_convert_image is None: + self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) + if self._deprecated_convert_image: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + 'overriden method does not accept response_body argument.', + category=ScrapyDeprecationWarning, stacklevel=1) + + if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) else: image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) @@ -144,7 +146,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if convert_image_overriden: + if self._deprecated_convert_image: thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) From 8b84a65a6b2d391fdd9c49426d748751a03351f0 Mon Sep 17 00:00:00 2001 From: drs-11 Date: Tue, 25 Aug 2020 00:30:17 +0530 Subject: [PATCH 14/78] cleaned up code relating to issue #3689 --- scrapy/pipelines/images.py | 14 ++----- tests/test_pipeline_images.py | 72 ++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index f709c5057..e265685fb 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -6,6 +6,7 @@ See documentation in topics/media-pipeline.rst import functools import hashlib import six +import warnings try: from cStringIO import StringIO as BytesIO @@ -19,6 +20,7 @@ from scrapy.utils.python import to_bytes, get_func_args from scrapy.http import Request from scrapy.settings import Settings from scrapy.exceptions import DropItem +from scrapy.exceptions import ScrapyDeprecationWarning #TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException, FilesPipeline @@ -132,8 +134,6 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) @@ -153,9 +153,7 @@ class ImagesPipeline(FilesPipeline): yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None, response_body=None): - if not response_body: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings + if response_body is None: warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) @@ -175,7 +173,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - elif response_body and image.format == 'JPEG': + elif response_body is not None and image.format == 'JPEG': return image, response_body buf = BytesIO() @@ -193,8 +191,6 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None): ## start of deprecation warning block (can be removed in the future) def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, ' 'please use file_path(request, response=None, info=None) instead', category=ScrapyDeprecationWarning, stacklevel=1) @@ -221,8 +217,6 @@ class ImagesPipeline(FilesPipeline): def thumb_path(self, request, thumb_id, response=None, info=None): ## start of deprecation warning block (can be removed in the future) def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use ' 'thumb_path(request, thumb_id, response=None, info=None) instead', category=ScrapyDeprecationWarning, stacklevel=1) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ec0c87264..915b6a579 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -75,41 +75,7 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') - def test_convert_image(self): - # tests for old API - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - SIZE = (100, 100) - # straigh forward case: RGB and JPEG - COLOUR = (0, 127, 255) - im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, 'RGB') - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # ensure that we recieved deprecation warnings - self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) - + def test_convert_image_new(self): # tests for new API SIZE = (100, 100) # straigh forward case: RGB and JPEG @@ -207,6 +173,42 @@ class DeprecatedImagesPipelineTestCase(unittest.TestCase): self.assertEqual(len(w), 1) self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message)) + def test_overriden_convert_image_method(self): + self.init_pipeline(ImagesPipeline) + # tests for old API + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + SIZE = (100, 100) + # straigh forward case: RGB and JPEG + COLOUR = (0, 127, 255) + im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # ensure that we recieved deprecation warnings + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) + def tearDown(self): rmtree(self.tempdir) From ecec5f9e5171568c4ead64336245a1bb3be2ecfe Mon Sep 17 00:00:00 2001 From: drs-11 Date: Tue, 25 Aug 2020 02:46:44 +0530 Subject: [PATCH 15/78] Cleaned up code --- scrapy/pipelines/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index d7f437adc..47d688c62 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -12,7 +12,7 @@ from io import BytesIO from itemadapter import ItemAdapter from PIL import Image -from scrapy.exceptions import DropItem,ScrapyDeprecationWarning +from scrapy.exceptions import DropItem, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -175,7 +175,7 @@ class ImagesPipeline(FilesPipeline): image.thumbnail(size, Image.ANTIALIAS) elif response_body is not None and image.format == 'JPEG': return image, response_body - + buf = BytesIO() image.save(buf, 'JPEG') return image, buf From 6565adc471360c2f542392befb7bb9448e8f171f Mon Sep 17 00:00:00 2001 From: drs-11 Date: Wed, 2 Sep 2020 20:44:26 +0530 Subject: [PATCH 16/78] added test case for get_images --- scrapy/pipelines/images.py | 6 +-- tests/test_pipeline_images.py | 75 ++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 47d688c62..09194a0fe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -18,7 +18,7 @@ from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.misc import md5sum -from scrapy.utils.python import to_bytes, get_func_args +from scrapy.utils.python import get_func_args, to_bytes class NoimagesDrop(DropItem): @@ -136,7 +136,7 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image: warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) + category=ScrapyDeprecationWarning) if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) @@ -156,7 +156,7 @@ class ImagesPipeline(FilesPipeline): if response_body is None: warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) + category=ScrapyDeprecationWarning, stacklevel=2) if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6b71b64bc..8e98b8734 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,6 +5,7 @@ import warnings from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +from unittest.mock import patch import attr from itemadapter import ItemAdapter @@ -12,7 +13,7 @@ from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImagesPipeline +from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.settings import Settings from scrapy.utils.python import to_bytes @@ -93,6 +94,78 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_get_images_exception(self): + self.pipeline.min_width = 100 + self.pipeline.min_height = 100 + + _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0)) + _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0)) + + resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue()) + resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue()) + resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp1, request=req, info=object())) + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp2, request=req, info=object())) + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp3, request=req, info=object())) + + def test_get_images_new(self): + self.pipeline.min_width = 0 + self.pipeline.min_height = 0 + self.pipeline.thumbs = {'small': (20, 20)} + + orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0)) + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + + path, new_im, new_buf = next(get_images_gen) + self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im, new_im) + self.assertEqual(buf.getvalue(), new_buf.getvalue()) + + thumb_path, thumb_img, thumb_buf = next(get_images_gen) + self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(thumb_img, thumb_img) + self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) + + def test_get_images_old(self): + self.pipeline.thumbs = {'small': (20, 20)} + orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + def overridden_convert_image(image, size=None): + im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + return im, buf + + with patch.object(self.pipeline, 'convert_image', overridden_convert_image): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + path, new_im, new_buf = next(get_images_gen) + self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im.mode, new_im.mode) + self.assertEqual(orig_im.getcolors(), new_im.getcolors()) + self.assertEqual(buf.getvalue(), new_buf.getvalue()) + + thumb_path, thumb_img, thumb_buf = next(get_images_gen) + self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im.mode, thumb_img.mode) + self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) + self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) + + expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + 'overriden method does not accept response_body argument.') + self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) + def test_convert_image_old(self): # tests for old API with warnings.catch_warnings(record=True) as w: From 0c24cdb2573958cc0fb9127c6f195d3174640ff4 Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Sun, 4 Oct 2020 02:09:21 +0530 Subject: [PATCH 17/78] Improved warning messages a little --- scrapy/pipelines/images.py | 4 ++-- tests/test_pipeline_images.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index d3254bc20..48e8a7b83 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -135,7 +135,7 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning) @@ -155,7 +155,7 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None, response_body=None): if response_body is None: - warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' + warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=2) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 380c775c4..0a294db80 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -162,7 +162,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + expected_warning_msg = ('.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.') self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) @@ -199,7 +199,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - expected_warning_msg = 'ImagesPipeline.convert_image() method called in a deprecated way' + expected_warning_msg = '.convert_image() method called in a deprecated way' self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4) def test_convert_image_new(self): From 5bacc4822e80a78a2a287a5d1e417c0d3b7ddbdf Mon Sep 17 00:00:00 2001 From: Gustavo Bordin Date: Sun, 16 May 2021 18:40:30 -0300 Subject: [PATCH 18/78] changing dunder-str to dunder-repr --- scrapy/core/http2/stream.py | 6 ++---- scrapy/http/request/__init__.py | 4 +--- scrapy/http/response/__init__.py | 4 +--- scrapy/settings/__init__.py | 4 +--- scrapy/spiders/__init__.py | 4 +--- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2a4b702f..a3a9e5e1d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -150,12 +150,10 @@ class Stream: self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) - - def __str__(self) -> str: + + def __repr__(self): return f'Stream(id={self.stream_id!r})' - __repr__ = __str__ - @property def _log_warnsize(self) -> bool: """Checks if we have received data which exceeds the download warnsize diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ad884feac..a1857c604 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -109,11 +109,9 @@ class Request(object_ref): def encoding(self): return self._encoding - def __str__(self): + def __repr__(self): return f"<{self.method} {self.url}>" - __repr__ = __str__ - def copy(self): """Return a copy of this Request""" return self.replace() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 185a9bb67..eb80c5214 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -87,11 +87,9 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) - def __str__(self): + def __repr__(self): return f"<{self.status} {self.url}>" - __repr__ = __str__ - def copy(self): """Return a copy of this Response""" return self.replace() diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..69d0476eb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -51,11 +51,9 @@ class SettingsAttribute: self.value = value self.priority = priority - def __str__(self): + def __repr__(self): return f"" - __repr__ = __str__ - class BaseSettings(MutableMapping): """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..1c079ff27 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -106,11 +106,9 @@ class Spider(object_ref): if callable(closed): return closed(reason) - def __str__(self): + def __repr__(self): return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" - __repr__ = __str__ - # Top-level imports from scrapy.spiders.crawl import CrawlSpider, Rule From 09a07c9b4ad02959e7416c25984fa715df7bd71f Mon Sep 17 00:00:00 2001 From: Gustavo Bordin Date: Tue, 18 May 2021 19:07:28 -0300 Subject: [PATCH 19/78] removed whitespaces --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a3a9e5e1d..780191505 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -150,7 +150,7 @@ class Stream: self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) - + def __repr__(self): return f'Stream(id={self.stream_id!r})' From 7de9ed5bd42052f6491c939513adde6f6243ce2f Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Sun, 1 May 2022 01:34:35 +0300 Subject: [PATCH 20/78] Deprecate scrapy.pipelines.images.NoimagesDrop --- scrapy/pipelines/images.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9c99dc69e..df4575a41 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,10 +15,12 @@ from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings +from scrapy.utils.decorators import deprecated from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes +@deprecated() class NoimagesDrop(DropItem): """Product with no images exception""" From 9f659bd63c04656f1c0601d6338f9050d5049fe8 Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Sun, 1 May 2022 13:36:15 +0300 Subject: [PATCH 21/78] Fix deprecation of scrapy.pipelines.images.NoimagesDrop --- scrapy/pipelines/images.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index df4575a41..c7a04a0cf 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -5,25 +5,28 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +import warnings from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from scrapy.exceptions import DropItem, NotConfigured +from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.decorators import deprecated from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes -@deprecated() class NoimagesDrop(DropItem): """Product with no images exception""" + def __init__(self, *args, **kwargs): + warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) + super().__init__(*args, **kwargs) + class ImageException(FileException): """General image error exception""" From cc16af35af8b20e04b42ca84431820b2b092f379 Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Tue, 3 May 2022 11:29:21 +0300 Subject: [PATCH 22/78] Add deprecation warning test for scrapy.pipelines.images.NoimagesDrop --- tests/test_pipeline_images.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c69cd0e4a..613190f9c 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,14 +4,16 @@ import random from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +from warnings import catch_warnings import attr from itemadapter import ItemAdapter from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImagesPipeline +from scrapy.pipelines.images import ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes @@ -413,6 +415,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): expected_value) +class NoimagesDropTestCase(unittest.TestCase): + + def test_deprecation_warning(self): + arg = str() + with catch_warnings(record=True) as warnings: + NoimagesDrop(arg) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedNoimagesDrop(NoimagesDrop): + pass + SubclassedNoimagesDrop(arg) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + + def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) From 13c5ad7e688271ae47f2a489bf665cb15387b3a3 Mon Sep 17 00:00:00 2001 From: "Alexandr N. Zamaraev" Date: Tue, 16 Aug 2022 11:03:37 +0700 Subject: [PATCH 23/78] Partial in is_generator_with_return_value See path in #5592 --- scrapy/utils/misc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..2e25f6421 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,6 +9,7 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules +from functools import partial from w3lib.html import replace_entities @@ -226,7 +227,11 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + func = callable + while isinstance(func, partial): + func = func.func + + code = re.sub(r"^[\t ]+", "", inspect.getsource(func)) tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From 77c055ee28a767b2c8222274cc7556ab9cc56edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Sep 2022 14:47:14 +0200 Subject: [PATCH 24/78] Relax Proxy-Authorization restrictions --- scrapy/downloadermiddlewares/httpproxy.py | 5 ++++- tests/test_downloadermiddleware_httpproxy.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 1deda42bd..dd8a7e797 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -78,4 +78,7 @@ class HttpProxyMiddleware: del request.headers[b'Proxy-Authorization'] del request.meta['_auth_proxy'] elif b'Proxy-Authorization' in request.headers: - del request.headers[b'Proxy-Authorization'] + if proxy_url: + request.meta['_auth_proxy'] = proxy_url + else: + del request.headers[b'Proxy-Authorization'] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 70eb94d77..44434f90e 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase): self.assertNotIn(b'Proxy-Authorization', request.headers) def test_proxy_authentication_header_proxy_without_credentials(self): + """As long as the proxy URL in request metadata remains the same, the + Proxy-Authorization header is used and kept, and may even be + changed.""" middleware = HttpProxyMiddleware() request = Request( 'https://example.com', @@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase): ) assert middleware.process_request(request, spider) is None self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + + request.headers['Proxy-Authorization'] = b'Basic bar' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar') def test_proxy_authentication_header_proxy_with_same_credentials(self): middleware = HttpProxyMiddleware() From e60e8224a23ddd4fd98c54318d128f6744f40a6a Mon Sep 17 00:00:00 2001 From: Abinash Satapathy Date: Thu, 6 Oct 2022 19:58:48 +0200 Subject: [PATCH 25/78] Update and rename INSTALL to INSTALL.md --- INSTALL | 4 ---- INSTALL.md | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 06e812936..000000000 --- a/INSTALL +++ /dev/null @@ -1,4 +0,0 @@ -For information about installing Scrapy see: - -* docs/intro/install.rst (local file) -* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..495413f97 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,4 @@ +For information about installing Scrapy see: + +* [Local docs](docs/intro/install.rst) +* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html) From d12fcc555b57dbd62c90212bd1d918d39e1ba45d Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 11 Oct 2022 10:32:45 -0700 Subject: [PATCH 26/78] Link to the Code of Conduct (#5659) --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 358302c76..d416ced3c 100644 --- a/README.rst +++ b/README.rst @@ -95,8 +95,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details. Code of Conduct --------------- -Please note that this project is released with a Contributor Code of Conduct -(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). +Please note that this project is released with a Contributor `Code of Conduct `_. By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@zyte.com. From 96fb663ae1a95286b0634bc51bc17b42eb29bd0f Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Tue, 11 Oct 2022 10:34:18 -0700 Subject: [PATCH 27/78] README: set Bash highlighting for pip install (#5648) --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d416ced3c..970bf2c35 100644 --- a/README.rst +++ b/README.rst @@ -64,7 +64,9 @@ Requirements Install ======= -The quick way:: +The quick way: + +.. code:: bash pip install scrapy From da9a2f8a946d6341be1cdd8cad0616a18bba6dbe Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 12 Oct 2022 11:10:39 -0300 Subject: [PATCH 28/78] Remove mention of minimum PyPy versions from the documentation (#5678) --- docs/intro/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 80a9c16d6..f28f5216a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -10,7 +10,7 @@ Supported Python versions ========================= Scrapy requires Python 3.7+, either the CPython implementation (default) or -the PyPy 7.3.5+ implementation (see :ref:`python:implementations`). +the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is 5.9.0. +We recommend using the latest PyPy version. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. From 715c05d504d22e87935ae42cee55ee35b12c2ebd Mon Sep 17 00:00:00 2001 From: gabrielztk <38334108+gabrielztk@users.noreply.github.com> Date: Thu, 13 Oct 2022 07:22:10 -0300 Subject: [PATCH 29/78] =?UTF-8?q?transport.producer.loseConnection()=20?= =?UTF-8?q?=E2=86=92=20transport.loseConnection()=20(#4995)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/handlers/http11.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 38935667d..6b8a18f1a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,8 +384,7 @@ class ScrapyAgent: logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": request, "handler": handler.__qualname__}) txresponse._transport.stopProducing() - with suppress(AttributeError): - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() return { "txresponse": txresponse, "body": b"", @@ -417,7 +416,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: @@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) self.transport.stopProducing() - self.transport._producer.loseConnection() + self.transport.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) From 62cc26e209b66ce5941f15736f669c75dcac9a59 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Oct 2022 22:03:54 +0600 Subject: [PATCH 30/78] Change TWISTED_REACTOR in the default template. --- docs/topics/settings.rst | 5 +++++ scrapy/templates/project/module/settings.py.tmpl | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a711fd197..0b1ef71cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,6 +1642,11 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. +.. versionchanged:: VERSION + The :command:`startproject` command now sets this setting to + ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated + ``settings.py`` file. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 5e541e2c0..c0c34e986 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -89,3 +89,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' From 22a59d0005c03d866ce98d352024974a9e48e7d1 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Fri, 14 Oct 2022 23:41:50 +0600 Subject: [PATCH 31/78] CI: use the latest version of Ubuntu (#5675) --- .github/workflows/checks.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cc8f20f44..439dfee51 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: checks: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e307189d..f6b098b80 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,7 @@ on: [push] jobs: publish: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags/') steps: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7915a9aab..d2bfe4a5f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: From 043575123c57db9f055c3668a3d80485989df0b2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 15 Oct 2022 11:41:05 +0330 Subject: [PATCH 32/78] Add async callback support to the parse command (#5577) --- scrapy/commands/parse.py | 55 ++++++++++++++++++++++--------------- tests/test_command_parse.py | 15 ++++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9b4fb0ed6..d93ab2ac5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,6 +5,8 @@ from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url +from twisted.internet.defer import maybeDeferred + from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display @@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, callback, cb_kwargs=None): - cb_kwargs = cb_kwargs or {} + def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): items, requests = [], [] - - for x in iterate_spider_output(callback(response, **cb_kwargs)): + for x in spider_output: if is_item(x): items.append(x) elif isinstance(x, Request): requests.append(x) - return items, requests + return items, requests, opts, depth, spider, callback + + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} + d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs)) + return d def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): @@ -158,6 +163,25 @@ class Command(BaseRunSpiderCommand): logger.error('No response downloaded for: %(url)s', {'url': url}) + def scraped_data(self, args): + items, requests, opts, depth, spider, callback = args + if opts.pipelines: + itemproc = self.pcrawler.engine.scraper.itemproc + for item in items: + itemproc.process_item(item, spider) + self.add_items(depth, items) + self.add_requests(depth, requests) + + scraped_data = items if opts.output else [] + if depth < opts.depth: + for req in requests: + req.meta['_depth'] = depth + 1 + req.meta['_callback'] = req.callback + req.callback = callback + scraped_data += requests + + return scraped_data + def prepare_request(self, spider, request, opts): def callback(response, **cb_kwargs): # memorize first request @@ -191,23 +215,10 @@ class Command(BaseRunSpiderCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb, cb_kwargs) - if opts.pipelines: - itemproc = self.pcrawler.engine.scraper.itemproc - for item in items: - itemproc.process_item(item, spider) - self.add_items(depth, items) - self.add_requests(depth, requests) - - scraped_data = items if opts.output else [] - if depth < opts.depth: - for req in requests: - req.meta['_depth'] = depth + 1 - req.meta['_callback'] = req.callback - req.callback = callback - scraped_data += requests - - return scraped_data + d = self.run_callback(response, cb, cb_kwargs) + d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) + d.addCallback(self.scraped_data) + return d # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0d992be56..8368356e2 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule +from scrapy.utils.test import get_from_asyncio_queue +class AsyncDefAsyncioSpider(scrapy.Spider): + + name = 'asyncdef{self.spider_name}' + + async def parse(self, response): + status = await get_from_asyncio_queue(response.status) + return [scrapy.Item(), dict(foo='bar')] class MySpider(scrapy.Spider): name = '{self.spider_name}' @@ -160,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url('/html')]) self.assertIn("INFO: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_asyncio_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( From 75bb516edbc39f9a657e71e75f05f0fd6a33d60d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 10:26:38 +0200 Subject: [PATCH 33/78] Adapt tests to the new value of TWISTED_REACTOR for new projects --- tests/test_commands.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 76d5f3935..eaca41102 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -689,8 +689,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_asyncio_enabled_false(self): + def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_asyncio_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor' + ]) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') From 960a7f68f6939744b39d528e28f8f925a2e12ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 11:27:00 +0200 Subject: [PATCH 34/78] Verify that the installed asyncio event loop matches ASYNCIO_EVENT_LOOP (#5529) Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- scrapy/crawler.py | 14 ++++++++-- scrapy/utils/reactor.py | 18 ++++++++++++ .../asyncio_enabled_reactor_different_loop.py | 25 +++++++++++++++++ .../asyncio_enabled_reactor_same_loop.py | 28 +++++++++++++++++++ tests/test_crawler.py | 23 +++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e768bca12..65174d846 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,7 +31,12 @@ from scrapy.utils.log import ( ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -from scrapy.utils.reactor import install_reactor, verify_installed_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + verify_installed_asyncio_event_loop, + verify_installed_reactor, +) logger = logging.getLogger(__name__) @@ -78,17 +83,20 @@ class Crawler: crawler=self, ) - reactor_class = self.settings.get("TWISTED_REACTOR") + reactor_class = self.settings["TWISTED_REACTOR"] + event_loop = self.settings["ASYNCIO_EVENT_LOOP"] if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor if reactor_class: - install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + install_reactor(reactor_class, event_loop) else: from twisted.internet import reactor # noqa: F401 log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) self.extensions = ExtensionManager.from_crawler(self) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index bc543b230..652733ce8 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) +def verify_installed_asyncio_event_loop(loop_path): + from twisted.internet import reactor + loop_class = load_object(loop_path) + if isinstance(reactor._asyncioEventloop, loop_class): + return + installed = ( + f"{reactor._asyncioEventloop.__class__.__module__}" + f".{reactor._asyncioEventloop.__class__.__qualname__}" + ) + specified = f"{loop_class.__module__}.{loop_class.__qualname__}" + raise Exception( + "Scrapy found an asyncio Twisted reactor already " + f"installed, and its event loop class ({installed}) does " + "not match the one specified in the ASYNCIO_EVENT_LOOP " + f"setting ({specified})" + ) + + def is_asyncio_reactor_installed(): from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..ea8242f67 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,25 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..d24bf3031 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,28 @@ +import asyncio +import sys + +from uvloop import Loop + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cf15ba9b9..c61d461f7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + self.assertNotIn("Spider closed (finished)", log) + self.assertIn( + ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ), + log, + ) + def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From c49764ffd7111d8a465a0d077d0df38bd40449fa Mon Sep 17 00:00:00 2001 From: mattkohl-flex Date: Mon, 17 Oct 2022 11:15:17 +0100 Subject: [PATCH 35/78] typo fixes --- docs/intro/install.rst | 2 +- docs/topics/contracts.rst | 2 +- docs/topics/extensions.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/request-response.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index f28f5216a..9ab479edd 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -187,7 +187,7 @@ solutions: * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be - used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly if you're using `zsh`_ as default shell):: echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ef296dc9e..c29a3a410 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -102,7 +102,7 @@ override three methods: .. method:: Contract.post_process(output) This allows processing the output of the callback. Iterators are - converted listified before being passed to this hook. + converted to lists before being passed to this hook. Raise :class:`~scrapy.exceptions.ContractFail` from :class:`~scrapy.contracts.Contract.pre_process` or diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 297e1fdc5..130657b0b 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a -hypothetic extension to handle `Google Sitemaps`_ would use settings like +hypothetical extension to handle `Google Sitemaps`_ would use settings like ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 477652704..33441838a 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -154,7 +154,7 @@ Too many spiders? If your project has too many spiders executed in parallel, the output of :func:`prefs()` can be difficult to read. For this reason, that function has a ``ignore`` argument which can be used to -ignore a particular class (and all its subclases). For +ignore a particular class (and all its subclasses). For example, this won't show any live references to spiders: >>> from scrapy.spiders import Spider diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..7eb6942ac 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -446,7 +446,7 @@ class). Scenarios where changing the request fingerprinting algorithm may cause undesired results include, for example, using the HTTP cache middleware (see :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidade the current +Changing the request fingerprinting algorithm would invalidate the current cache, requiring you to redownload all requests again. Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in From 06c8f673afe9af08784e62d67930a8cbc9887ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Oct 2022 15:04:29 +0200 Subject: [PATCH 36/78] 2.7 release notes (#5680) * Fix the display name of documented fingerprinter class methods * Initial draft for the Scrapy 2.7 release notes * Update VERSION and PREVIOUS_VERSION references * Clarify the restrictions lifted for item field output names * Fix the description of the BOM bug fix * Fix the note about changes in MIME sniffing * Fix typo * Extend highlights * Fyx typo --- docs/_ext/scrapydocs.py | 2 +- docs/news.rst | 193 +++++++++++++++++- docs/topics/components.rst | 4 +- docs/topics/coroutines.rst | 12 +- docs/topics/request-response.rst | 45 ++-- docs/topics/settings.rst | 10 +- docs/topics/spider-middleware.rst | 8 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/utils/request.py | 12 +- scrapy/utils/test.py | 2 +- tests/test_crawl.py | 2 +- tests/test_crawler.py | 8 +- tests/test_dupefilters.py | 12 +- tests/test_pipeline_crawl.py | 2 +- tests/test_scheduler.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_utils_request.py | 4 +- 18 files changed, 259 insertions(+), 65 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index d02a2e17b..f0f382da3 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -15,7 +15,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): - if node.tagname == 'index': + if node.tagname == 'index' and node['entries']: # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] diff --git a/docs/news.rst b/docs/news.rst index 9469d0fe5..d8b9fcd1e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,195 @@ Release notes ============= +.. _release-2.7.0: + +Scrapy 2.7.0 (to be released) +----------------------------- + +Highlights: + +- Added Python 3.11 support, dropped Python 3.6 support +- Improved support for :ref:`asynchronous callbacks ` +- :ref:`Asyncio support ` is enabled by default on new + projects +- Output names of item fields can now be arbitrary strings +- Centralized :ref:`request fingerprinting ` + configuration is now possible + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +Python 3.7 or greater is now required; support for Python 3.6 has been dropped. +Support for the upcoming Python 3.11 has been added. + +The minimum required version of some dependencies has changed as well: + +- lxml_: 3.5.0 → 4.3.0 + +- Pillow_ (:ref:`images pipeline `): 4.0.0 → 7.1.0 + +- zope.interface_: 5.0.0 → 5.1.0 + +(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`, +:issue:`5670`, :issue:`5678`) + + +Deprecations +~~~~~~~~~~~~ + +- :meth:`ImagesPipeline.thumb_path + ` must now accept an + ``item`` parameter (:issue:`5504`, :issue:`5508`). + +- The ``scrapy.downloadermiddlewares.decompression`` module is now + deprecated (:issue:`5546`, :issue:`5547`). + + +New features +~~~~~~~~~~~~ + +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares ` can now be + defined as an :term:`asynchronous generator` (:issue:`4978`). + +- The output of :class:`~scrapy.Request` callbacks defined as + :ref:`coroutines ` is now processed asynchronously + (:issue:`4978`). + +- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous + callbacks ` (:issue:`5657`). + +- New projects created with the :command:`startproject` command have + :ref:`asyncio support ` enabled by default (:issue:`5590`, + :issue:`5679`). + +- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a + dictionary to customize the output name of item fields, lifting the + restriction that required output names to be valid Python identifiers, e.g. + preventing them to have whitespace (:issue:`1008`, :issue:`3266`, + :issue:`3696`). + +- You can now customize :ref:`request fingerprinting ` + through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of + having to change it on every Scrapy component that relies on request + fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`, + :issue:`4524`). + +- ``jsonl`` is now supported and encouraged as a file extension for `JSON + Lines`_ files (:issue:`4848`). + + .. _JSON Lines: https://jsonlines.org/ + +- :meth:`ImagesPipeline.thumb_path + ` now receives the + source :ref:`item ` (:issue:`5504`, :issue:`5508`). + + +Bug fixes +~~~~~~~~~ + +- When using Google Cloud Storage with a :ref:`media pipeline + `, :setting:`FILES_EXPIRES` now also works when + :setting:`FILES_STORE` does not point at the root of your Google Cloud + Storage bucket (:issue:`5317`, :issue:`5318`). + +- The :command:`parse` command now supports :ref:`asynchronous callbacks + ` (:issue:`5424`, :issue:`5577`). + +- When using the :command:`parse` command with a URL for which there is no + available spider, an exception is no longer raised (:issue:`3264`, + :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`). + +- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte + order mark`_ when determining the text encoding of the response body, + following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`). + + .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark + .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding + +- MIME sniffing takes the response body into account in FTP and HTTP/1.0 + requests, as well as in cached requests (:issue:`4873`). + +- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag + is missing (:issue:`4873`). + +- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value + that does not match the asyncio event loop actually installed + (:issue:`5529`). + +- Fixed :meth:`Headers.getlist ` + returning only the last header (:issue:`5515`, :issue:`5526`). + +- Fixed :class:`LinkExtractor + ` not ignoring the + ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`, + :issue:`4066`) + + +Documentation +~~~~~~~~~~~~~ + +- Clarified the return type of :meth:`Spider.parse ` + (:issue:`5602`, :issue:`5608`). + +- To enable + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + to do `brotli compression`_, installing brotli_ is now recommended instead + of installing brotlipy_, as the former provides a more recent version of + brotli. + + .. _brotli: https://github.com/google/brotli + .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt + +- :ref:`Signal documentation ` now mentions :ref:`coroutine + support ` and uses it in code examples (:issue:`4852`, + :issue:`5358`). + +- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_ + (:issue:`3582`, :issue:`5432`). + + .. _Common Crawl: https://commoncrawl.org/ + .. _Google cache: http://www.googleguide.com/cached_pages.html + +- The new :ref:`topics-components` topic covers enforcing requirements on + Scrapy components, like :ref:`downloader middlewares + `, :ref:`extensions `, + :ref:`item pipelines `, :ref:`spider middlewares + `, and more; :ref:`enforce-asyncio-requirement` + has also been added (:issue:`4978`). + +- :ref:`topics-settings` now indicates that setting values must be + :ref:`picklable ` (:issue:`5607`, :issue:`5629`). + +- Removed outdated documentation (:issue:`5446`, :issue:`5373`, + :issue:`5369`, :issue:`5370`, :issue:`5554`). + +- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`, + :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`). + +- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`, + :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`). + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added a continuous integration job to run `twine check`_ (:issue:`5655`, + :issue:`5656`). + + .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check + +- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`, + :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`, + :issue:`5671`, :issue:`5675`). + +- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`, + :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`). + +- Applied minor code improvements (:issue:`5661`). + + .. _release-2.6.3: Scrapy 2.6.3 (2022-09-27) @@ -3139,7 +3328,7 @@ New Features ~~~~~~~~~~~~ - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) -- Support `brotli`_-compressed content; requires optional `brotlipy`_ +- Support `brotli-compressed`_ content; requires optional `brotlipy`_ (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) @@ -3176,7 +3365,7 @@ New Features - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command (:issue:`2740`) -.. _brotli: https://github.com/google/brotli +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://github.com/python-hyper/brotlipy/ Bug fixes diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c44f3def2..ca301b827 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -75,9 +75,9 @@ If your requirement is a minimum Scrapy version, you may use class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('VERSION'): + if parse_version(scrapy.__version__) < parse_version('2.7'): raise RuntimeError( - f"{MyComponent.__qualname__} requires Scrapy VERSION or " + f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " f"method of spider middlewares as an asynchronous " f"generator." diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 750263385..a1ba4ba5c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -22,7 +22,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): If you are using any custom or third-party :ref:`spider middleware `, see :ref:`sync-async-spider-middleware`. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 Output of async callbacks is now processed asynchronously instead of collecting all of it first. @@ -49,7 +49,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. - .. versionadded:: VERSION + .. versionadded:: 2.7 General usage ============= @@ -129,7 +129,7 @@ Common use cases for asynchronous code include: Mixing synchronous and asynchronous spider middlewares ====================================================== -.. versionadded:: VERSION +.. versionadded:: 2.7 The output of a :class:`~scrapy.Request` callback is passed as the ``result`` parameter to the @@ -182,10 +182,10 @@ process_spider_output_async method `. Universal spider middlewares ============================ -.. versionadded:: VERSION +.. versionadded:: 2.7 To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy VERSION and later (avoiding +its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding :ref:`asynchronous-to-synchronous conversions `) while maintaining support for older Scrapy versions, you may define ``process_spider_output`` as a synchronous method and define an @@ -206,7 +206,7 @@ For example:: yield r .. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy VERSION and later without requiring + works in Scrapy 2.7 and later without requiring asynchronous-to-synchronous conversions, and works in earlier Scrapy versions as well. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..4393e1c68 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the REQUEST_FINGERPRINTER_CLASS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 Default: :class:`scrapy.utils.request.RequestFingerprinter` @@ -409,38 +409,38 @@ import path. REQUEST_FINGERPRINTER_IMPLEMENTATION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 -Default: ``'PREVIOUS_VERSION'`` +Default: ``'2.6'`` Determines which request fingerprinting algorithm is used by the default request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). Possible values are: -- ``'PREVIOUS_VERSION'`` (default) +- ``'2.6'`` (default) This implementation uses the same request fingerprinting algorithm as - Scrapy PREVIOUS_VERSION and earlier versions. + Scrapy 2.6 and earlier versions. Even though this is the default value for backward compatibility reasons, it is a deprecated value. -- ``'VERSION'`` +- ``'2.7'`` - This implementation was introduced in Scrapy VERSION to fix an issue of the + This implementation was introduced in Scrapy 2.7 to fix an issue of the previous implementation. New projects should use this value. The :command:`startproject` command sets this value in the generated ``settings.py`` file. -If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are +If you are using the default value (``'2.6'``) for this setting, and you are using Scrapy components where changing the request fingerprinting algorithm would cause undesired results, you need to carefully decide when to change the value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request +setting to a custom request fingerprinter class that implements the 2.6 request fingerprinting algorithm and does not log this warning ( -:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a +:ref:`2.6-request-fingerprinter` includes an example implementation of such a class). Scenarios where changing the request fingerprinting algorithm may cause @@ -449,14 +449,14 @@ undesired results include, for example, using the HTTP cache middleware (see Changing the request fingerprinting algorithm would invalidade the current cache, requiring you to redownload all requests again. -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in +Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in your settings to switch already to the request fingerprinting implementation that will be the only request fingerprinting implementation available in a future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'PREVIOUS_VERSION'``). +the default value (``'2.6'``). -.. _PREVIOUS_VERSION-request-fingerprinter: +.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter @@ -464,6 +464,8 @@ Writing your own request fingerprinter A request fingerprinter is a class that must implement the following method: +.. currentmodule:: None + .. method:: fingerprint(self, request) Return a :class:`bytes` object that uniquely identifies *request*. @@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method: Additionally, it may also implement the following methods: .. classmethod:: from_crawler(cls, crawler) + :noindex: If present, this class method is called to create a request fingerprinter instance from a :class:`~scrapy.crawler.Crawler` object. It must return a @@ -495,11 +498,13 @@ Additionally, it may also implement the following methods: :class:`~scrapy.settings.Settings` object. It must return a new instance of the request fingerprinter. -The ``fingerprint`` method of the default request fingerprinter, +.. currentmodule:: scrapy.http + +The :meth:`fingerprint` method of the default request fingerprinter, :class:`scrapy.utils.request.RequestFingerprinter`, uses :func:`scrapy.utils.request.fingerprint` with its default parameters. For some -common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well -in your ``fingerprint`` method implementation: +common use cases you can use :func:`scrapy.utils.request.fingerprint` as well +in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint @@ -519,7 +524,7 @@ account:: You can also write your own fingerprinting logic from scratch. -However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure +However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints: - Caching saves CPU by ensuring that fingerprints are calculated only once @@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter that reads fingerprints from :attr:`request.meta ` when available, and then falls back to -:func:`~scrapy.utils.request.fingerprint`. For example:: +:func:`scrapy.utils.request.fingerprint`. For example:: from scrapy.utils.request import fingerprint @@ -564,8 +569,8 @@ when available, and then falls back to return request.meta['fingerprint'] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION -without using the deprecated ``'PREVIOUS_VERSION'`` value of the +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 +without using the deprecated ``'2.6'`` value of the :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following request fingerprinter:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b1ef71cf..40bcda288 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,7 +1642,7 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. -.. versionchanged:: VERSION +.. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. @@ -1661,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. -This setting can act as a stopping condition in case of URLs of ever-increasing -length, which may be caused for example by a programming error either in the -target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and :setting:`DEPTH_LIMIT`. Use ``0`` to allow URLs of any length. -The default value is copied from the `Microsoft Internet Explorer maximum URL +The default value is copied from the `Microsoft Internet Explorer maximum URL length`_, even though this setting exists for different reasons. .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 816cb5e03..303401a3c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -105,17 +105,17 @@ object gives you access, for example, to the :ref:`settings `. :class:`~scrapy.Request` objects and :ref:`item objects `. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 This method may be defined as an :term:`asynchronous generator`, in which case ``result`` is an :term:`asynchronous iterable`. Consider defining this method as an :term:`asynchronous generator`, which will be a requirement in a future version of Scrapy. However, if you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy VERSION ` + either :ref:`enforcing Scrapy 2.7 ` as a minimum requirement of your spider middleware, or :ref:`making your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy VERSION. + it works with Scrapy versions earlier than Scrapy 2.7. :param response: the response which generated this output from the spider @@ -130,7 +130,7 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output_async(response, result, spider) - .. versionadded:: VERSION + .. versionadded:: 2.7 If defined, this method must be an :term:`asynchronous generator`, which will be called instead of :meth:`process_spider_output` if diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ff86af125..29ff028be 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -248,7 +248,7 @@ REFERER_ENABLED = True REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index c0c34e986..bbf60982c 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,5 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cf33317ce..fbddc41fb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -236,10 +236,10 @@ class RequestFingerprinter: 'REQUEST_FINGERPRINTER_IMPLEMENTATION' ) else: - implementation = 'PREVIOUS_VERSION' - if implementation == 'PREVIOUS_VERSION': + implementation = '2.6' + if implementation == '2.6': message = ( - '\'PREVIOUS_VERSION\' is a deprecated value for the ' + '\'2.6\' is a deprecated value for the ' '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' '\n' 'It is also the default value. In other words, it is normal ' @@ -254,14 +254,14 @@ class RequestFingerprinter: ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = _request_fingerprint_as_bytes - elif implementation == 'VERSION': + elif implementation == '2.7': self._fingerprint = fingerprint else: raise ValueError( f'Got an invalid value on setting ' f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' - f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) ' - f'and \'VERSION\'.' + f'{implementation!r}. Valid values are \'2.6\' (deprecated) ' + f'and \'2.7\'.' ) def fingerprint(self, request): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 0b828f7c0..445cd2e3a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -65,7 +65,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): # Set by default settings that prevent deprecation warnings. settings = {} if prevent_warnings: - settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION' + settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7' settings.update(settings_dict or {}) runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8be4b6fe1..5383ec652 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -350,7 +350,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c61d461f7..da6024c2b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -110,7 +110,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file, # settings to avoid extra warnings - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } @@ -235,7 +235,7 @@ class NoRequestsSpider(scrapy.Spider): class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -283,14 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 8a37a8ebe..6ebb716b0 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -51,7 +51,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -60,7 +60,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -68,7 +68,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_direct_scheduler(self): settings = {'DUPEFILTER_CLASS': DirectDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -172,7 +172,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -199,7 +199,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -233,7 +233,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e46532a1c..0e174cd34 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,7 +64,7 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'ITEM_PIPELINES': {self.pipeline_class: 1}, self.store_setting_key: self.tmpmediastore, } diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ac66056ba..50a7755c1 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -52,7 +52,7 @@ class MockCrawler(Crawler): SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', - REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION', + REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7', ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3719c7c9f..7a590f96c 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -98,7 +98,7 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.spider1' runner = CrawlerRunner({ 'SPIDER_MODULES': [module], - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', }) self.assertRaisesRegex(KeyError, 'Spider not found', diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 8bc7922b6..a92d9a0ac 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -505,7 +505,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_deprecated_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) @@ -518,7 +518,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_recommended_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) From 20b79a0f2e47800bf4648c7f890c8170fc8f5ede Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Oct 2022 19:09:22 +0600 Subject: [PATCH 37/78] =?UTF-8?q?Bump=20version:=202.6.2=20=E2=86=92=202.7?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2e2f7949a..f88071685 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.2 +current_version = 2.7.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index d8b9fcd1e..1ec183a1d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.7.0: -Scrapy 2.7.0 (to be released) +Scrapy 2.7.0 (2022-10-17) ----------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 097a15a2a..24ba9a38d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.2 +2.7.0 From b33244e2f0d877b8911f949308222db0b076d665 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 21 Oct 2022 19:17:04 +0500 Subject: [PATCH 38/78] Fix the flake8 per-file ignore syntax (#5688) --- .flake8 | 17 +++++++++-------- scrapy/utils/url.py | 1 + tests/test_loader.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.flake8 b/.flake8 index d7aebc24b..0c64d009e 100644 --- a/.flake8 +++ b/.flake8 @@ -6,16 +6,17 @@ ignore = W503 exclude = docs/conf.py +per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 + scrapy/__init__.py:E402 + scrapy/core/downloader/handlers/http.py:F401 + scrapy/http/__init__.py:F401 + scrapy/linkextractors/__init__.py:E402,F401 + scrapy/selector/__init__.py:F401 + scrapy/spiders/__init__.py:E402,F401 # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 + scrapy/utils/url.py:F403,F405 + tests/test_loader.py:E741 diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d5e9ae82..21201ace5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -34,6 +34,7 @@ def url_has_any_extension(url, extensions): lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) + def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already parsed url) diff --git a/tests/test_loader.py b/tests/test_loader.py index c0937b349..b3e44d36b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -295,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) - + def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") From b61b71c6f015d45f6e98a4280bf4993517180045 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Tue, 25 Oct 2022 08:44:43 +0530 Subject: [PATCH 39/78] Replace indentation of source before parsing with ast. closes #5323 --- scrapy/utils/misc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..c0258c8d9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,7 +226,14 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + pattern = r"(^[\t ]+)" + src = inspect.getsource(callable) + match = re.match(pattern, src) # Find indentation + code = re.sub(pattern, "", src) + if match: + # Remove indentation + code = re.sub(f"\n{match.group(0)}", "\n", code) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From f4e2a10ed6a44300738bd3701ea62b432c9c1a06 Mon Sep 17 00:00:00 2001 From: Kaushal Sharma Date: Tue, 25 Oct 2022 02:45:46 -0700 Subject: [PATCH 40/78] =?UTF-8?q?Image.ANTIALIAS=20=E2=86=92=20Image.Resam?= =?UTF-8?q?pling.LANCZOS=20(#5692)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/pipelines/images.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6b97190ee..67b3224b3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -160,7 +160,14 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, self._Image.ANTIALIAS) + try: + # Image.Resampling.LANCZOS was added in Pillow 9.1.0 + # remove this try except block, + # when updating the minimum requirements for Pillow. + resampling_filter = self._Image.Resampling.LANCZOS + except AttributeError: + resampling_filter = self._Image.ANTIALIAS + image.thumbnail(size, resampling_filter) buf = BytesIO() image.save(buf, 'JPEG') From 830e1c5dd85618a27749bbe41d35b11fb2bdd348 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 01:26:54 +0530 Subject: [PATCH 41/78] Add test for parsing decorated methods --- ...t_return_with_argument_inside_generator.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1c85ca353..72277d701 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -165,6 +165,89 @@ https://example.org warn_on_generator_with_return_value(None, l2) self.assertEqual(len(w), 0) + def test_generators_return_none_with_decorator(self): + def decorator(func): + def inner_func(): + func() + return inner_func + + @decorator + def f3(): + yield 1 + return None + + @decorator + def g3(): + yield 1 + return + + @decorator + def h3(): + yield 1 + + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() + + @decorator + def j3(): + yield 1 + + def helper(): + return 0 + + yield helper() + + @decorator + def k3(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l3) + self.assertEqual(len(w), 0) + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: From b0ddffc47b9cee5e6146497b42de3787da76d2ad Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 06:53:43 +0530 Subject: [PATCH 42/78] Misc. changes: - compile regex - readability improvements --- scrapy/utils/misc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index c0258c8d9..4d4fb9600 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,13 +226,13 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - pattern = r"(^[\t ]+)" src = inspect.getsource(callable) - match = re.match(pattern, src) # Find indentation - code = re.sub(pattern, "", src) + pattern = re.compile(r"(^[\t ]+)") + code = pattern.sub("", src) + + match = pattern.match(src) # finds indentation if match: - # Remove indentation - code = re.sub(f"\n{match.group(0)}", "\n", code) + code = re.sub(f"\n{match.group(0)}", "\n", code) # remove indentation tree = ast.parse(code) for node in walk_callable(tree): From 2464939b7ee8381f904f7b0625aee6de0989b6c8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 26 Oct 2022 15:58:20 -0300 Subject: [PATCH 43/78] Fixed deprecation warning in scrapy.core.engine (#5589) * Change `download` function logic * Fix CI error in 3.7 checks * Make `spider` parameter optional in `_download` function, assign spider value from self if `None` --- scrapy/core/engine.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6602f661d..1228e78da 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -257,9 +257,7 @@ class ExecutionEngine: def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is None: - spider = self.spider - else: + if spider is not None: warnings.warn( "Passing a 'spider' argument to ExecutionEngine.download is deprecated", category=ScrapyDeprecationWarning, @@ -267,7 +265,7 @@ class ExecutionEngine: ) if spider is not self.spider: logger.warning("The spider '%s' does not match the open spider", spider.name) - if spider is None: + if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") return self._download(request, spider).addBoth(self._downloaded, request, spider) @@ -278,11 +276,14 @@ class ExecutionEngine: self.slot.remove_request(request) return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Spider) -> Deferred: + def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) + if spider is None: + spider = self.spider + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") From b394f2165acd662e55762d524b916f146291e422 Mon Sep 17 00:00:00 2001 From: Andrei Andrukhovich Date: Wed, 26 Oct 2022 23:11:28 +0300 Subject: [PATCH 44/78] =?UTF-8?q?Fix=20typo:=20[they]=20depends=20?= =?UTF-8?q?=E2=86=92=20depend=20(#5694)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 9ab479edd..2c2079f68 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -52,7 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -Some of these packages themselves depends on non-Python packages +Some of these packages themselves depend on non-Python packages that might require additional installation steps depending on your platform. Please check :ref:`platform-specific guides below `. From a214147359b8840abbb67bbe2a4a4066b271d2a5 Mon Sep 17 00:00:00 2001 From: Johanan Idicula Date: Wed, 26 Oct 2022 21:43:35 -0400 Subject: [PATCH 45/78] ci: Update macos runner The GitHub Actions macos-10.15 runner image is now deprecated, and GitHub Actions has begun to temporarily fail jobs referencing it during brownout periods. The image will be fully unsupported by 2022-12-01, which is just about a month away. This change updates the macOS runner image to the latest generally-available version, to help reduce spurious CI failures during the brownout periods, and to stay abreast of the sunsetting of the macos-10.15 image. See also: actions/runner-images#5583 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index fdb9f4980..61f1857f8 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: macos-10.15 + runs-on: macos-11 strategy: fail-fast: false matrix: From ca50af645390e38299082d9ef4682c4be482ae70 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 17:12:26 +0600 Subject: [PATCH 46/78] Remove an unused import. --- tests/test_pipeline_images.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c4ce2736f..c189d08bf 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,7 +4,6 @@ import io import random from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf from warnings import catch_warnings import attr From b6541830849a6edd41da6b539d272790a4661d7d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 27 Oct 2022 17:00:36 +0500 Subject: [PATCH 47/78] Use Python 3.11 as the default in CI (#5696) --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 9 +++------ .github/workflows/tests-windows.yml | 7 +++++++ .readthedocs.yml | 2 +- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 439dfee51..8c1ae4bd3 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,10 +8,10 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: security - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: flake8 # Pylint requires installing reppy, which does not support Python 3.9 @@ -22,10 +22,10 @@ jobs: - python-version: 3.7 env: TOXENV: typing - - python-version: "3.10" # Keep in sync with .readthedocs.yml + - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f6b098b80..991b0b6e8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 61f1857f8..174d245ca 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index d2bfe4a5f..9c3ce8115 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,13 +17,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - - python-version: "3.11.0-rc.2" + - python-version: "3.11" env: TOXENV: py - - python-version: "3.11.0-rc.2" + - python-version: "3.11" env: TOXENV: asyncio - python-version: pypy3.9 @@ -57,7 +54,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 14683fd53..f60c48841 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,13 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio +# no binary package for lxml for 3.11 yet +# - python-version: "3.11" +# env: +# TOXENV: py +# - python-version: "3.11" +# env: +# TOXENV: asyncio steps: - uses: actions/checkout@v3 diff --git a/.readthedocs.yml b/.readthedocs.yml index 390be3749..e71d34f3a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.10" # Keep in sync with .github/workflows/checks.yml + python: "3.11" # Keep in sync with .github/workflows/checks.yml python: install: From 3a34fa839938d7640e56f3ce12ba60aed31a6ecf Mon Sep 17 00:00:00 2001 From: Godson <30664729+Godson-Gnanaraj@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:32:12 +0530 Subject: [PATCH 48/78] Get the event loop from event_loop_policy to avoid a deprecation warning (#5689) --- scrapy/utils/defer.py | 8 +++++--- scrapy/utils/reactor.py | 21 ++++++++++++++++----- tests/test_utils_asyncio.py | 6 ++++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8fcf31cab..38aefd6d0 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,7 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy def defer_fail(_failure: Failure) -> Deferred: @@ -269,7 +269,8 @@ def deferred_from_coro(o) -> Any: return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return Deferred.fromFuture(asyncio.ensure_future(o)) + event_loop = get_asyncio_event_loop_policy().get_event_loop() + return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -320,7 +321,8 @@ def deferred_to_future(d: Deferred) -> Future: d = treq.get('https://example.com/additional') additional_response = await deferred_to_future(d) """ - return d.asFuture(asyncio.get_event_loop()) + policy = get_asyncio_event_loop_policy() + return d.asFuture(policy.get_event_loop()) def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 652733ce8..ddf354d88 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -51,6 +51,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) +def get_asyncio_event_loop_policy(): + policy = asyncio.get_event_loop_policy() + if ( + sys.version_info >= (3, 8) + and sys.platform == "win32" + and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) + ): + policy = asyncio.WindowsSelectorEventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + return policy + + def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import @@ -58,16 +71,14 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - if sys.version_info >= (3, 8) and sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + policy = get_asyncio_event_loop_policy() if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.get_event_loop() + event_loop = policy.get_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4d..741c6a505 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,3 +1,4 @@ +import warnings from unittest import TestCase from pytest import mark @@ -13,5 +14,6 @@ class AsyncioTest(TestCase): self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') def test_install_asyncio_reactor(self): - # this should do nothing - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + with warnings.catch_warnings(record=True) as w: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + self.assertEqual(len(w), 0) From b71d0292d5ff85d652e2943ee1a727a128b55594 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 18:13:47 +0600 Subject: [PATCH 49/78] Add a test for processing partial callbacks. --- .../test_return_with_argument_inside_generator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 72277d701..562f72fee 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,6 @@ import unittest import warnings +from functools import partial from unittest import mock from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value @@ -254,3 +255,10 @@ https://example.org warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 1) self.assertIn('Unable to determine', str(w[0].message)) + + def test_partial(self): + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb) From fd692f309105d917f5f46bd00a88c550d6cc7da3 Mon Sep 17 00:00:00 2001 From: Magnus Offermanns Date: Thu, 27 Oct 2022 14:43:31 +0200 Subject: [PATCH 50/78] Prevent running the -O and -t command-line options together (#5605) Co-authored-by: Andrey Rakhmatullin --- docs/topics/commands.rst | 20 ++++++++++++++++++++ scrapy/commands/__init__.py | 6 ++++-- scrapy/utils/conf.py | 19 ++++++++++++++++--- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8c0b8e55f..362190116 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -271,11 +271,31 @@ crawl Start crawling using a spider. +Supported options: + +* ``-h, --help``: show a help message and exit + +* ``-a NAME=VALUE``: set a spider argument (may be repeated) + +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) + +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) + +* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` + Usage examples:: $ scrapy crawl myspider [ ... myspider starts crawling ... ] + $ scrapy -o myfile:csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] + + $ scrapy -O myfile:json myspider + [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] + + $ scrapy -o myfile -t csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] .. command:: check diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index fb304b8c0..49c4e2f42 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_argument("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)," + " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)") parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") + help="dump scraped items into FILE, overwriting any existing file," + " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)") parser.add_argument("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items") diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 00cc53725..6404edda6 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -155,6 +155,15 @@ def feed_process_params_from_cli(settings, output, output_format=None, raise UsageError( "Please use only one of -o/--output and -O/--overwrite-output" ) + if output_format: + raise UsageError( + "-t/--output-format is a deprecated command line option" + " and does not work in combination with -O/--overwrite-output." + " To specify a format please specify it after a colon at the end of the" + " output URI (i.e. -O :)." + " Example working in the tutorial: " + "scrapy crawl quotes -O quotes.json:json" + ) output = overwrite_output overwrite = True @@ -162,9 +171,13 @@ def feed_process_params_from_cli(settings, output, output_format=None, if len(output) == 1: check_valid_format(output_format) message = ( - 'The -t command line option is deprecated in favor of ' - 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.' + "The -t/--output-format command line option is deprecated in favor of " + "specifying the output format within the output URI using the -o/--output or the" + " -O/--overwrite-output option (i.e. -o/-O :). See the documentation" + " of the -o or -O option or the following examples for more information. " + "Examples working in the tutorial: " + "scrapy crawl quotes -o quotes.csv:csv or " + "scrapy crawl quotes -O quotes.json:json" ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From bd9e482c2f0db92065708c8291be6e8bc1f05218 Mon Sep 17 00:00:00 2001 From: iamkaushal Date: Thu, 27 Oct 2022 23:21:55 +0530 Subject: [PATCH 51/78] added typing.io and typing.re in pytest warning filter to ignore --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index af0f2fb6e..f5fbf2529 100644 --- a/pytest.ini +++ b/pytest.ini @@ -24,3 +24,5 @@ markers = filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated + ignore:typing.re is deprecated + ignore:typing.io is deprecated From 9f45be439de8a3b9a6d201c33e98b408a73c02bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=B4=8E=E0=B4=A4=E0=B4=BF=E0=B4=B0=E0=B4=BE=E0=B4=B3?= =?UTF-8?q?=E0=B4=BF=E0=B4=95=E0=B5=8D=E0=B4=95=E0=B5=8A=E0=B4=B0=E0=B5=81?= =?UTF-8?q?=20=E0=B4=AA=E0=B5=8B=E0=B4=B0=E0=B4=BE=E0=B4=B3=E0=B4=BF?= <108031802+pankali@users.noreply.github.com> Date: Fri, 28 Oct 2022 02:13:37 +0200 Subject: [PATCH 52/78] Update Code of Conduct to Contributor Covenant v2.1 --- CODE_OF_CONDUCT.md | 152 +++++++++++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 902cd523e..3c8e4d1b5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,77 +1,133 @@ + # Contributor Covenant Code of Conduct ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to creating a positive environment -include: +Examples of behavior that contributes to a positive environment for our +community include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community -Examples of unacceptable behavior by participants include: +Examples of unacceptable behavior include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting -## Our Responsibilities +## Enforcement Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. ## Scope -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@zyte.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. +reported to the community leaders responsible for enforcement at +opensource@zyte.com. +All complaints will be reviewed and investigated promptly and fairly. -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version]. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations From 3259a4252566f4130f14ce2acd03531a735dae90 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 30 Oct 2022 13:17:37 -0300 Subject: [PATCH 53/78] CrawlSpider: pass cb_kwargs from process_request --- scrapy/spiders/crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index edac082d0..2d9328633 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -102,9 +102,9 @@ class CrawlSpider(Spider): request = self._build_request(rule_index, link) yield rule.process_request(request, response) - def _callback(self, response): + def _callback(self, response, **cb_kwargs): rule = self._rules[response.meta['rule']] - return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow) + return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow) def _errback(self, failure): rule = self._rules[failure.request.meta['rule']] From b18560315bda057610ecda165694f9c0f7445c1f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 30 Oct 2022 18:28:16 -0300 Subject: [PATCH 54/78] Add tests --- tests/spiders.py | 11 +++++++++++ tests/test_crawl.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/spiders.py b/tests/spiders.py index 5ea8a4a21..7952e3d47 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -419,6 +419,17 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): self.logger.info('[errback] status %i', failure.value.response.status) +class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): + name = 'crawl_spider_with_process_request_cb_kwargs' + rules = ( + Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"), + ) + + def process_request(self, request, response): + request.cb_kwargs["foo"] = "process_request" + return request + + class BytesReceivedCallbackSpider(MetaSpider): full_response_length = 2**18 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5383ec652..5ec96e4a7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -41,6 +41,7 @@ from tests.spiders import ( CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, + CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, @@ -426,6 +427,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[errback] status 500", str(log)) self.assertIn("[errback] status 501", str(log)) + @defer.inlineCallbacks + def test_crawlspider_process_request_cb_kwargs(self): + crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse] status 200 (foo: process_request)", str(log)) + self.assertIn("[parse] status 201 (foo: process_request)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) From 940a73863bf7dcb16b3f2d9f5efb83efe4599712 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Nov 2022 19:00:33 +0600 Subject: [PATCH 55/78] Release notes for 2.7.1. --- docs/news.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 1ec183a1d..b7c8c85b5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,57 @@ Release notes ============= +.. _release-2.7.1: + +Scrapy 2.7.1 (Not relased yet) +------------------------------ + +New features +~~~~~~~~~~~~ + +- Relaxed the restriction introduced in 2.6.2 so that the + ``Proxy-Authentication`` header can again be set explicitly, as long as the + proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and + for as long as that proxy URL remains the same (:issue:`5626`). + +Bug fixes +~~~~~~~~~ + +- Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options + together now produces an error instead of ignoring the former option + (:issue:`5516`, :issue:`5605`). + +- Replaced deprecated :mod:`asyncio` APIs that implicitly use the current + event loop with code that explicitly requests a loop from the event loop + policy (:issue:`5685`, :issue:`5689`). + +- Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`, + :issue:`5589`). + +- Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). + +- Improved code that checks if generators return values, so that it no longer + fails on decorated methods and partial methods (:issue:`5323`, + :issue:`5592`, :issue:`5599`, :issue:`5691`). + +Documentation +~~~~~~~~~~~~~ + +- Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`). + +- Fixed typos (:issue:`5681`, :issue:`5694`). + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Re-enabled some erroneously disabled flake8 checks (:issue:`5688`). + +- Ignored harmless deprecation warnings from :mod:`typing` in tests + (:issue:`5686`, :issue:`5697`). + +- Modernized CI configuration (:issue:`5695`, :issue:`5696`). + + .. _release-2.7.0: Scrapy 2.7.0 (2022-10-17) From 5ec175b8bb08f93c431d7d64d2389b90ec7a1f37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Nov 2022 13:54:00 +0600 Subject: [PATCH 56/78] Small relnotes fixes. --- docs/news.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b7c8c85b5..257626526 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,7 +14,8 @@ New features - Relaxed the restriction introduced in 2.6.2 so that the ``Proxy-Authentication`` header can again be set explicitly, as long as the proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and - for as long as that proxy URL remains the same (:issue:`5626`). + for as long as that proxy URL remains the same; this restores compatibility + with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). Bug fixes ~~~~~~~~~ @@ -27,7 +28,7 @@ Bug fixes event loop with code that explicitly requests a loop from the event loop policy (:issue:`5685`, :issue:`5689`). -- Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`, +- Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`, :issue:`5589`). - Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). @@ -51,7 +52,7 @@ Quality assurance - Ignored harmless deprecation warnings from :mod:`typing` in tests (:issue:`5686`, :issue:`5697`). -- Modernized CI configuration (:issue:`5695`, :issue:`5696`). +- Modernized our CI configuration (:issue:`5695`, :issue:`5696`). .. _release-2.7.0: From 6ded3cf4cd134b615239babe28bb28c3ff524b05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Nov 2022 17:00:47 +0600 Subject: [PATCH 57/78] =?UTF-8?q?Bump=20version:=202.7.0=20=E2=86=92=202.7?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 4 ++-- scrapy/VERSION | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f88071685..b949d81c4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.7.0 +current_version = 2.7.1 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 257626526..e5fc2971a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,8 +5,8 @@ Release notes .. _release-2.7.1: -Scrapy 2.7.1 (Not relased yet) ------------------------------- +Scrapy 2.7.1 (2022-11-02) +------------------------- New features ~~~~~~~~~~~~ diff --git a/scrapy/VERSION b/scrapy/VERSION index 24ba9a38d..860487ca1 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.7.0 +2.7.1 From 6c0890ff54a8d49237415e5b7d7dfbf216e88577 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 7 Nov 2022 16:36:54 +0500 Subject: [PATCH 58/78] Simplify the changes after the merge --- tests/test_pipeline_images.py | 37 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 81c9a027a..0c9a5733f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -7,7 +7,6 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf from unittest.mock import patch -from warnings import catch_warnings import attr from itemadapter import ItemAdapter @@ -91,6 +90,22 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_thumbnail_name_from_item(self): + """ + Custom thumbnail name based on item data, overriding default implementation + """ + + class CustomImagesPipeline(ImagesPipeline): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + return f"thumb/{thumb_id}/{item.get('path')}" + + thumb_path = CustomImagesPipeline.from_settings(Settings( + {'IMAGES_STORE': self.tempdir} + )).thumb_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + def test_get_images_exception(self): self.pipeline.min_width = 100 self.pipeline.min_height = 100 @@ -231,22 +246,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - def test_thumbnail_name_from_item(self): - """ - Custom thumbnail name based on item data, overriding default implementation - """ - - class CustomImagesPipeline(ImagesPipeline): - def thumb_path(self, request, thumb_id, response=None, info=None, item=None): - return f"thumb/{thumb_id}/{item.get('path')}" - - thumb_path = CustomImagesPipeline.from_settings(Settings( - {'IMAGES_STORE': self.tempdir} - )).thumb_path - item = dict(path='path-to-store-file') - request = Request("http://example.com") - self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') - class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): @@ -536,11 +535,11 @@ class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): arg = str() - with catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as warnings: NoimagesDrop(arg) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as warnings: class SubclassedNoimagesDrop(NoimagesDrop): pass SubclassedNoimagesDrop(arg) From bbe24d79a5ee6a2afc8cd50bff4ac0e6df26886c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 7 Nov 2022 17:08:54 +0500 Subject: [PATCH 59/78] Fix test issues --- tests/test_pipeline_images.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0c9a5733f..f98d40fda 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,7 +5,6 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf from unittest.mock import patch import attr @@ -535,16 +534,16 @@ class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): arg = str() - with warnings.catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with warnings.catch_warnings(record=True) as warnings: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, ScrapyDeprecationWarning) + with warnings.catch_warnings(record=True) as w: class SubclassedNoimagesDrop(NoimagesDrop): pass SubclassedNoimagesDrop(arg) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, ScrapyDeprecationWarning) def _create_image(format, *a, **kw): From ae3fd0172972f672c3cf3291bcf9f28073e8d4d0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Nov 2022 11:38:46 -0300 Subject: [PATCH 60/78] =?UTF-8?q?Fix=20stray=20=E2=80=9Ccommands=E2=80=9D?= =?UTF-8?q?=20(#5712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/cmdline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 5ee1f0f44..68267fb74 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -7,7 +7,7 @@ import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -32,7 +32,7 @@ def _iter_command_classes(module_name): inspect.isclass(obj) and issubclass(obj, ScrapyCommand) and obj.__module__ == module.__name__ - and not obj == ScrapyCommand + and obj not in (ScrapyCommand, BaseRunSpiderCommand) ): yield obj From 29bf7f5a6c8460e030e465351d2e6d38acf22f3d Mon Sep 17 00:00:00 2001 From: Hassan Shoayb <79839316+Hassan-Shoayb@users.noreply.github.com> Date: Mon, 14 Nov 2022 14:15:00 +0530 Subject: [PATCH 61/78] broad-crawls.rst: fix a typo (#5714) --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 63b60312e..0927ac2d2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -68,7 +68,7 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much -to increase it will depend on how much CPU and memory you crawler will have +to increase it will depend on how much CPU and memory your crawler will have available. A good starting point is ``100``:: From 1200a545439677942085f392d7477ee37b62691e Mon Sep 17 00:00:00 2001 From: islem-esi Date: Tue, 15 Nov 2022 16:28:45 +0100 Subject: [PATCH 62/78] minor fix for readability --- scrapy/cmdline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 68267fb74..8218a51c8 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -78,7 +78,8 @@ def _pop_command_name(argv): def _print_header(settings, inproject): version = scrapy.__version__ if inproject: - print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") + print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") + else: print(f"Scrapy {version} - no active project\n") From d5b6c236a90aac37b7942793e4943347ebaa65b8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 21 Nov 2022 05:32:26 -0300 Subject: [PATCH 63/78] Remove deprecated code (#5719) --- scrapy/utils/boto.py | 23 ----------------------- scrapy/utils/gz.py | 11 ----------- scrapy/utils/python.py | 38 -------------------------------------- tests/test_utils_python.py | 34 ++-------------------------------- 4 files changed, 2 insertions(+), 104 deletions(-) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 3374c57c7..39a681001 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,27 +1,4 @@ """Boto/botocore helpers""" -import warnings - -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning - - -def is_botocore(): - """ Returns True if botocore is available, otherwise raises NotConfigured. Never returns False. - - Previously, when boto was supported in addition to botocore, this returned False if boto was available - but botocore wasn't. - """ - message = ( - 'is_botocore() is deprecated and always returns True or raises an Exception, ' - 'so it cannot be used for checking if boto is available instead of botocore. ' - 'You can use scrapy.utils.boto.is_botocore_available() to check if botocore ' - 'is available.' - ) - warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - try: - import botocore # noqa: F401 - return True - except ImportError: - raise NotConfigured('missing botocore library') def is_botocore_available(): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 76156a4b8..0810e1f1d 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -2,17 +2,6 @@ import struct from gzip import GzipFile from io import BytesIO -from scrapy.utils.decorators import deprecated - - -# - GzipFile's read() has issues returning leftover uncompressed data when -# input is corrupted -# - read1(), which fetches data before raising EOFError on next call -# works here -@deprecated('GzipFile.read1') -def read1(gzf, size=-1): - return gzf.read1(size) - def gunzip(data): """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8ce030d9d..0d9fdbf23 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,20 +1,16 @@ """ This module contains essential stuff that should've come with Python itself ;) """ -import errno import gc import inspect import re import sys -import warnings import weakref from functools import partial, wraps from itertools import chain from typing import AsyncGenerator, AsyncIterable, Iterable, Union -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator -from scrapy.utils.decorators import deprecated def flatten(x): @@ -112,12 +108,6 @@ def to_bytes(text, encoding=None, errors='strict'): return text.encode(encoding, errors) -@deprecated('to_unicode') -def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of ``text``. """ - return to_unicode(text, encoding, errors) - - def re_rsearch(pattern, text, chunk_size=1024): """ This function does a reverse search in a text using a regular expression @@ -263,30 +253,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - -@deprecated -def retry_on_eintr(function, *args, **kw): - """Run a function and retry it while getting EINTR errors""" - while True: - try: - return function(*args, **kw) - except IOError as e: - if e.errno != errno.EINTR: - raise - - def without_none_values(iterable): """Return a copy of ``iterable`` with all ``None`` entries removed. @@ -337,10 +303,6 @@ class MutableChain(Iterable): def __next__(self): return next(self.data) - @deprecated("scrapy.utils.python.MutableChain.__next__") - def next(self): - return self.__next__() - async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: for it in iterables: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b1a8fdc04..403e4f8fe 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,18 +1,14 @@ import functools -import gc import operator import platform -from itertools import count -from warnings import catch_warnings, filterwarnings from twisted.trial import unittest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain, MutableAsyncChain) @@ -27,12 +23,7 @@ class MutableChainTest(unittest.TestCase): m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) self.assertEqual(m.__next__(), 1) - with catch_warnings(record=True) as warnings: - self.assertEqual(m.next(), 2) - self.assertEqual(len(warnings), 1) - self.assertIn('scrapy.utils.python.MutableChain.__next__', - str(warnings[0].message)) - self.assertEqual(list(m), list(range(3, 13))) + self.assertEqual(list(m), list(range(2, 13))) class MutableAsyncChainTest(unittest.TestCase): @@ -209,27 +200,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - wk = WeakKeyCache(lambda k: next(_values)) - - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) - def test_get_func_args(self): def f1(a, b, c): pass From 1a6408c3faadbd2b8b7622b8aee230b112620fad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 21 Nov 2022 07:30:20 -0300 Subject: [PATCH 64/78] Remove `FilteringLinkExtractor` --- scrapy/linkextractors/__init__.py | 89 ------------------------------- scrapy/linkextractors/lxmlhtml.py | 83 ++++++++++++++++++++++------ tests/test_linkextractors.py | 32 ----------- 3 files changed, 66 insertions(+), 138 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 08a6ca1e8..b3b1eea55 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,18 +6,6 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re -from urllib.parse import urlparse -from warnings import warn - -from parsel.csstranslator import HTMLTranslator -from w3lib.url import canonicalize_url - -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import ( - url_is_from_any_domain, url_has_any_extension, -) - # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -55,82 +43,5 @@ def _is_valid_url(url): return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} -class FilteringLinkExtractor: - - _csstranslator = HTMLTranslator() - - def __new__(cls, *args, **kwargs): - from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): - warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' - 'please use scrapy.linkextractors.LinkExtractor instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls) - - def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): - - self.link_extractor = link_extractor - - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] - - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) - - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, - arg_to_iter(restrict_css))) - - self.canonicalize = canonicalize - if deny_extensions is None: - deny_extensions = IGNORED_EXTENSIONS - self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text)] - - def _link_allowed(self, link): - if not _is_valid_url(link.url): - return False - if self.allow_res and not _matches(link.url, self.allow_res): - return False - if self.deny_res and _matches(link.url, self.deny_res): - return False - parsed_url = urlparse(link.url) - if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): - return False - if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): - return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True - - def matches(self, url): - - if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): - return False - - allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] - denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] - return any(allowed) and not any(denied) - - def _process_links(self, links): - links = [x for x in links if self._link_allowed(x)] - if self.canonicalize: - for link in links: - link.url = canonicalize_url(link.url) - links = self.link_extractor._process_links(links) - return links - - def _extract_links(self, *args, **kwargs): - return self.link_extractor._extract_links(*args, **kwargs) - - # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index b5d2585a8..55639f504 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -3,18 +3,20 @@ Link extractor based on lxml.html """ import operator from functools import partial -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse import lxml.etree as etree +from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor +from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches, + _re_type, re) from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url - +from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain # from lxml/src/lxml/html/__init__.py XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" @@ -98,7 +100,8 @@ class LxmlParserLinkExtractor: return links -class LxmlLinkExtractor(FilteringLinkExtractor): +class LxmlLinkExtractor: + _csstranslator = HTMLTranslator() def __init__( self, @@ -118,7 +121,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): restrict_text=None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) - lx = LxmlParserLinkExtractor( + self.link_extractor = LxmlParserLinkExtractor( tag=partial(operator.contains, tags), attr=partial(operator.contains, attrs), unique=unique, @@ -126,18 +129,64 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super().__init__( - link_extractor=lx, - allow=allow, - deny=deny, - allow_domains=allow_domains, - deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, - restrict_css=restrict_css, - canonicalize=canonicalize, - deny_extensions=deny_extensions, - restrict_text=restrict_text, - ) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + self.allow_domains = set(arg_to_iter(allow_domains)) + self.deny_domains = set(arg_to_iter(deny_domains)) + + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, + arg_to_iter(restrict_css))) + + if deny_extensions is None: + deny_extensions = IGNORED_EXTENSIONS + self.canonicalize = canonicalize + self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] + + def _link_allowed(self, link): + if not _is_valid_url(link.url): + return False + if self.allow_res and not _matches(link.url, self.allow_res): + return False + if self.deny_res and _matches(link.url, self.deny_res): + return False + parsed_url = urlparse(link.url) + if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): + return False + if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): + return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False + return True + + def matches(self, url): + + if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): + return False + + allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] + return any(allowed) and not any(denied) + + def _process_links(self, links): + links = [x for x in links if self._link_allowed(x)] + if self.canonicalize: + for link in links: + link.url = canonicalize_url(link.url) + links = self.link_extractor._process_links(links) + return links + + def _extract_links(self, *args, **kwargs): + return self.link_extractor._extract_links(*args, **kwargs) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 6f133d77a..e28dc9bdb 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,12 +1,9 @@ import pickle import re import unittest -from warnings import catch_warnings -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata @@ -517,32 +514,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() - - def test_filteringlinkextractor_deprecation_warning(self): - """Make sure the FilteringLinkExtractor deprecation warning is not - issued for LxmlLinkExtractor""" - with catch_warnings(record=True) as warnings: - LxmlLinkExtractor() - self.assertEqual(len(warnings), 0) - - class SubclassedLxmlLinkExtractor(LxmlLinkExtractor): - pass - - SubclassedLxmlLinkExtractor() - self.assertEqual(len(warnings), 0) - - -class FilteringLinkExtractorTest(unittest.TestCase): - - def test_deprecation_warning(self): - args = [None] * 10 - with catch_warnings(record=True) as warnings: - FilteringLinkExtractor(*args) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedFilteringLinkExtractor(FilteringLinkExtractor): - pass - SubclassedFilteringLinkExtractor(*args) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) From c04ccbceb91b99976b674f6a57e15ed5ad5b7565 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 21 Nov 2022 15:49:33 +0100 Subject: [PATCH 65/78] doc: add instructions to debug Scrapy spiders in Visual Studio Code --- docs/topics/debug.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 4d452b4df..edbcaf432 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -150,3 +150,31 @@ available in all future runs should they be necessary again:: For more information, check the :ref:`topics-logging` section. .. _base tag: https://www.w3schools.com/tags/tag_base.asp + +Visual Studio Code +================== + +.. highlight:: json + +To debug spiders with Visual Studio Code you can use the following ``launch.json``:: + + { + "version": "0.1.0", + "configurations": [ + { + "name": "Python: Launch Scrapy Spider", + "type": "python", + "request": "launch", + "module": "scrapy", + "args": [ + "runspider", + "${file}" + ], + "console": "integratedTerminal" + } + ] + } + + +Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in +your Scrapy spider. From 24a18e9af13a482c1dd6036046226cf57318477b Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Mon, 21 Nov 2022 17:41:06 -0500 Subject: [PATCH 66/78] Adds virtualsize property to _check_limit error log. --- scrapy/extensions/memusage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index f5081a7d7..aba0c8d7e 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem}, extra={'crawler': self.crawler}) + {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " From 8f2adad7a7f2d858acd082afd6e9a6835bfddcd7 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 00:48:24 -0500 Subject: [PATCH 67/78] Log self.get_virtual_size() on every call of _check_limit --- scrapy/extensions/memusage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index aba0c8d7e..c94899e0f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) + {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " @@ -92,6 +92,8 @@ class MemoryUsage: self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() + else: + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) def _check_warning(self): if self.warned: # warn only once From eb159c78f10f546e38e62a9230608269ac13acdd Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 03:36:00 -0500 Subject: [PATCH 68/78] Use variable and convert to megabytes. --- scrapy/extensions/memusage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index c94899e0f..bf2ee4e6d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,7 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - if self.get_virtual_size() > self.limit: + current_mem_usage = self.get_virtual_size() + if current_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -93,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once From 8e0025f53dc724d986855b572b6237d0a96fd821 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 22 Nov 2022 09:38:45 -0300 Subject: [PATCH 69/78] Remove support for override settings with `SCRAPY_` environment variables --- scrapy/utils/project.py | 17 +++++------------ tests/test_utils_project.py | 18 +++++++----------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c66af497e..fce198db4 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -6,7 +6,7 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured ENVVAR = 'SCRAPY_SETTINGS_MODULE' @@ -67,23 +67,16 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if - k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', } - setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars} - if setting_envvars: - setting_envvar_list = ', '.join(sorted(setting_envvars)) - warnings.warn( - 'Use of environment variables prefixed with SCRAPY_ to override ' - 'settings is deprecated. The following environment variables are ' - f'currently defined: {setting_envvar_list}', - ScrapyDeprecationWarning - ) + + scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if + k.startswith('SCRAPY_') and k.replace('SCRAPY_', '') in valid_envvars} + settings.setdict(scrapy_envvars, priority='project') return settings diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 46452415a..e77ffa18b 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -5,9 +5,6 @@ import shutil import contextlib import warnings -from pytest import warns - -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.project import data_path, get_project_settings @@ -80,10 +77,10 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { 'SCRAPY_FOO': 'bar', } - with warns(ScrapyDeprecationWarning, match=': FOO') as record: - with set_env(**envvars): - get_project_settings() - assert len(record) == 1 + with set_env(**envvars): + settings = get_project_settings() + + assert settings.get("SCRAPY_FOO") is None def test_valid_and_invalid_envvars(self): value = 'tests.test_cmdline.settings' @@ -91,8 +88,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): 'SCRAPY_FOO': 'bar', 'SCRAPY_SETTINGS_MODULE': value, } - with warns(ScrapyDeprecationWarning, match=': FOO') as record: - with set_env(**envvars): - settings = get_project_settings() - assert len(record) == 1 + with set_env(**envvars): + settings = get_project_settings() assert settings.get('SETTINGS_MODULE') == value + assert settings.get('SCRAPY_FOO') is None From 1506479672ee54adc2d7d1ecffc0b224f2fcf7aa Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 22 Nov 2022 10:07:32 -0300 Subject: [PATCH 70/78] Remove deprecated test --- tests/test_cmdline/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 8233e0101..802f5c198 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -31,10 +31,6 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), 'override') - def test_override_settings_using_envvar(self): - self.env['SCRAPY_TEST1'] = 'override' - self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') - def test_profiling(self): path = tempfile.mkdtemp() filename = os.path.join(path, 'res.prof') From fc8968672a5cc699f0103aeb2da1ebea7ed3c235 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 11:49:28 -0500 Subject: [PATCH 71/78] renamed variables to clarify that we are using peak memory and not current memory utilization. --- scrapy/extensions/memusage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index bf2ee4e6d..7bc6564e7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,8 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - current_mem_usage = self.get_virtual_size() - if current_mem_usage > self.limit: + peak_mem_usage = self.get_virtual_size() + if peak_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -94,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) + logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once From bdc0bca5b1aab15873b82f93a4bdf8fb3ce82824 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 12:10:49 -0500 Subject: [PATCH 72/78] Replace M occurrences with MiB for accuracy. --- scrapy/extensions/memusage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 7bc6564e7..4fdf86479 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,12 +79,12 @@ class MemoryUsage: if peak_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 - logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", + logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " - f"memory usage exceeded {mem}M at {socket.gethostname()}" + f"memory usage exceeded {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -94,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024}) + logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once @@ -102,12 +102,12 @@ class MemoryUsage: if self.get_virtual_size() > self.warning: self.crawler.stats.set_value('memusage/warning_reached', 1) mem = self.warning / 1024 / 1024 - logger.warning("Memory usage reached %(memusage)dM", + logger.warning("Memory usage reached %(memusage)dMiB", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " - f"memory usage reached {mem}M at {socket.gethostname()}" + f"memory usage reached {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) From 1fdd0a70a0d56bc7829aa3028c191c062cd9f935 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Nov 2022 12:16:48 +0500 Subject: [PATCH 73/78] Restore lost typing --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 780191505..14bf4c5fe 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -151,7 +151,7 @@ class Stream: self._deferred_response = Deferred(_cancel) - def __repr__(self): + def __repr__(self) -> str: return f'Stream(id={self.stream_id!r})' @property From c3b1700774bd16623622963e98fd3ec759b8a88f Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Nov 2022 12:17:30 +0500 Subject: [PATCH 74/78] Restore lost typing --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 305eef918..1ececaf1d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -121,7 +121,7 @@ class Request(object_ref): def encoding(self) -> str: return self._encoding - def __repr__(self): + def __repr__(self) -> str: return f"<{self.method} {self.url}>" def copy(self) -> "Request": From e769532644e1176c7984dd513249285c7f64c7d0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 23 Nov 2022 08:30:11 -0300 Subject: [PATCH 75/78] Remove `noconnect` deprecate code --- scrapy/core/downloader/handlers/http11.py | 17 ++--------------- scrapy/core/downloader/handlers/http2.py | 14 +------------- tests/test_downloader_handlers.py | 14 +------------- tests/test_downloader_handlers_http2.py | 15 --------------- 4 files changed, 4 insertions(+), 56 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6b8a18f1a..f07f0780e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -3,7 +3,6 @@ import ipaddress import logging import re -import warnings from contextlib import suppress from io import BytesIO from time import time @@ -22,7 +21,7 @@ from zope.interface import implementer from scrapy import signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload +from scrapy.exceptions import StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode @@ -279,17 +278,7 @@ class ScrapyAgent: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) - omitConnectTunnel = b'noconnect' in proxyParams - if omitConnectTunnel: - warnings.warn( - "Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Zyte Smart Proxy Manager, it doesn't require " - "this mode anymore, so you should update scrapy-crawlera " - "to scrapy-zyte-smartproxy and remove '?noconnect' " - "from the Zyte Smart Proxy Manager URL.", - ScrapyDeprecationWarning, - ) - if scheme == b'https' and not omitConnectTunnel: + if scheme == b'https': proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) return self._TunnelingAgent( @@ -302,8 +291,6 @@ class ScrapyAgent: ) else: proxyScheme = proxyScheme or b'http' - proxyHost = to_bytes(proxyHost, encoding='ascii') - proxyPort = to_bytes(str(proxyPort), encoding='ascii') proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 7bb88a193..3f1b36e92 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,4 +1,3 @@ -import warnings from time import time from typing import Optional, Type, TypeVar from urllib.parse import urldefrag @@ -69,19 +68,8 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = proxy_host.decode() - omit_connect_tunnel = b'noconnect' in proxy_params - if omit_connect_tunnel: - warnings.warn( - "Using HTTPS proxies in the noconnect mode is not " - "supported by the downloader handler. If you use Zyte " - "Smart Proxy Manager, it doesn't require this mode " - "anymore, so you should update scrapy-crawlera to " - "scrapy-zyte-smartproxy and remove '?noconnect' from the " - "Zyte Smart Proxy Manager URL." - ) - if scheme == b'https' and not omit_connect_tunnel: + if scheme == b'https': # ToDo raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') return self._ProxyAgent( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 883960084..c69bd3da1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes @@ -757,18 +757,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') - - http_proxy = f'{self.getURL("")}?noconnect' - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex(ScrapyDeprecationWarning, - r'Using HTTPS proxies in the noconnect mode is deprecated'): - return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 3a9db3ee5..079267535 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -242,21 +242,6 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/') - - http_proxy = f"{self.getURL('')}?noconnect" - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex( - Warning, - r'Using HTTPS proxies in the noconnect mode is not supported by the ' - r'downloader handler.' - ): - return self.download_request(request, Spider('foo')).addCallback(_test) - @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): with self.assertRaises(NotImplementedError): From f6e9e6592a28a11517295847b177b14f41cb8a26 Mon Sep 17 00:00:00 2001 From: Hanzallah Burney Date: Wed, 23 Nov 2022 19:48:34 +0500 Subject: [PATCH 76/78] Cleanup settings._DictProxy and scrapy.telnet (#5730) --- scrapy/settings/__init__.py | 24 ------------------------ scrapy/utils/deprecate.py | 5 ++--- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b4e12ffdc..43ee433d1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -435,30 +435,6 @@ class BaseSettings(MutableMapping): p.text(pformat(self.copy_to_dict())) -class _DictProxy(MutableMapping): - - def __init__(self, settings, priority): - self.o = {} - self.settings = settings - self.priority = priority - - def __len__(self): - return len(self.o) - - def __getitem__(self, k): - return self.o[k] - - def __setitem__(self, k, v): - self.settings.set(k, v, priority=self.priority) - self.o[k] = v - - def __delitem__(self, k): - del self.o[k] - - def __iter__(self, k, v): - return iter(self.o) - - class Settings(BaseSettings): """ This object stores Scrapy settings for the configuration of internal diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ae727464c..a0c83f9f1 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,6 +2,7 @@ import warnings import inspect +from typing import List, Tuple from scrapy.exceptions import ScrapyDeprecationWarning @@ -126,9 +127,7 @@ def _clspath(cls, forced=None): return f'{cls.__module__}.{cls.__name__}' -DEPRECATION_RULES = [ - ('scrapy.telnet.', 'scrapy.extensions.telnet.'), -] +DEPRECATION_RULES: List[Tuple[str, str]] = [] def update_classpath(path): From b6e98ce6b6c766ee735f45f25f78d957327f54e9 Mon Sep 17 00:00:00 2001 From: Hanzallah Burney Date: Thu, 24 Nov 2022 15:01:15 +0100 Subject: [PATCH 77/78] Remove unnecessary backwards compatibility comments (#5732) --- scrapy/utils/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 6404edda6..7a5f8f065 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -44,14 +44,12 @@ def build_component_list(compdict, custom=None, convert=update_classpath): raise ValueError(f'Invalid value {value} for component {name}, ' 'please provide a real number or None instead') - # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) return type(custom)(convert(c) for c in custom) if custom is not None: compdict.update(custom) - # END Backward compatibility _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) From f85c3f3d68b03b12c0af3b3aa09ab5faad19bc37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Nov 2022 18:46:14 +0600 Subject: [PATCH 78/78] Add a comment about `tmpname + '^'`. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c69bd3da1..8835267fe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -109,6 +109,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): self.tmpname = self.mktemp() + # add a special char to check that they are handled correctly with open(self.tmpname + '^', 'w') as f: f.write('0123456789') handler = create_instance(FileDownloadHandler, None, get_crawler())