From 25e616fa04bf4199b9017553b3e57699c40b0683 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 15 Mar 2019 18:09:47 +0530 Subject: [PATCH 001/237] do not degrade JPEG files. --- scrapy/pipelines/images.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a1457c7e9..add606a2e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -127,7 +127,7 @@ class ImagesPipeline(FilesPipeline): raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - image, buf = self.convert_image(orig_image) + image, buf = self.convert_image(orig_image, BytesIO(response.body)) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): @@ -135,7 +135,7 @@ class ImagesPipeline(FilesPipeline): thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, size=None): + def convert_image(self, image, response_body, size=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -152,6 +152,9 @@ class ImagesPipeline(FilesPipeline): image = image.copy() image.thumbnail(size, Image.ANTIALIAS) + if not size and image.format == 'JPEG': + return image, response_body + buf = BytesIO() image.save(buf, 'JPEG') return image, buf From 07487dd487f7a10fbf0693e378b7b66909a67d6b Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 15 Mar 2019 23:29:53 +0530 Subject: [PATCH 002/237] make tests work with new convert_image --- tests/test_pipeline_images.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index a7c652959..eb3347442 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -79,28 +79,28 @@ class ImagesPipelineTestCase(unittest.TestCase): SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) + im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, _ = self.pipeline.convert_image(im) + converted, buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -406,7 +406,7 @@ def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) - return Image.open(buf) + return Image.open(buf), buf if __name__ == "__main__": From ca882d8d9f094244d9f1d3476fb72fab9c230765 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Wed, 27 Mar 2019 19:10:44 +0530 Subject: [PATCH 003/237] include test --- tests/test_pipeline_images.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index eb3347442..efa96e146 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -80,9 +80,10 @@ class ImagesPipelineTestCase(unittest.TestCase): # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, buf = self.pipeline.convert_image(im, buf) + converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + self.assertEqual(converted_buf.read(), buf.read()) # check that thumbnail keep image ratio thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) From 484bd0d22a11a04ab775ac4f72c75f3ec9050d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 22 Mar 2019 16:59:29 +0100 Subject: [PATCH 004/237] Allow customizing export column names --- docs/topics/exporters.rst | 35 +++++++++++++++----- docs/topics/feed-exports.rst | 17 +++------- scrapy/exporters.py | 32 +++++++++++++++--- scrapy/extensions/feedexport.py | 3 +- scrapy/settings/__init__.py | 35 +++++++++++++++++++- tests/test_exporters.py | 15 ++++++++- tests/test_feedexport.py | 57 +++++++++++++++++++++++++++++++++ 7 files changed, 165 insertions(+), 29 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index f5048d2da..42a93e459 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -190,14 +190,33 @@ BaseItemExporter .. attribute:: fields_to_export - A list with the name of the fields that will be exported, or None if you - want to export all fields. Defaults to None. + Fields to export, their order [1]_ and their output names. - Some exporters (like :class:`CsvItemExporter`) respect the order of the - fields defined in this attribute. + Possible values are: - Some exporters may require fields_to_export list in order to export the - data properly when spiders return dicts (not :class:`~Item` instances). + - ``None`` (all fields [2]_, default) + + - A list of fields:: + + ['field1', 'field2'] + + - A dict [3]_ where keys are fields and values are output names:: + + {'field1': 'Field 1', 'field2': 'Field 2'} + + .. [1] Not all exporters respect the specified field order. + .. [2] If you yield items as dicts (not :class:`Item` instances), + exporters that need to know the fields to export beforehand, like + :class:`CsvItemExporter`, only export the fields found in the + first item. + .. [3] Dicts preserve insertion order since `Python 3.7`_ + (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version + of Python, use an OrderedDict_ to enforce a specific field order. + + .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html + .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation + .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html + .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict .. attribute:: export_empty_fields @@ -286,8 +305,8 @@ CsvItemExporter Exports Items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the - CSV columns and their order. The :attr:`export_empty_fields` attribute has - no effect on this exporter. + CSV columns, their order and their column names. The + :attr:`export_empty_fields` attribute has no effect on this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cf70b8aca..968cb8884 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -56,7 +56,7 @@ CSV * :setting:`FEED_FORMAT`: ``csv`` * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` - * To specify columns to export and their order use + * To specify columns to export, their order and their column names, use :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this option, but it is important for CSV because unlike many other export formats CSV uses a fixed header. @@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS Default: ``None`` -A list of fields to export, optional. -Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``. - -Use FEED_EXPORT_FIELDS option to define fields to export and their order. - -When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields -defined in dicts or :class:`~.Item` subclasses a spider is yielding. - -If an exporter requires a fixed set of fields (this is the case for -:ref:`CSV ` export format) and FEED_EXPORT_FIELDS -is empty or None, then Scrapy tries to infer field names from the -exported data - currently it uses field names from the first item. +Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their +order and their output names. See :attr:`BaseItemExporter.fields_to_export +` for more information. .. setting:: FEED_EXPORT_INDENT diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 695c74fec..c05acaca5 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,6 +2,7 @@ Item Exporters are used to export/serialize items into different formats. """ +from collections import Mapping import csv import io import sys @@ -64,6 +65,14 @@ class BaseItemExporter(object): field_iter = six.iterkeys(item.fields) else: field_iter = six.iterkeys(item) + elif isinstance(self.fields_to_export, Mapping): + if include_empty: + field_iter = self.fields_to_export.items() + else: + field_iter = ( + (x, y) for x, y in self.fields_to_export.items() + if x in item + ) else: if include_empty: field_iter = self.fields_to_export @@ -71,13 +80,22 @@ class BaseItemExporter(object): field_iter = (x for x in self.fields_to_export if x in item) for field_name in field_iter: - if field_name in item: - field = {} if isinstance(item, dict) else item.fields[field_name] - value = self.serialize_field(field, field_name, item[field_name]) + if isinstance(field_name, six.string_types): + item_field, output_field = field_name, field_name + else: + item_field, output_field = field_name + if item_field in item: + if isinstance(item, dict): + field = {} + else: + field = item.fields[item_field] + value = self.serialize_field( + field, output_field, item[item_field] + ) else: value = default_value - yield field_name, value + yield output_field, value class JsonLinesItemExporter(BaseItemExporter): @@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter): else: # use fields declared in Item self.fields_to_export = list(item.fields.keys()) - row = list(self._build_row(self.fields_to_export)) + if isinstance(self.fields_to_export, Mapping): + fields = self.fields_to_export.values() + else: + fields = self.fields_to_export + row = list(self._build_row(fields)) self.csv_writer.writerow(row) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index b2f7267a2..1ed476d83 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -201,7 +201,8 @@ class FeedExporter(object): raise NotConfigured self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False - self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None + self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS') + self.export_fields = self.export_fields or None self.indent = None if settings.get('FEED_EXPORT_INDENT') is not None: self.indent = settings.getint('FEED_EXPORT_INDENT') diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 14c93bef2..69b324e84 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,7 +1,7 @@ import six import json import copy -from collections import MutableMapping +from collections import MutableMapping, OrderedDict from importlib import import_module from pprint import pformat @@ -198,6 +198,39 @@ class BaseSettings(MutableMapping): value = json.loads(value) return dict(value) + def getdictorlist(self, name, default=None): + """Get a setting value as either an ``OrderedDict`` or a list. + + If the setting is already a dict or a list, a copy of it will be + returned. + + If it is a string it will be evaluated as JSON, or as a comma-separated + list of strings as a fallback. + + For example, settings populated through environment variables will + return: + + - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to + ``'{"key1": "value1", "key2": "value2"}'`` + + - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + """ + value = self.get(name, default) + if value is None: + return {} + if isinstance(value, six.string_types): + try: + return json.loads(value, object_pairs_hook=OrderedDict) + except ValueError: + return value.split(',') + return copy.deepcopy(value) + def getwithbase(self, name): """Get a composition of a dictionary-like setting and its `_BASE` counterpart. diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cd72c661a..1b3dc14a1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,4 +1,7 @@ +# -*- coding:utf-8 -*- + from __future__ import absolute_import +from collections import OrderedDict import re import json import marshal @@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase): assert isinstance(name, six.text_type) self.assertEqual(name, u'John\xa3') + ie = self._get_exporter( + fields_to_export=OrderedDict([('name', u'名稱')]) + ) + self.assertEqual( + list(ie._get_serialized_fields(self.i)), + [(u'名稱', u'John\xa3')] + ) + def test_field_custom_serializer(self): def custom_serializer(value): return str(int(value) + 2) @@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest): class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): + self.output = tempfile.TemporaryFile() return CsvItemExporter(self.output, **kwargs) def assertCsvEqual(self, first, second, msg=None): @@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(csvsplit(first), csvsplit(second), msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.output.seek(0) + self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eef0384cf..4f72c0ff4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from collections import OrderedDict import os import csv import json @@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + # fields may be defined as a comma-separated list + header = ["foo", "baz", "hello"] + settings = {'FEED_EXPORT_FIELDS': ",".join(header)} + rows = [ + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, + {'foo': '', 'baz': '', 'hello': 'world4'}, + ] + yield self.assertExported(items, header, rows, + settings=settings, ordered=True) + + # fields may also be defined as a JSON array + header = ["foo", "baz", "hello"] + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [ + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, + {'foo': '', 'baz': '', 'hello': 'world4'}, + ] + yield self.assertExported(items, header, rows, + settings=settings, ordered=True) + + # custom output field names can be specified + header = OrderedDict(( + ("foo", "Foo"), + ("baz", "Baz"), + ("hello", "Hello"), + )) + settings = {'FEED_EXPORT_FIELDS': header} + rows = [ + {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, + {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, + {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, + {'Foo': '', 'Baz': '', 'Hello': 'world4'}, + ] + yield self.assertExported(items, list(header.values()), rows, + settings=settings, ordered=True) + + # custom output field names can be specified as a JSON object + header = OrderedDict(( + ("foo", "Foo"), + ("baz", "Baz"), + ("hello", "Hello"), + )) + settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} + rows = [ + {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, + {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, + {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, + {'Foo': '', 'Baz': '', 'Hello': 'world4'}, + ] + yield self.assertExported(items, list(header.values()), rows, + settings=settings, ordered=True) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as From 398639a0bfe749eee07e51249f04b2c6c93eab73 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 8 Apr 2019 12:27:36 +0530 Subject: [PATCH 005/237] fix test --- scrapy/pipelines/images.py | 5 +++-- tests/test_pipeline_images.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index add606a2e..746244dab 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -132,7 +132,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - thumb_image, thumb_buf = self.convert_image(image, size) + thumb_image, thumb_buf = self.convert_image(image, buf, size) yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, response_body, size=None): @@ -153,7 +153,8 @@ class ImagesPipeline(FilesPipeline): image.thumbnail(size, Image.ANTIALIAS) if not size and image.format == 'JPEG': - return image, response_body + buf = BytesIO(response_body.read()) + return image, buf buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index efa96e146..dde0fa030 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -83,6 +83,47 @@ class ImagesPipelineTestCase(unittest.TestCase): converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that we don't convert JPEGs again + buf = io.BytesIO((b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00' + b'\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c' + b'\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c ' + b'$.\' ",#\x1c\x1c(7),01444\x1f\'9=82<.342\xff\xdb\x00C\x01\t\t\t\x0c\x0b\x0c' + b'\x18\r\r\x182!\x1c!222222222222222222222222222222222222222222222222' + b'22\xff\xc0\x00\x11\x08\x00\x14\x00\x14\x03\x01"\x00\x02\x11\x01\x03\x11' + b'\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' + b'\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00' + b'\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81' + b"\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456" + b'789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a' + b'\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8' + b'\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6' + b'\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3' + b'\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' + b'\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01' + b'\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' + b'\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00' + b'\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08' + b"\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&'" + b'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88' + b'\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6' + b'\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4' + b'\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2' + b'\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' + b'\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbb' + b'\xe2\x1b\xcb\x88\xe4\x90\t\x0e1\xd2\xb9\xab\x1dF\xe6\xda\xe0\xb4l\xd9' + b'5>\xaf\xac\xc7y}$k\x92\x03u\xaaL\xeb\x18VL\x9fZ+\xb9\xb9EBV]Ow\x0f\x8a\xc0' + b'\xac\x14\xbd\xa2NG_\xa5x\x8esg\xfb\xd9\x8e\xed\xc79\xa2\xa1\xd0\xf4\x8bk' + b'\x9d5e\xdeN\xe6=\xa8\xae\xa9V\xc2\xb6|\xdf2\xeep:w\xfc\x84d\x04g\x9e\xf5\xd3' + b'L\xa9\x0c\x1f*/>\xa2\x8a+\xca\x93z\x1c\x15]\xa9\xc6\xc4Q\xea\x97V\xca' + b'c\x85\xc2\xaes\x80(\xa2\x8a\xe3\x92W<\xdb\x9f\xff\xd9')) + im = Image.open(buf) + buf.seek(0) + converted, converted_buf = self.pipeline.convert_image(im, buf) + converted_buf.seek(0) + buf.seek(0) + self.assertEqual(im.format, "JPEG") self.assertEqual(converted_buf.read(), buf.read()) # check that thumbnail keep image ratio From c6769d6887b5c311c9083a5ac0349e81d1a2aea7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 13 Apr 2019 10:10:08 +0530 Subject: [PATCH 006/237] make suggested changes --- scrapy/pipelines/images.py | 6 ++--- tests/test_pipeline_images.py | 42 +---------------------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 746244dab..3450ee721 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -151,10 +151,8 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - - if not size and image.format == 'JPEG': - buf = BytesIO(response_body.read()) - return image, buf + elif image.format == 'JPEG': + return image, response_body buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dde0fa030..b419039b3 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -83,48 +83,8 @@ class ImagesPipelineTestCase(unittest.TestCase): converted, converted_buf = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - # check that we don't convert JPEGs again - buf = io.BytesIO((b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00' - b'\xff\xdb\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c' - b'\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c ' - b'$.\' ",#\x1c\x1c(7),01444\x1f\'9=82<.342\xff\xdb\x00C\x01\t\t\t\x0c\x0b\x0c' - b'\x18\r\r\x182!\x1c!222222222222222222222222222222222222222222222222' - b'22\xff\xc0\x00\x11\x08\x00\x14\x00\x14\x03\x01"\x00\x02\x11\x01\x03\x11' - b'\x01\xff\xc4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' - b'\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00' - b'\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07"q\x142\x81' - b"\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&'()*456" - b'789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a' - b'\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8' - b'\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6' - b'\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3' - b'\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' - b'\xfa\xff\xc4\x00\x1f\x01\x00\x03\x01\x01\x01\x01\x01\x01\x01\x01' - b'\x01\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xff\xc4' - b'\x00\xb5\x11\x00\x02\x01\x02\x04\x04\x03\x04\x07\x05\x04\x04\x00' - b'\x01\x02w\x00\x01\x02\x03\x11\x04\x05!1\x06\x12AQ\x07aq\x13"2\x81\x08' - b"\x14B\x91\xa1\xb1\xc1\t#3R\xf0\x15br\xd1\n\x16$4\xe1%\xf1\x17\x18\x19\x1a&'" - b'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x82\x83\x84\x85\x86\x87\x88' - b'\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6' - b'\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4' - b'\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe2' - b'\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9' - b'\xfa\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbb' - b'\xe2\x1b\xcb\x88\xe4\x90\t\x0e1\xd2\xb9\xab\x1dF\xe6\xda\xe0\xb4l\xd9' - b'5>\xaf\xac\xc7y}$k\x92\x03u\xaaL\xeb\x18VL\x9fZ+\xb9\xb9EBV]Ow\x0f\x8a\xc0' - b'\xac\x14\xbd\xa2NG_\xa5x\x8esg\xfb\xd9\x8e\xed\xc79\xa2\xa1\xd0\xf4\x8bk' - b'\x9d5e\xdeN\xe6=\xa8\xae\xa9V\xc2\xb6|\xdf2\xeep:w\xfc\x84d\x04g\x9e\xf5\xd3' - b'L\xa9\x0c\x1f*/>\xa2\x8a+\xca\x93z\x1c\x15]\xa9\xc6\xc4Q\xea\x97V\xca' - b'c\x85\xc2\xaes\x80(\xa2\x8a\xe3\x92W<\xdb\x9f\xff\xd9')) - im = Image.open(buf) - buf.seek(0) - converted, converted_buf = self.pipeline.convert_image(im, buf) - converted_buf.seek(0) - buf.seek(0) - self.assertEqual(im.format, "JPEG") - self.assertEqual(converted_buf.read(), buf.read()) + self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) From 6039b66f42fe2c6e7708febb70939952cedaedd0 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 13 Apr 2019 10:17:26 +0530 Subject: [PATCH 007/237] aesthetic changes --- tests/test_pipeline_images.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index b419039b3..0a2153ddb 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -87,14 +87,14 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, buf = self.pipeline.convert_image(converted, buf, size=(10, 25)) + thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25)) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, buf = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, buf = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) From 2a6bcdb413da0a4202fe905279d46d581368769d Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 29 Apr 2019 21:21:02 +0530 Subject: [PATCH 008/237] makes fix backward compatible --- scrapy/pipelines/images.py | 45 ++++++++++++++++++++++++++++++----- tests/test_pipeline_images.py | 8 +++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 3450ee721..ca8ac7b83 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -126,16 +126,42 @@ class ImagesPipeline(FilesPipeline): if width < self.min_width or height < self.min_height: raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) + + def _is_convert_image_overriden(): + import inspect + if six.PY2: + convert_image_signature = inspect.getargspec(self.convert_image) + elif six.PY3: + convert_image_signature = inspect.getfullargspec(self.convert_image) + if 'response_body' not in convert_image_signature.args: + return True + return False - image, buf = self.convert_image(orig_image, BytesIO(response.body)) + def _warn(): + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' + 'overriden method does not accept response_body attribute.', + category=ScrapyDeprecationWarning, stacklevel=1) + + convert_image_overriden = _is_convert_image_overriden() + if convert_image_overriden: + _warn() + image, buf = self.convert_image(orig_image) + else: + image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) yield path, image, buf for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - thumb_image, thumb_buf = self.convert_image(image, buf, size) + if convert_image_overriden: + _warn() + thumb_image, thumb_buf = self.convert_image(image, size) + else: + thumb_image, thumb_buf = self.convert_image(image, size, buf) yield thumb_path, thumb_image, thumb_buf - def convert_image(self, image, response_body, size=None): + def convert_image(self, image, size=None, response_body=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -151,9 +177,16 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - elif image.format == 'JPEG': - return image, response_body - + else: + if not response_body: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + 'method called without response_body attribute.', + category=ScrapyDeprecationWarning, stacklevel=1) + elif image.format == 'JPEG': + return image, response_body + buf = BytesIO() image.save(buf, 'JPEG') return image, buf diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0a2153ddb..3fe71589d 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -80,21 +80,21 @@ class ImagesPipelineTestCase(unittest.TestCase): # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, converted_buf = self.pipeline.convert_image(im, buf) + converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, converted_buf, size=(10, 25)) + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) self.assertEqual(thumbnail.mode, 'RGB') self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) @@ -102,7 +102,7 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255, 50) im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') - converted, _ = self.pipeline.convert_image(im, buf) + converted, _ = self.pipeline.convert_image(im, response_body=buf) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) From 33925a77610293c44a169efd1c274239a75ba968 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 7 May 2019 15:44:21 +0530 Subject: [PATCH 009/237] test for deprecation warning --- scrapy/pipelines/images.py | 20 +++++------ tests/test_pipeline_images.py | 63 ++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index ca8ac7b83..9776817bc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -141,7 +141,7 @@ class ImagesPipeline(FilesPipeline): from scrapy.exceptions import ScrapyDeprecationWarning import warnings warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' - 'overriden method does not accept response_body attribute.', + 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) convert_image_overriden = _is_convert_image_overriden() @@ -162,6 +162,13 @@ class ImagesPipeline(FilesPipeline): yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None, response_body=None): + if not response_body: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + 'method called without response_body argument.', + category=ScrapyDeprecationWarning, stacklevel=1) + if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) @@ -177,15 +184,8 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - else: - if not response_body: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' - 'method called without response_body attribute.', - category=ScrapyDeprecationWarning, stacklevel=1) - elif image.format == 'JPEG': - return image, response_body + elif response_body and image.format == 'JPEG': + return image, response_body buf = BytesIO() image.save(buf, 'JPEG') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3fe71589d..651691862 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -76,37 +76,48 @@ class ImagesPipelineTestCase(unittest.TestCase): 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') def test_convert_image(self): - SIZE = (100, 100) - # straigh forward case: RGB and JPEG + # tests for old API + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + + SIZE = (100, 100) + # straigh forward case: RGB and JPEG + COLOUR = (0, 127, 255) + im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, converted_buf = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # ensure that we recieved deprecation warnings + self.assertTrue(len(w) >= 4) + + # tests for new API + # check that we don't convert JPEGs again COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) - self.assertEqual(thumbnail.mode, 'RGB') - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') - converted, _ = self.pipeline.convert_image(im, response_body=buf) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): return self.image_key(url) From 881bade2c1b2d5842f7d0bdf2648455996f814b7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 7 May 2019 16:12:26 +0530 Subject: [PATCH 010/237] tests for new API --- tests/test_pipeline_images.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 651691862..1dfca5c11 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -79,12 +79,11 @@ class ImagesPipelineTestCase(unittest.TestCase): # tests for old API with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') - SIZE = (100, 100) # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) - im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, converted_buf = self.pipeline.convert_image(im) + im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) @@ -95,14 +94,14 @@ class ImagesPipelineTestCase(unittest.TestCase): # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) - im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') converted, _ = self.pipeline.convert_image(im) self.assertEqual(converted.mode, 'RGB') @@ -112,12 +111,36 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertTrue(len(w) >= 4) # tests for new API - # check that we don't convert JPEGs again + SIZE = (100, 100) + # straigh forward case: RGB and JPEG COLOUR = (0, 127, 255) im, buf = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, converted_buf = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + # check that we don't convert JPEGs again self.assertEqual(converted_buf, buf) + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25), response_body=converted_buf) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, buf = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im, response_body=buf) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): return self.image_key(url) From 653ac3eebe96bdfff8128cdb1e500728215486a2 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 16 May 2019 06:20:39 +0000 Subject: [PATCH 011/237] makes suggested changes --- scrapy/pipelines/images.py | 1 - tests/test_pipeline_images.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9776817bc..b95383b3b 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -155,7 +155,6 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) if convert_image_overriden: - _warn() thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 1dfca5c11..ba79dd6bd 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -108,7 +108,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - self.assertTrue(len(w) >= 4) + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a incompatible way' in str(warning.message)]) == 4) # tests for new API SIZE = (100, 100) From 2994b624e03ec9a2cba2c024eacf549329c056de Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Fri, 17 May 2019 12:14:43 +0000 Subject: [PATCH 012/237] makes suggested changes --- scrapy/pipelines/images.py | 21 +++++---------------- tests/test_pipeline_images.py | 2 +- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index b95383b3b..fd34c9df6 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,7 +15,7 @@ except ImportError: from PIL import Image from scrapy.utils.misc import md5sum -from scrapy.utils.python import to_bytes +from scrapy.utils.python import to_bytes, get_func_args from scrapy.http import Request from scrapy.settings import Settings from scrapy.exceptions import DropItem @@ -127,25 +127,14 @@ class ImagesPipeline(FilesPipeline): raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - def _is_convert_image_overriden(): - import inspect - if six.PY2: - convert_image_signature = inspect.getargspec(self.convert_image) - elif six.PY3: - convert_image_signature = inspect.getfullargspec(self.convert_image) - if 'response_body' not in convert_image_signature.args: - return True - return False - def _warn(): from scrapy.exceptions import ScrapyDeprecationWarning import warnings - warnings.warn('ImagesPipeline.convert_image() method overriden in a incompatible way, ' + warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) - convert_image_overriden = _is_convert_image_overriden() - if convert_image_overriden: + if 'response_body' not in get_func_args(self.convert_image): _warn() image, buf = self.convert_image(orig_image) else: @@ -154,7 +143,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if convert_image_overriden: + if 'response_body' not in get_func_args(self.convert_image): thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) @@ -164,7 +153,7 @@ class ImagesPipeline(FilesPipeline): if not response_body: from scrapy.exceptions import ScrapyDeprecationWarning import warnings - warnings.warn('ImagesPipeline.convert_image() method called in a incompatible way, ' + warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ba79dd6bd..ec0c87264 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -108,7 +108,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a incompatible way' in str(warning.message)]) == 4) + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) # tests for new API SIZE = (100, 100) From c8e28ec194b3730e7954a18526ac77cb44138feb Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Thu, 23 May 2019 15:04:21 +0530 Subject: [PATCH 013/237] makes suggested changes --- scrapy/pipelines/images.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fd34c9df6..a3f5f9292 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -134,7 +134,8 @@ class ImagesPipeline(FilesPipeline): 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) - if 'response_body' not in get_func_args(self.convert_image): + convert_image_overriden = 'response_body' not in get_func_args(self.convert_image) + if convert_image_overriden: _warn() image, buf = self.convert_image(orig_image) else: @@ -143,7 +144,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if 'response_body' not in get_func_args(self.convert_image): + if convert_image_overriden: thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) From 90fdefcbca89d0ef2cc81955b9fd8b8af1dff392 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Sat, 25 May 2019 19:11:48 +0530 Subject: [PATCH 014/237] cache if convert_image has deprecated signature --- scrapy/pipelines/images.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a3f5f9292..f709c5057 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -84,6 +84,8 @@ class ImagesPipeline(FilesPipeline): resolve('IMAGES_THUMBS'), self.THUMBS ) + self._deprecated_convert_image = None + @classmethod def from_settings(cls, settings): s3store = cls.STORE_SCHEMES['s3'] @@ -126,17 +128,17 @@ class ImagesPipeline(FilesPipeline): if width < self.min_width or height < self.min_height: raise ImageException("Image too small (%dx%d < %dx%d)" % (width, height, self.min_width, self.min_height)) - - def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' - 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) - convert_image_overriden = 'response_body' not in get_func_args(self.convert_image) - if convert_image_overriden: - _warn() + if self._deprecated_convert_image is None: + self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) + if self._deprecated_convert_image: + from scrapy.exceptions import ScrapyDeprecationWarning + import warnings + warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + 'overriden method does not accept response_body argument.', + category=ScrapyDeprecationWarning, stacklevel=1) + + if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) else: image, buf = self.convert_image(orig_image, response_body=BytesIO(response.body)) @@ -144,7 +146,7 @@ class ImagesPipeline(FilesPipeline): for thumb_id, size in six.iteritems(self.thumbs): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) - if convert_image_overriden: + if self._deprecated_convert_image: thumb_image, thumb_buf = self.convert_image(image, size) else: thumb_image, thumb_buf = self.convert_image(image, size, buf) From 20719bac5cdc4898e9f01fcf4f92aa781a9d0ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 17 Dec 2019 15:09:43 +0100 Subject: [PATCH 015/237] Fix import error --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c1fff4d95..98421be18 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -224,7 +224,7 @@ class BaseSettings(MutableMapping): value = self.get(name, default) if value is None: return {} - if isinstance(value, six.string_types): + if isinstance(value, str): try: return json.loads(value, object_pairs_hook=OrderedDict) except ValueError: From 5834088e670d93b2a63ad8afb258e687af0a9b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 18 Feb 2020 14:18:15 +0100 Subject: [PATCH 016/237] Apply feedback --- scrapy/settings/__init__.py | 5 +- tests/test_feedexport.py | 109 ++++++++++++++++++------------------ 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 98421be18..6f5b1ef97 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -207,8 +207,7 @@ class BaseSettings(MutableMapping): If it is a string it will be evaluated as JSON, or as a comma-separated list of strings as a fallback. - For example, settings populated through environment variables will - return: + For example, settings populated from the command line will return: - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to ``'{"key1": "value1", "key2": "value2"}'`` @@ -223,7 +222,7 @@ class BaseSettings(MutableMapping): """ value = self.get(name, default) if value is None: - return {} + return OrderedDict() if isinstance(value, str): try: return json.loads(value, object_pairs_hook=OrderedDict) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 291c47702..781cdc543 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -5,6 +5,7 @@ import warnings import tempfile import shutil import string +import sys from collections import OrderedDict from io import BytesIO from pathlib import Path @@ -12,6 +13,7 @@ from unittest import mock from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url +import pytest from zope.interface.verify import verifyObject from twisted.trial import unittest from twisted.internet import defer @@ -590,78 +592,75 @@ class FeedExportTest(unittest.TestCase): yield self.assertExportedCsv(items, header, rows_csv, ordered=False) yield self.assertExportedJsonLines(items, rows_jl) - # edge case: FEED_EXPORT_FIELDS==[] means the same as default None + @defer.inlineCallbacks + def test_export_items_empty_field_list(self): + # FEED_EXPORT_FIELDS==[] means the same as default None + items = [{'foo': 'bar'}] + header = ["foo"] + rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) - yield self.assertExportedJsonLines(items, rows_jl, settings) + yield self.assertExportedCsv(items, header, rows, ordered=False) + yield self.assertExportedJsonLines(items, rows, settings) - # it is possible to override fields using FEED_EXPORT_FIELDS - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': header} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) - # fields may be defined as a comma-separated list - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_comma_separated_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': ",".join(header)} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) - # fields may also be defined as a JSON array - header = ["foo", "baz", "hello"] + @defer.inlineCallbacks + def test_export_items_json_field_list(self): + items = [{'foo': 'bar'}] + header = ["foo", "baz"] + rows = [{'foo': 'bar', 'baz': ''}] settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} - rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, - {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, - ] - yield self.assertExported(items, header, rows, - settings=settings, ordered=True) + yield self.assertExported(items, header, rows, settings=settings) - # custom output field names can be specified + @defer.inlineCallbacks + def test_export_items_field_names(self): + items = [{'foo': 'bar'}] header = OrderedDict(( ("foo", "Foo"), - ("baz", "Baz"), - ("hello", "Hello"), )) + rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': header} - rows = [ - {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, - {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, - {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, - {'Foo': '', 'Baz': '', 'Hello': 'world4'}, - ] yield self.assertExported(items, list(header.values()), rows, - settings=settings, ordered=True) + settings=settings) - # custom output field names can be specified as a JSON object + @pytest.mark.skipif(sys.version_info < (3, 7), + reason='Only official in Python 3.7+') + @defer.inlineCallbacks + def test_export_items_dict_field_names(self): + items = [{'foo': 'bar'}] + header = { + 'baz': 'Baz', + 'foo': 'Foo', + } + rows = [{'Baz': '', 'Foo': 'bar'}] + settings = {'FEED_EXPORT_FIELDS': header} + yield self.assertExported(items, ['Baz', 'Foo'], rows, + settings=settings) + + @defer.inlineCallbacks + def test_export_items_json_field_names(self): + items = [{'foo': 'bar'}] header = OrderedDict(( ("foo", "Foo"), - ("baz", "Baz"), - ("hello", "Hello"), )) + rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} - rows = [ - {'Foo': 'bar1', 'Baz': '', 'Hello': ''}, - {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'}, - {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''}, - {'Foo': '', 'Baz': '', 'Hello': 'world4'}, - ] yield self.assertExported(items, list(header.values()), rows, - settings=settings, ordered=True) + settings=settings) @defer.inlineCallbacks def test_export_dicts(self): @@ -697,7 +696,7 @@ class FeedExportTest(unittest.TestCase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] yield self.assertExported(items, ['foo', 'baz', 'egg'], rows, - settings=settings, ordered=True) + settings=settings) # export a subset of columns settings = {'FEED_EXPORT_FIELDS': 'egg,baz'} @@ -706,7 +705,7 @@ class FeedExportTest(unittest.TestCase): {'egg': 'spam2', 'baz': 'quux2'} ] yield self.assertExported(items, ['egg', 'baz'], rows, - settings=settings, ordered=True) + settings=settings) @defer.inlineCallbacks def test_export_encoding(self): From 4605c66a80dabd64924e397580224a667cd73ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 May 2020 12:38:51 +0200 Subject: [PATCH 017/237] Fix AttributeError --- scrapy/utils/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 376c1f992..6a6d38a5c 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -114,7 +114,7 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) - out.setdefault("fields", settings.settings.getdictorlist("FEED_EXPORT_FIELDS") or None) + out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: From 8b84a65a6b2d391fdd9c49426d748751a03351f0 Mon Sep 17 00:00:00 2001 From: drs-11 Date: Tue, 25 Aug 2020 00:30:17 +0530 Subject: [PATCH 018/237] cleaned up code relating to issue #3689 --- scrapy/pipelines/images.py | 14 ++----- tests/test_pipeline_images.py | 72 ++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index f709c5057..e265685fb 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -6,6 +6,7 @@ See documentation in topics/media-pipeline.rst import functools import hashlib import six +import warnings try: from cStringIO import StringIO as BytesIO @@ -19,6 +20,7 @@ from scrapy.utils.python import to_bytes, get_func_args from scrapy.http import Request from scrapy.settings import Settings from scrapy.exceptions import DropItem +from scrapy.exceptions import ScrapyDeprecationWarning #TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException, FilesPipeline @@ -132,8 +134,6 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) @@ -153,9 +153,7 @@ class ImagesPipeline(FilesPipeline): yield thumb_path, thumb_image, thumb_buf def convert_image(self, image, size=None, response_body=None): - if not response_body: - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings + if response_body is None: warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=1) @@ -175,7 +173,7 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - elif response_body and image.format == 'JPEG': + elif response_body is not None and image.format == 'JPEG': return image, response_body buf = BytesIO() @@ -193,8 +191,6 @@ class ImagesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None): ## start of deprecation warning block (can be removed in the future) def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, ' 'please use file_path(request, response=None, info=None) instead', category=ScrapyDeprecationWarning, stacklevel=1) @@ -221,8 +217,6 @@ class ImagesPipeline(FilesPipeline): def thumb_path(self, request, thumb_id, response=None, info=None): ## start of deprecation warning block (can be removed in the future) def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use ' 'thumb_path(request, thumb_id, response=None, info=None) instead', category=ScrapyDeprecationWarning, stacklevel=1) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index ec0c87264..915b6a579 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -75,41 +75,7 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') - def test_convert_image(self): - # tests for old API - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - SIZE = (100, 100) - # straigh forward case: RGB and JPEG - COLOUR = (0, 127, 255) - im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) - - # check that thumbnail keep image ratio - thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEqual(thumbnail.mode, 'RGB') - self.assertEqual(thumbnail.size, (10, 10)) - - # transparency case: RGBA and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # transparency case with palette: P and PNG - COLOUR = (0, 127, 255, 50) - im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) - im = im.convert('P') - converted, _ = self.pipeline.convert_image(im) - self.assertEqual(converted.mode, 'RGB') - self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - - # ensure that we recieved deprecation warnings - self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) - + def test_convert_image_new(self): # tests for new API SIZE = (100, 100) # straigh forward case: RGB and JPEG @@ -207,6 +173,42 @@ class DeprecatedImagesPipelineTestCase(unittest.TestCase): self.assertEqual(len(w), 1) self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message)) + def test_overriden_convert_image_method(self): + self.init_pipeline(ImagesPipeline) + # tests for old API + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + SIZE = (100, 100) + # straigh forward case: RGB and JPEG + COLOUR = (0, 127, 255) + im, _ = _create_image('JPEG', 'RGB', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) + + # check that thumbnail keep image ratio + thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) + + # transparency case: RGBA and PNG + COLOUR = (0, 127, 255, 50) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im, _ = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) + + # ensure that we recieved deprecation warnings + self.assertTrue(len([warning for warning in w if 'ImagesPipeline.convert_image() method called in a deprecated way' in str(warning.message)]) == 4) + def tearDown(self): rmtree(self.tempdir) From ecec5f9e5171568c4ead64336245a1bb3be2ecfe Mon Sep 17 00:00:00 2001 From: drs-11 Date: Tue, 25 Aug 2020 02:46:44 +0530 Subject: [PATCH 019/237] Cleaned up code --- scrapy/pipelines/images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index d7f437adc..47d688c62 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -12,7 +12,7 @@ from io import BytesIO from itemadapter import ItemAdapter from PIL import Image -from scrapy.exceptions import DropItem,ScrapyDeprecationWarning +from scrapy.exceptions import DropItem, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline @@ -175,7 +175,7 @@ class ImagesPipeline(FilesPipeline): image.thumbnail(size, Image.ANTIALIAS) elif response_body is not None and image.format == 'JPEG': return image, response_body - + buf = BytesIO() image.save(buf, 'JPEG') return image, buf From 6565adc471360c2f542392befb7bb9448e8f171f Mon Sep 17 00:00:00 2001 From: drs-11 Date: Wed, 2 Sep 2020 20:44:26 +0530 Subject: [PATCH 020/237] added test case for get_images --- scrapy/pipelines/images.py | 6 +-- tests/test_pipeline_images.py | 75 ++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 47d688c62..09194a0fe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -18,7 +18,7 @@ from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings from scrapy.utils.misc import md5sum -from scrapy.utils.python import to_bytes, get_func_args +from scrapy.utils.python import get_func_args, to_bytes class NoimagesDrop(DropItem): @@ -136,7 +136,7 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image: warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) + category=ScrapyDeprecationWarning) if self._deprecated_convert_image: image, buf = self.convert_image(orig_image) @@ -156,7 +156,7 @@ class ImagesPipeline(FilesPipeline): if response_body is None: warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', - category=ScrapyDeprecationWarning, stacklevel=1) + category=ScrapyDeprecationWarning, stacklevel=2) if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 6b71b64bc..8e98b8734 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,6 +5,7 @@ import warnings from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +from unittest.mock import patch import attr from itemadapter import ItemAdapter @@ -12,7 +13,7 @@ from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImagesPipeline +from scrapy.pipelines.images import ImageException, ImagesPipeline from scrapy.settings import Settings from scrapy.utils.python import to_bytes @@ -93,6 +94,78 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_get_images_exception(self): + self.pipeline.min_width = 100 + self.pipeline.min_height = 100 + + _, buf1 = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + _, buf2 = _create_image('JPEG', 'RGB', (150, 50), (0, 0, 0)) + _, buf3 = _create_image('JPEG', 'RGB', (50, 150), (0, 0, 0)) + + resp1 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf1.getvalue()) + resp2 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf2.getvalue()) + resp3 = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf3.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp1, request=req, info=object())) + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp2, request=req, info=object())) + with self.assertRaises(ImageException): + next(self.pipeline.get_images(response=resp3, request=req, info=object())) + + def test_get_images_new(self): + self.pipeline.min_width = 0 + self.pipeline.min_height = 0 + self.pipeline.thumbs = {'small': (20, 20)} + + orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + orig_thumb, orig_thumb_buf = _create_image('JPEG', 'RGB', (20, 20), (0, 0, 0)) + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + + path, new_im, new_buf = next(get_images_gen) + self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im, new_im) + self.assertEqual(buf.getvalue(), new_buf.getvalue()) + + thumb_path, thumb_img, thumb_buf = next(get_images_gen) + self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(thumb_img, thumb_img) + self.assertEqual(orig_thumb_buf.getvalue(), thumb_buf.getvalue()) + + def test_get_images_old(self): + self.pipeline.thumbs = {'small': (20, 20)} + orig_im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + def overridden_convert_image(image, size=None): + im, buf = _create_image('JPEG', 'RGB', (50, 50), (0, 0, 0)) + return im, buf + + with patch.object(self.pipeline, 'convert_image', overridden_convert_image): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + get_images_gen = self.pipeline.get_images(response=resp, request=req, info=object()) + path, new_im, new_buf = next(get_images_gen) + self.assertEqual(path, 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im.mode, new_im.mode) + self.assertEqual(orig_im.getcolors(), new_im.getcolors()) + self.assertEqual(buf.getvalue(), new_buf.getvalue()) + + thumb_path, thumb_img, thumb_buf = next(get_images_gen) + self.assertEqual(thumb_path, 'thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') + self.assertEqual(orig_im.mode, thumb_img.mode) + self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) + self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) + + expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + 'overriden method does not accept response_body argument.') + self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) + def test_convert_image_old(self): # tests for old API with warnings.catch_warnings(record=True) as w: From 0c24cdb2573958cc0fb9127c6f195d3174640ff4 Mon Sep 17 00:00:00 2001 From: D R Siddhartha Date: Sun, 4 Oct 2020 02:09:21 +0530 Subject: [PATCH 021/237] Improved warning messages a little --- scrapy/pipelines/images.py | 4 ++-- tests/test_pipeline_images.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index d3254bc20..48e8a7b83 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -135,7 +135,7 @@ class ImagesPipeline(FilesPipeline): if self._deprecated_convert_image is None: self._deprecated_convert_image = 'response_body' not in get_func_args(self.convert_image) if self._deprecated_convert_image: - warnings.warn('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + warnings.warn(f'{self.__class__.__name__}.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.', category=ScrapyDeprecationWarning) @@ -155,7 +155,7 @@ class ImagesPipeline(FilesPipeline): def convert_image(self, image, size=None, response_body=None): if response_body is None: - warnings.warn('ImagesPipeline.convert_image() method called in a deprecated way, ' + warnings.warn(f'{self.__class__.__name__}.convert_image() method called in a deprecated way, ' 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=2) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 380c775c4..0a294db80 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -162,7 +162,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(orig_im.getcolors(), thumb_img.getcolors()) self.assertEqual(buf.getvalue(), thumb_buf.getvalue()) - expected_warning_msg = ('ImagesPipeline.convert_image() method overriden in a deprecated way, ' + expected_warning_msg = ('.convert_image() method overriden in a deprecated way, ' 'overriden method does not accept response_body argument.') self.assertEqual(len([warning for warning in w if expected_warning_msg in str(warning.message)]), 1) @@ -199,7 +199,7 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # ensure that we recieved deprecation warnings - expected_warning_msg = 'ImagesPipeline.convert_image() method called in a deprecated way' + expected_warning_msg = '.convert_image() method called in a deprecated way' self.assertTrue(len([warning for warning in w if expected_warning_msg in str(warning.message)]) == 4) def test_convert_image_new(self): From 76abcedaf4f31a7c76de9e19680dd7499d9eccf4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 13:36:25 +0500 Subject: [PATCH 022/237] Add as_async_generator. --- scrapy/utils/asyncgen.py | 12 ++++++++++++ tests/test_utils_asyncgen.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/test_utils_asyncgen.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 7f697af5f..db2173f85 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,5 +1,17 @@ +import collections + + async def collect_asyncgen(result): results = [] async for x in result: results.append(x) return results + + +async def as_async_generator(it): + if isinstance(it, collections.abc.AsyncIterator): + async for r in it: + yield r + else: + for r in it: + yield r diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py new file mode 100644 index 000000000..9ae66c57c --- /dev/null +++ b/tests/test_utils_asyncgen.py @@ -0,0 +1,20 @@ +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f + + +class AsyncgenUtilsTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_as_async_generator(self): + ag = as_async_generator(range(42)) + results = [] + async for i in ag: + results.append(i) + self.assertEqual(results, list(range(42))) + + @deferred_f_from_coro_f + async def test_collect_asyncgen(self): + ag = as_async_generator(range(42)) + results = await collect_asyncgen(ag) + self.assertEqual(results, list(range(42))) From acff1eb4960940eb360f3cf00d5b33ed71acc351 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 14:36:38 +0500 Subject: [PATCH 023/237] Add aiter_errback. --- scrapy/utils/defer.py | 14 ++++++++++++++ tests/test_utils_defer.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6db9cc117..2d02c0621 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -124,6 +124,20 @@ def iter_errback(iterable, errback, *a, **kw): errback(failure.Failure(), *a, **kw) +async def aiter_errback(aiterable, errback, *a, **kw): + """Wraps an async iterable calling an errback if an error is caught while + iterating it. Similar to scrapy.utils.defer.iter_errback() + """ + it = aiterable.__aiter__() + while True: + try: + yield await it.__anext__() + except StopAsyncIteration: + break + except Exception: + errback(failure.Failure(), *a, **kw) + + def deferred_from_coro(o): """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, defer.Deferred): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index e60242a3b..06d91c574 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -2,12 +2,15 @@ from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import ( iter_errback, + aiter_errback, mustbe_deferred, process_chain, process_chain_both, process_parallel, + deferred_f_from_coro_f, ) @@ -117,3 +120,31 @@ class IterErrbackTest(unittest.TestCase): self.assertEqual(out, [0, 1, 2, 3, 4]) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0].value, ZeroDivisionError) + + +class AiterErrbackTest(unittest.TestCase): + + @deferred_f_from_coro_f + async def test_aiter_errback_good(self): + async def itergood(): + for x in range(10): + yield x + + errors = [] + out = await collect_asyncgen(aiter_errback(itergood(), errors.append)) + self.assertEqual(out, list(range(10))) + self.assertFalse(errors) + + @deferred_f_from_coro_f + async def test_iter_errback_bad(self): + async def iterbad(): + for x in range(10): + if x == 5: + 1 / 0 + yield x + + errors = [] + out = await collect_asyncgen(aiter_errback(iterbad(), errors.append)) + self.assertEqual(out, [0, 1, 2, 3, 4]) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0].value, ZeroDivisionError) From 7e9f498e00fd36c76ac139eb285526c9e70b4054 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 14:37:27 +0500 Subject: [PATCH 024/237] Add MutableAsyncChain. --- scrapy/utils/python.py | 25 ++++++++++++++++ tests/test_utils_python.py | 58 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5703fd4c3..0bf9bff70 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -12,6 +12,7 @@ from functools import partial, wraps from itertools import chain from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.decorators import deprecated @@ -355,3 +356,27 @@ class MutableChain: @deprecated("scrapy.utils.python.MutableChain.__next__") def next(self): return self.__next__() + + +async def _async_chain(*iterables): + for it in iterables: + async for o in as_async_generator(it): + yield o + + +class MutableAsyncChain: + """ + Similar to MutableChain but for async iterables + """ + + def __init__(self, *args): + self.data = _async_chain(*args) + + def extend(self, *aiterables): + self.data = _async_chain(self.data, _async_chain(*aiterables)) + + def __aiter__(self): + return self + + async def __anext__(self): + return await self.data.__anext__() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3115cc92f..58b384591 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -2,15 +2,18 @@ import functools import gc import operator import platform -import unittest from datetime import datetime from itertools import count from warnings import catch_warnings +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, get_func_args, to_bytes, to_unicode, - without_none_values, MutableChain) + without_none_values, MutableChain, MutableAsyncChain) __doctests__ = ['scrapy.utils.python'] @@ -32,6 +35,57 @@ class MutableChainTest(unittest.TestCase): self.assertEqual(list(m), list(range(3, 13))) +class MutableAsyncChainTest(unittest.TestCase): + @staticmethod + async def g1(): + for i in range(3): + yield i + + @staticmethod + async def g2(): + return + yield + + @staticmethod + async def g3(): + for i in range(7, 10): + yield i + + @staticmethod + async def g4(): + for i in range(3, 5): + yield i + 1 / 0 + for i in range(5, 7): + yield i + + @staticmethod + async def collect_asyncgen_exc(asyncgen): + results = [] + async for x in asyncgen: + results.append(x) + return results + + @deferred_f_from_coro_f + async def test_mutableasyncchain(self): + m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7))) + m.extend(self.g2()) + m.extend(self.g3()) + + self.assertEqual(await m.__anext__(), 0) + results = await collect_asyncgen(m) + self.assertEqual(results, list(range(1, 10))) + + @deferred_f_from_coro_f + async def test_mutableasyncchain_exc(self): + m = MutableAsyncChain(self.g1()) + m.extend(self.g4()) + m.extend(self.g3()) + + results = await collect_asyncgen(aiter_errback(m, lambda _: None)) + self.assertEqual(results, list(range(5))) + + class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') From d658552f232547cc227e597b0d8489c4762eb9d2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 15:20:04 +0500 Subject: [PATCH 025/237] Add only_not_asyncio. --- conftest.py | 6 ++++++ pytest.ini | 1 + 2 files changed, 7 insertions(+) diff --git a/conftest.py b/conftest.py index 68b855c08..407bf9e62 100644 --- a/conftest.py +++ b/conftest.py @@ -55,5 +55,11 @@ def only_asyncio(request, reactor_pytest): pytest.skip('This test is only run with --reactor=asyncio') +@pytest.fixture(autouse=True) +def only_not_asyncio(request, reactor_pytest): + if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio': + pytest.skip('This test is only run without --reactor=asyncio') + + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/pytest.ini b/pytest.ini index d4deeb57c..416b228f9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,6 +21,7 @@ addopts = twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed + only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed flake8-max-line-length = 119 flake8-ignore = W503 From d66d52d3ed8aab0b4f126169c2855d00f4053907 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 15:21:12 +0500 Subject: [PATCH 026/237] Add process_iterable_helper. --- scrapy/utils/middlewares.py | 35 +++++++++++++ tests/test_utils_middlewares.py | 87 +++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 scrapy/utils/middlewares.py create mode 100644 tests/test_utils_middlewares.py diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py new file mode 100644 index 000000000..da28e0ddf --- /dev/null +++ b/scrapy/utils/middlewares.py @@ -0,0 +1,35 @@ +# coding: utf-8 +import inspect + + +def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): + for o in it: + if in_predicate and not in_predicate(o): + continue + if processor is not None: + o = processor(o) + if out_predicate and not out_predicate(o): + continue + yield o + + +async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): + async for o in it: + if in_predicate and not in_predicate(o): + continue + if processor is not None: + o = processor(o) + if out_predicate and not out_predicate(o): + continue + yield o + + +def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): + """ + For each item in the iterable: skips it if in_predicate is False, applies processor, + skips the result if out_predicate is False, else yields it. + """ + if inspect.isasyncgen(it): + return process_async_iterable_helper(it, in_predicate, out_predicate, processor) + else: + return process_normal_iterable_helper(it, in_predicate, out_predicate, processor) diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py new file mode 100644 index 000000000..d395ba1a9 --- /dev/null +++ b/tests/test_utils_middlewares.py @@ -0,0 +1,87 @@ +import collections + +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.middlewares import process_iterable_helper + + +def predicate1(o): + return bool(o % 2) + + +def predicate2(o): + return o < 10 + + +def processor(o): + return o * 2 + + +class ProcessIterableHelperNormalTest(unittest.TestCase): + + def test_normal_in_predicate(self): + iterable1 = iter([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [1, 3]) + + def test_normal_out_predicate(self): + iterable1 = iter([1, 2, 10, 3, 15]) + iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [1, 2, 3]) + + def test_normal_processor(self): + iterable1 = iter([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, processor=processor) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [2, 4, 6]) + + def test_normal_combined(self): + iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, + out_predicate=predicate2, processor=processor) + self.assertIsInstance(iterable2, collections.abc.Iterable) + list2 = list(iterable2) + self.assertEqual(list2, [2, 6]) + + +class ProcessIterableHelperAsyncTest(unittest.TestCase): + + @deferred_f_from_coro_f + async def test_async_in_predicate(self): + iterable1 = as_async_generator([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [1, 3]) + + @deferred_f_from_coro_f + async def test_async_out_predicate(self): + iterable1 = as_async_generator([1, 2, 10, 3, 15]) + iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [1, 2, 3]) + + @deferred_f_from_coro_f + async def test_async_processor(self): + iterable1 = as_async_generator([1, 2, 3]) + iterable2 = process_iterable_helper(iterable1, processor=processor) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [2, 4, 6]) + + @deferred_f_from_coro_f + async def test_async_combined(self): + iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15]) + iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, + out_predicate=predicate2, processor=processor) + self.assertIsInstance(iterable2, collections.abc.AsyncIterable) + list2 = await collect_asyncgen(iterable2) + self.assertEqual(list2, [2, 6]) From 92f2c9e308a5eda361229a8e74f74a21b9ff770a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 15:22:20 +0500 Subject: [PATCH 027/237] Move spider middlewares to process_iterable_helper. --- scrapy/spidermiddlewares/depth.py | 4 ++-- scrapy/spidermiddlewares/offsite.py | 32 ++++++++++++++------------- scrapy/spidermiddlewares/referer.py | 3 ++- scrapy/spidermiddlewares/urllength.py | 3 ++- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 776a6879a..73079bca9 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -3,10 +3,10 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ - import logging from scrapy.http import Request +from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -55,4 +55,4 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - return (r for r in result or () if _filter(r)) + return process_iterable_helper(result or (), in_predicate=_filter) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 6e4efda97..e7f481269 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -10,6 +10,7 @@ import warnings from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -26,21 +27,22 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - for x in result: - if isinstance(x, Request): - if x.dont_filter or self.should_follow(x, spider): - yield x - else: - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - else: - yield x + def in_predicate(x): + if not isinstance(x, Request): + return True + if x.dont_filter or self.should_follow(x, spider): + return True + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + return False + + return process_iterable_helper(result or (), in_predicate=in_predicate) def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..91c8727e1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,6 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.utils.middlewares import process_iterable_helper from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -337,7 +338,7 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return (_set_referer(r) for r in result or ()) + return process_iterable_helper(result or (), processor=_set_referer) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..c7359fecd 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,6 +8,7 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -34,4 +35,4 @@ class UrlLengthMiddleware: else: return True - return (r for r in result or () if _filter(r)) + return process_iterable_helper(result or (), in_predicate=_filter) From 2a1e9359caa0e16b336fdb2944ab7f4a69549896 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 16:16:29 +0500 Subject: [PATCH 028/237] Add parallel_async. --- scrapy/core/scraper.py | 24 +++++++-- scrapy/utils/defer.py | 110 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 0d3e3450f..3eae71af7 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,6 +1,6 @@ """This module implements the Scraper component which parses responses and extracts information from them""" - +import collections import logging from collections import deque @@ -12,7 +12,16 @@ from scrapy import signals from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy.http import Request, Response -from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel +from scrapy.utils.defer import ( + aiter_errback, + defer_fail, + defer_succeed, + deferred_from_coro, + iter_errback, + parallel, + parallel_async, +) + from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output @@ -180,9 +189,14 @@ class Scraper: def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) - it = iter_errback(result, self.handle_spider_error, request, response, spider) - dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, - request, response, spider) + if isinstance(result, collections.abc.AsyncIterable): + it = aiter_errback(result, self.handle_spider_error, request, response, spider) + dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider)) + else: + it = iter_errback(result, self.handle_spider_error, request, response, spider) + dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider) return dfd def _process_spidermw_output(self, output, request, response, spider): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 2d02c0621..bd5f9c8fc 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -75,6 +75,116 @@ def parallel(iterable, count, callable, *args, **named): return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) +class _AsyncCooperatorAdapter: + """ A class that wraps an async iterator into a normal iterator suitable + for using in Cooperator.coiterate(). As it's only needed for parallel_async(), + it calls the callable directly in the callback, instead of providing a more + generic interface. + + On the outside, this class behaves as an iterator that yields Deferreds. + Each Deferred is fired with the result of the callable which was called on + the next result from aiterator. It raises StopIteration when aiterator is + exhausted, as expected. + + Cooperator calls __next__() multiple times and waits on the Deferreds + returned from it. As async generators (since Python 3.8) don't support + awaiting on __anext__() several times in parallel, we need to serialize + this. It's done by storing the Deferreds returned from __next__() and + firing the oldest one when a result from __anext__() is available. + + The workflow: + 1. When __next__() is called for the first time, it creates a Deferred, stores it + in self.waiting_deferreds and returns it. It also makes a Deferred that will wait + for self.aiterator.__anext__() and puts it into self.anext_deferred. + 2. If __next__() is called again before self.anext_deferred fires, more Deferreds + are added to self.waiting_deferreds. + 3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both + clear self.anext_deferred. + 3.1. _callback() calls the callable passing the result value that it takes, pops a + Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it + chains those Deferreds so that the waiting Deferred will fire when the result + Deferred does, otherwise it fires it directly. This causes one awaiting task to + receive a result. If self.waiting_deferreds is still not empty, new __anext__() is + called and self.anext_deferred is populated. + 3.2. _errback() checks the exception class. If it's StopAsyncIteration it means + self.aiterator is exhausted and so it sets self.finished and fires all + self.waiting_deferreds. Other exceptions are propagated. + 4. If __next__() is called after __anext__() was handled, then if self.finished is + True, it raises StopIteration, otherwise it acts like in step 2, but if + self.anext_deferred is now empty is also populates it with a new __anext__(). + + Note that CooperativeTask ignores the value returned from the Deferred that it waits + for, so we fire them with None when needed. + + It may be possible to write an async iterator-aware replacement for + Cooperator/CooperativeTask and use it instead of this adapter to achieve the same + goal. + """ + def __init__(self, aiterator, callable, *callable_args, **callable_kwargs): + self.aiterator = aiterator + self.callable = callable + self.callable_args = callable_args + self.callable_kwargs = callable_kwargs + self.finished = False + self.waiting_deferreds = [] + self.anext_deferred = None + + def _callback(self, result): + # This gets called when the result from aiterator.__anext__() is available. + # It calls the callable on it and sends the result to the oldest waiting Deferred + # (by chaining if the result is a Deferred too or by firing if not). + self.anext_deferred = None + result = self.callable(result, *self.callable_args, **self.callable_kwargs) + d = self.waiting_deferreds.pop(0) + if d.called: + raise ValueError('Deferred in waiting_deferreds already called') + if isinstance(result, defer.Deferred): + result.chainDeferred(d) + else: + d.callback(None) + if self.waiting_deferreds: + self._call_anext() + + def _errback(self, failure): + # This gets called on any exceptions in aiterator.__anext__(). + # It handles StopAsyncIteration by stopping the iteration and reraises all others. + self.anext_deferred = None + failure.trap(StopAsyncIteration) + self.finished = True + for d in self.waiting_deferreds: + if d.called: + raise ValueError('Deferred in waiting_deferreds already called') + d.callback(None) + + def _call_anext(self): + # This starts waiting for the next result from aiterator. + # If aiterator is exhausted, _errback will be called. + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred.addCallbacks(self._callback, self._errback) + + def __iter__(self): + return self + + def __next__(self): + # This puts a new Deferred into self.waiting_deferreds and returns it. + # It also calls __anext__() if needed. + if self.finished: + raise StopIteration + d = defer.Deferred() + self.waiting_deferreds.append(d) + if not self.anext_deferred: + self._call_anext() + return d + + +def parallel_async(async_iterable, count, callable, *args, **named): + """ Like parallel but for async iterables """ + coop = task.Cooperator() + work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) + dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + return dl + + def process_chain(callbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() From 2152a2a50898b91a44e6b0c282b69b7bfb8d18f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 16:19:52 +0500 Subject: [PATCH 029/237] Add main infrastructure for async callbacks. --- scrapy/core/spidermw.py | 45 ++++++++++++++++----- scrapy/utils/defer.py | 19 +++++++++ scrapy/utils/spider.py | 8 ++-- scrapy/utils/test.py | 7 ++++ tests/spiders.py | 38 ++++++++++++++++- tests/test_crawl.py | 33 +++++++++++++++ tests/test_spidermiddleware_output_chain.py | 27 +++++++++++++ 7 files changed, 161 insertions(+), 16 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 763e0cdf6..961606f29 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +import inspect from itertools import islice from twisted.python.failure import Failure @@ -11,11 +12,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred -from scrapy.utils.python import MutableChain +from scrapy.utils.python import MutableAsyncChain, MutableChain def _isiterable(possible_iterator): - return hasattr(possible_iterator, '__iter__') + return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__') def _fname(f): @@ -58,15 +59,31 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(iterable, exception_processor_index, recover_to): - try: - for r in iterable: - yield r - except Exception as ex: + def _process_exception(ex): exception_result = process_spider_exception(Failure(ex), exception_processor_index) if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) + def _evaluate_normal_iterable(iterable): + try: + for r in iterable: + yield r + except Exception as ex: + _process_exception(ex) + + async def _evaluate_async_iterable(iterable): + try: + async for r in iterable: + yield r + except Exception as ex: + _process_exception(ex) + + if inspect.isasyncgen(iterable): + return _evaluate_async_iterable(iterable) + else: + return _evaluate_normal_iterable(iterable) + def process_spider_exception(_failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception @@ -92,7 +109,11 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_spider_output(result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered = MutableChain() + if inspect.isasyncgen(result): + iter_class = MutableAsyncChain + else: + iter_class = MutableChain + recovered = iter_class() method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): @@ -113,12 +134,16 @@ class SpiderMiddlewareManager(MiddlewareManager): f"iterable, got {type(result)}") raise _InvalidOutput(msg) - return MutableChain(result, recovered) + return iter_class(result, recovered) def process_callback_output(result): - recovered = MutableChain() + if inspect.isasyncgen(result): + iter_class = MutableAsyncChain + else: + iter_class = MutableChain + recovered = iter_class() result = _evaluate_iterable(result, 0, recovered) - return MutableChain(process_spider_output(result), recovered) + return iter_class(process_spider_output(result), recovered) dfd = mustbe_deferred(process_spider_input, response) dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bd5f9c8fc..554edc38c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -290,3 +290,22 @@ def maybeDeferred_coro(f, *args, **kw): return defer.fail(result) else: return defer.succeed(result) + + +def deferred_to_future(d): + """ Wraps a Deferred into a Future. Requires the asyncio reactor. + """ + return d.asFuture(asyncio.get_event_loop()) + + +def maybe_deferred_to_future(d): + """ Converts a Deferred to something that can be awaited in a callback or other user coroutine. + + If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be + awaited inside them. Otherwise, coroutines are wrapped into Deferreds and Deferreds can be awaited + directly inside them. + """ + if not is_asyncio_reactor_installed(): + return d + else: + return deferred_to_future(d) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 59fc9202f..d0fd1757d 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,7 +4,6 @@ import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -from scrapy.utils.asyncgen import collect_asyncgen logger = logging.getLogger(__name__) @@ -12,14 +11,13 @@ logger = logging.getLogger(__name__) def iterate_spider_output(result): if inspect.isasyncgen(result): - d = deferred_from_coro(collect_asyncgen(result)) - d.addCallback(iterate_spider_output) - return d + return result elif inspect.iscoroutine(result): d = deferred_from_coro(result) d.addCallback(iterate_spider_output) return d - return arg_to_iter(result) + else: + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 24c38283a..d8fc25094 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -110,3 +110,10 @@ def mock_google_cloud_storage(): bucket_mock.blob.return_value = blob_mock return (client_mock, bucket_mock, blob_mock) + + +def get_web_client_agent_req(url): + from twisted.internet import reactor + from twisted.web.client import Agent # imports twisted.internet.reactor + agent = Agent(reactor) + return agent.request(b'GET', url.encode('utf-8')) diff --git a/tests/spiders.py b/tests/spiders.py index 106392ea6..3e0ec001b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -14,7 +14,8 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.utils.test import get_from_asyncio_queue +from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future +from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req class MockServerSpider(Spider): @@ -148,6 +149,41 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenExcSpider(SimpleSpider): + name = 'asyncdef_asyncio_gen_exc' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + if i > 5: + raise ValueError("Stopping the processing") + + +class AsyncDefDeferredDirectSpider(SimpleSpider): + name = 'asyncdef_deferred_direct' + + async def parse(self, response): + resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200")) + yield {'code': resp.code} + + +class AsyncDefDeferredWrappedSpider(SimpleSpider): + name = 'asyncdef_deferred_wrapped' + + async def parse(self, response): + resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200"))) + yield {'code': resp.code} + + +class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider): + name = 'asyncdef_deferred_wrapped' + + async def parse(self, response): + resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200"))) + yield {'code': resp.code} + + class AsyncDefAsyncioGenSpider(SimpleSpider): name = 'asyncdef_asyncio_gen' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1083c1678..cda52f0d4 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -20,12 +20,16 @@ from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenExcSpider, AsyncDefAsyncioGenLoopSpider, AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, AsyncDefAsyncioSpider, + AsyncDefDeferredDirectSpider, + AsyncDefDeferredMaybeWrappedSpider, + AsyncDefDeferredWrappedSpider, AsyncDefSpider, BrokenStartRequestsSpider, BytesReceivedCallbackSpider, @@ -430,6 +434,18 @@ class CrawlSpiderTestCase(TestCase): for i in range(10): self.assertIn({'foo': i}, items) + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_exc(self): + log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) + log = str(log) + self.assertIn("Spider error processing", log) + self.assertIn("ValueError", log) + itemcount = stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 7) + for i in range(7): + self.assertIn({'foo': i}, items) + @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse_complex(self): @@ -449,6 +465,23 @@ class CrawlSpiderTestCase(TestCase): for req_id in range(3): self.assertIn(f"Got response 200, req_id {req_id}", str(log)) + @mark.only_not_asyncio() + @defer.inlineCallbacks + def test_async_def_deferred_direct(self): + _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) + self.assertEqual(items, [{'code': 200}]) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_deferred_wrapped(self): + log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) + self.assertEqual(items, [{'code': 200}]) + + @defer.inlineCallbacks + def test_async_def_deferred_maybe_wrapped(self): + _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + self.assertEqual(items, [{'code': 200}]) + @defer.inlineCallbacks def test_response_ssl_certificate_none(self): crawler = self.runner.create_crawler(SingleRequestSpider) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 029bf8bd6..4d1a7fcb0 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -43,6 +43,23 @@ class RecoverySpider(Spider): raise TabError() +class RecoveryAsyncGenSpider(RecoverySpider): + name = 'RecoveryAsyncGenSpider' + + async def parse(self, response): + for r in super().parse(response): + yield r + + +class RecoveryMiddleware: + def process_spider_exception(self, response, exception, spider): + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: @@ -307,6 +324,16 @@ class TestSpiderMiddleware(TestCase): self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks + def test_recovery_asyncgen(self): + """ + Same as test_recovery but with an async callback. + """ + log = yield self.crawl_log(RecoveryAsyncGenSpider) + self.assertIn("Middleware: TabError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) + self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks def test_process_spider_input_without_errback(self): """ From 58f848130145649a0191e98b182977278e2b9b6c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 16:20:32 +0500 Subject: [PATCH 030/237] Update docs. --- docs/topics/asyncio.rst | 24 ++++++++++++++++++++++++ docs/topics/coroutines.rst | 12 +++--------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 91e1cca0d..4addaa178 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -39,4 +39,28 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the use it instead of the default asyncio event loop. +.. _asyncio-await-dfd: +Awaiting on Deferreds +===================== + +When the asyncio reactor isn't installed, you can await on Deferreds in the +coroutines directly. When it is installed, this is not possible anymore, due to +specifics of the Scrapy coroutine integration (the coroutines are wrapped into +asyncio Futures, not into Deferreds directly), and you need to wrap them into +Futures. Scrapy provides two helpers for this: + +.. autofunction:: scrapy.utils.defer.deferred_to_future +.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +If you want to write universal code that works on any reactors, +you should use ``maybe_deferred_to_future`` on all Deferreds:: + + from scrapy.utils.defer import maybe_deferred_to_future + + class MySpider(Spider): + # ... + async def parse_with_deferred(self, response): + additional_response = await maybe_deferred_to_future(treq.get('https://additional.url')) + additional_data = await maybe_deferred_to_future(treq.content(additional_response)) + # ... use response and additional_data to yield items and requests diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3b1549bd3..279632653 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,15 +17,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. - .. note:: The callback output is not processed until the whole callback - finishes. - - As a side effect, if the callback raises an exception, none of its - output is processed. - - This is a known caveat of the current implementation that we aim to - address in a future version of Scrapy. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -92,6 +83,9 @@ This means you can use many useful Python libraries providing such code:: :mod:`asyncio` loop and to use them you need to :doc:`enable asyncio support in Scrapy`. +.. note:: If you want to ``await`` on Deferreds, you may need to + :ref:`wrap them`. + Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in callbacks, From 5cf403295d44bd2531ee3fe2f07057cf155e9804 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 5 Feb 2021 19:40:14 +0500 Subject: [PATCH 031/237] Remove a duplicate definition. --- tests/test_spidermiddleware_output_chain.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 4d1a7fcb0..088c14ca8 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -51,15 +51,6 @@ class RecoveryAsyncGenSpider(RecoverySpider): yield r -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class FailProcessSpiderInputMiddleware: From 67cff0e8a949a83220ae6d7d2c2ca40e6c8a8207 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 10 Feb 2021 22:44:14 +0500 Subject: [PATCH 032/237] Silence pylint "naked raise" error. --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 96392aae7..d3d7e5f8c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_result = self._process_spider_exception(response, spider, Failure(ex), exception_processor_index) if isinstance(exception_result, Failure): - raise + raise # pylint: disable=E0704 recover_to.extend(exception_result) def _evaluate_normal_iterable(iterable): From 40eab1d473a78369702490a901b5d304a40baf3b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 18 Feb 2021 19:56:12 +0500 Subject: [PATCH 033/237] Drop a duplicate import. --- tests/test_utils_defer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index d220f969b..62f6ff194 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -5,14 +5,13 @@ from twisted.python.failure import Failure from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.defer import ( + aiter_errback, deferred_f_from_coro_f, iter_errback, - aiter_errback, mustbe_deferred, process_chain, process_chain_both, process_parallel, - deferred_f_from_coro_f, ) From f9a5385146c741825d8c0b089cf8e8318dca9403 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 19 Feb 2021 21:19:21 +0500 Subject: [PATCH 034/237] Revert "Move spider middlewares to process_iterable_helper." This reverts commit 92f2c9e308a5eda361229a8e74f74a21b9ff770a. --- scrapy/spidermiddlewares/depth.py | 4 ++-- scrapy/spidermiddlewares/offsite.py | 32 +++++++++++++-------------- scrapy/spidermiddlewares/referer.py | 3 +-- scrapy/spidermiddlewares/urllength.py | 3 +-- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 73079bca9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -3,10 +3,10 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + import logging from scrapy.http import Request -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -55,4 +55,4 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - return process_iterable_helper(result or (), in_predicate=_filter) + return (r for r in result or () if _filter(r)) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index e7f481269..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -10,7 +10,6 @@ import warnings from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -27,22 +26,21 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - def in_predicate(x): - if not isinstance(x, Request): - return True - if x.dont_filter or self.should_follow(x, spider): - return True - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - return False - - return process_iterable_helper(result or (), in_predicate=in_predicate) + for x in result: + if isinstance(x, Request): + if x.dont_filter or self.should_follow(x, spider): + yield x + else: + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + else: + yield x def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 91c8727e1..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,7 +10,6 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.middlewares import process_iterable_helper from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -338,7 +337,7 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return process_iterable_helper(result or (), processor=_set_referer) + return (_set_referer(r) for r in result or ()) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index c7359fecd..5be1f80cb 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,7 +8,6 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -35,4 +34,4 @@ class UrlLengthMiddleware: else: return True - return process_iterable_helper(result or (), in_predicate=_filter) + return (r for r in result or () if _filter(r)) From c51ec1ae1cef3fd68fc512f0636b75d0f3a2da13 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Feb 2021 19:32:45 +0500 Subject: [PATCH 035/237] Drop process_iterable_helper, add _process_iterable_universal. --- scrapy/core/spidermw.py | 28 +++------ scrapy/spidermiddlewares/depth.py | 18 ++++-- scrapy/spidermiddlewares/offsite.py | 32 +++++----- scrapy/spidermiddlewares/referer.py | 9 ++- scrapy/spidermiddlewares/urllength.py | 9 ++- scrapy/utils/asyncgen.py | 45 ++++++++++++++ scrapy/utils/middlewares.py | 35 ----------- tests/test_utils_asyncgen.py | 22 ++++++- tests/test_utils_middlewares.py | 87 --------------------------- 9 files changed, 120 insertions(+), 165 deletions(-) delete mode 100644 scrapy/utils/middlewares.py delete mode 100644 tests/test_utils_middlewares.py diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d3d7e5f8c..33e215971 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -10,6 +10,7 @@ from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager +from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableAsyncChain, MutableChain @@ -57,31 +58,18 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(response, request, spider) def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to): - def _process_exception(ex): - exception_result = self._process_spider_exception(response, spider, Failure(ex), - exception_processor_index) - if isinstance(exception_result, Failure): - raise # pylint: disable=E0704 - recover_to.extend(exception_result) - - def _evaluate_normal_iterable(iterable): - try: - for r in iterable: - yield r - except Exception as ex: - _process_exception(ex) - + @_process_iterable_universal async def _evaluate_async_iterable(iterable): try: async for r in iterable: yield r except Exception as ex: - _process_exception(ex) - - if inspect.isasyncgen(iterable): - return _evaluate_async_iterable(iterable) - else: - return _evaluate_normal_iterable(iterable) + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + return _evaluate_async_iterable(iterable) def _process_spider_exception(self, response, spider, _failure, start_index=0): exception = _failure.value diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 776a6879a..973404b2b 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,6 +7,7 @@ See documentation in docs/topics/spider-middleware.rst import logging from scrapy.http import Request +from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -49,10 +50,15 @@ class DepthMiddleware: spider=spider) return True - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) + @_process_iterable_universal + async def process(result): + # base case (depth=0) + if 'depth' not in response.meta: + response.meta['depth'] = 0 + if self.verbose_stats: + self.stats.inc_value('request_depth_count/0', spider=spider) - return (r for r in result or () if _filter(r)) + async for r in result or (): + if _filter(r): + yield r + return process(result) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 6e4efda97..074ec7a4e 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,6 +9,7 @@ import warnings from scrapy import signals from scrapy.http import Request +from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) @@ -26,21 +27,24 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - for x in result: - if isinstance(x, Request): - if x.dont_filter or self.should_follow(x, spider): - yield x + @_process_iterable_universal + async def process(result): + async for x in result: + if isinstance(x, Request): + if x.dont_filter or self.should_follow(x, spider): + yield x + else: + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) else: - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - else: - yield x + yield x + return process(result) def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f81041376..8d862d1d0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,6 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals +from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -337,7 +338,13 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return (_set_referer(r) for r in result or ()) + + @_process_iterable_universal + async def process(result): + async for r in result or (): + yield _set_referer(r) + + return process(result) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..d40d43ff1 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,6 +8,7 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -34,4 +35,10 @@ class UrlLengthMiddleware: else: return True - return (r for r in result or () if _filter(r)) + @_process_iterable_universal + async def process(result): + async for r in result or (): + if _filter(r): + yield r + + return process(result) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index db2173f85..39c94ad8a 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,4 +1,6 @@ import collections +import functools +import inspect async def collect_asyncgen(result): @@ -15,3 +17,46 @@ async def as_async_generator(it): else: for r in it: yield r + + +# https://stackoverflow.com/a/66170760/113586 +def _process_iterable_universal(process_async): + """ Takes a function that takes an async iterable, args and kwargs. Returns + a function that takes any iterable, args and kwargs. + + Requires that process_async only awaits on the iterable and synchronous functions, + so it's better to use this only in the Scrapy code itself. + """ + + # If this stops working, all internal uses can be just replaced with manually-written + # process_sync functions. + + def process_sync(iterable, *args, **kwargs): + agen = process_async(as_async_generator(iterable), *args, **kwargs) + if not inspect.isasyncgen(agen): + raise ValueError(f"process_async returned wrong type {type(agen)}") + sent = None + while True: + try: + gen = agen.asend(sent) + gen.send(None) + except StopIteration as e: + sent = yield e.value + except StopAsyncIteration: + return + else: + gen.throw(RuntimeError, + f"Synchronously-called function '{process_async.__name__}' has blocked, " + f"you can't use {_process_iterable_universal.__name__} with it.") + + @functools.wraps(process_async) + def process(iterable, *args, **kwargs): + if inspect.isasyncgen(iterable): + # call process_async directly + return process_async(iterable, *args, **kwargs) + if hasattr(iterable, '__iter__'): + # convert process_async to process_sync + return process_sync(iterable, *args, **kwargs) + raise ValueError(f"Wrong iterable type {type(iterable)}") + + return process diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py deleted file mode 100644 index da28e0ddf..000000000 --- a/scrapy/utils/middlewares.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding: utf-8 -import inspect - - -def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - for o in it: - if in_predicate and not in_predicate(o): - continue - if processor is not None: - o = processor(o) - if out_predicate and not out_predicate(o): - continue - yield o - - -async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - async for o in it: - if in_predicate and not in_predicate(o): - continue - if processor is not None: - o = processor(o) - if out_predicate and not out_predicate(o): - continue - yield o - - -def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - """ - For each item in the iterable: skips it if in_predicate is False, applies processor, - skips the result if out_predicate is False, else yields it. - """ - if inspect.isasyncgen(it): - return process_async_iterable_helper(it, in_predicate, out_predicate, processor) - else: - return process_normal_iterable_helper(it, in_predicate, out_predicate, processor) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 9ae66c57c..2f4181d3d 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,6 +1,6 @@ from twisted.trial import unittest -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal from scrapy.utils.defer import deferred_f_from_coro_f @@ -18,3 +18,23 @@ class AsyncgenUtilsTest(unittest.TestCase): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) self.assertEqual(results, list(range(42))) + + +@_process_iterable_universal +async def process_iterable(iterable): + async for i in iterable: + yield i * 2 + + +class ProcessIterableUniversalTest(unittest.TestCase): + + def test_normal(self): + iterable = iter([1, 2, 3]) + results = list(process_iterable(iterable)) + self.assertEqual(results, [2, 4, 6]) + + @deferred_f_from_coro_f + async def test_async(self): + iterable = as_async_generator([1, 2, 3]) + results = await collect_asyncgen(process_iterable(iterable)) + self.assertEqual(results, [2, 4, 6]) diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py deleted file mode 100644 index d395ba1a9..000000000 --- a/tests/test_utils_middlewares.py +++ /dev/null @@ -1,87 +0,0 @@ -import collections - -from twisted.trial import unittest - -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.middlewares import process_iterable_helper - - -def predicate1(o): - return bool(o % 2) - - -def predicate2(o): - return o < 10 - - -def processor(o): - return o * 2 - - -class ProcessIterableHelperNormalTest(unittest.TestCase): - - def test_normal_in_predicate(self): - iterable1 = iter([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [1, 3]) - - def test_normal_out_predicate(self): - iterable1 = iter([1, 2, 10, 3, 15]) - iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [1, 2, 3]) - - def test_normal_processor(self): - iterable1 = iter([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, processor=processor) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [2, 4, 6]) - - def test_normal_combined(self): - iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, - out_predicate=predicate2, processor=processor) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [2, 6]) - - -class ProcessIterableHelperAsyncTest(unittest.TestCase): - - @deferred_f_from_coro_f - async def test_async_in_predicate(self): - iterable1 = as_async_generator([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [1, 3]) - - @deferred_f_from_coro_f - async def test_async_out_predicate(self): - iterable1 = as_async_generator([1, 2, 10, 3, 15]) - iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [1, 2, 3]) - - @deferred_f_from_coro_f - async def test_async_processor(self): - iterable1 = as_async_generator([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, processor=processor) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_async_combined(self): - iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, - out_predicate=predicate2, processor=processor) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [2, 6]) From a6034f942b034946538855cc53644b5a89ba081a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Mar 2021 22:53:52 +0500 Subject: [PATCH 036/237] Add tests for _AsyncCooperatorAdapter. --- tests/test_utils_defer.py | 68 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 62f6ff194..543bbee09 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -1,14 +1,18 @@ +import random + from pytest import mark from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure -from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator from scrapy.utils.defer import ( aiter_errback, deferred_f_from_coro_f, iter_errback, + maybe_deferred_to_future, mustbe_deferred, + parallel_async, process_chain, process_chain_both, process_parallel, @@ -164,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred_f_from_coro_f_xfail(self): raise Exception("This is expected to be raised") + + +class AsyncCooperatorTest(unittest.TestCase): + """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. + + parallel_async is called with the results of a callback (so an iterable of items, requests and None, + with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable + (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and + None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting. + + We want to test different concurrency values compared to the iterable length. + We also want to simulate the real usage, with arbitrary delays between getting the values + from the iterable. We also want to simulate sync and async results from the callable. + """ + CONCURRENT_ITEMS = 50 + + @staticmethod + def callable(o, results): + if random.random() < 0.4: + # simulate async processing + dfd = defer.Deferred() + dfd.addCallback(lambda _: results.append(o)) + delay = random.random() / 8 + reactor.callLater(delay, dfd.callback, None) + return dfd + else: + # simulate trivial sync processing + results.append(o) + + @staticmethod + def get_async_iterable(length): + # simulate a simple callback without delays between results + return as_async_generator(range(length)) + + @staticmethod + async def get_async_iterable_with_delays(length): + # simulate a callback with delays between some of the results + for i in range(length): + if random.random() < 0.1: + dfd = defer.Deferred() + delay = random.random() / 20 + reactor.callLater(delay, dfd.callback, None) + await maybe_deferred_to_future(dfd) + yield i + + @defer.inlineCallbacks + def test_simple(self): + for length in [20, 50, 100]: + results = [] + ait = self.get_async_iterable(length) + dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + yield dl + self.assertEqual(list(range(length)), sorted(results)) + + @defer.inlineCallbacks + def test_delays(self): + for length in [20, 50, 100]: + results = [] + ait = self.get_async_iterable_with_delays(length) + dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results) + yield dl + self.assertEqual(list(range(length)), sorted(results)) From 0596f2bf6e7fc404333345d1cece01e31fabed8e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 23 Mar 2021 22:47:48 +0500 Subject: [PATCH 037/237] Remove not needed deferred_from_coro call. --- scrapy/core/scraper.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 3eae71af7..7eedbf33e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,7 +16,6 @@ from scrapy.utils.defer import ( aiter_errback, defer_fail, defer_succeed, - deferred_from_coro, iter_errback, parallel, parallel_async, @@ -191,8 +190,8 @@ class Scraper: return defer_succeed(None) if isinstance(result, collections.abc.AsyncIterable): it = aiter_errback(result, self.handle_spider_error, request, response, spider) - dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output, - request, response, spider)) + dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider) else: it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, From a97dc55c714d90378dbc8a819cae1a3a40056d6e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 25 Mar 2021 17:37:32 +0500 Subject: [PATCH 038/237] Add/improve docs. --- docs/topics/asyncio.rst | 1 - docs/topics/coroutines.rst | 13 ++++++++++++- docs/topics/spider-middleware.rst | 17 +++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 4addaa178..18712c928 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -38,7 +38,6 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. - .. _asyncio-await-dfd: Awaiting on Deferreds diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 279632653..9b50d9312 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,6 +17,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.http.Request` callbacks. + .. versionchanged:: VERSION + Output of async callbacks is now processed asynchronously instead of collecting + all of it first. + - The :meth:`process_item` method of :ref:`item pipelines `. @@ -30,6 +34,13 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares `. + + .. versionadded:: VERSION + .. note:: This method needs to be an async generator, not just a coroutine that + returns an iterable. + Usage ===== @@ -76,7 +87,7 @@ This means you can use many useful Python libraries providing such code:: async def parse_with_asyncio(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: - additional_data = await r.text() + additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index fc114a63f..d09693c16 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -98,20 +98,25 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output(response, result, spider) + .. versionchanged:: VERSION + Since VERSION this can take and return an :term:`python:asynchronous + iterable`. + This method is called with the results returned from the Spider, after it has processed the response. - :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request` objects and :ref:`item object - `. + :meth:`process_spider_output` must return an iterable (normal or + asynchronous) of :class:`~scrapy.http.Request` objects and + :ref:`item objects `. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.http.Request` objects and - :ref:`item object ` + :type result: an iterable (normal or asynchronous) of + :class:`~scrapy.http.Request` objects and :ref:`item objects + ` :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.spiders.Spider` object @@ -122,7 +127,7 @@ object gives you access, for example, to the :ref:`settings `. method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` objects and :ref:`item object + iterable of :class:`~scrapy.http.Request` objects and :ref:`item objects `. If it returns ``None``, Scrapy will continue processing this exception, From f422861ef49e8d0e0c2aa4de937fb77b95e063ca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 25 Mar 2021 20:47:40 +0500 Subject: [PATCH 039/237] Add more tests for spider middlewares. --- tests/test_spidermiddleware.py | 188 +++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 78e926adc..2584dec21 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,11 +1,15 @@ +import collections.abc from unittest import mock +from twisted.internet import defer from twisted.trial.unittest import TestCase from twisted.python.failure import Failure from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput +from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager @@ -101,3 +105,187 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): result = self._scrape_response() self.assertIsInstance(result, Failure) self.assertIsInstance(result.value, ZeroDivisionError) + + +class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): + """ Helpers for testing sync, async and mixed middlewares. + + Should work for process_spider_output and, when it's supported, process_start_requests. + """ + + RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects + + @defer.inlineCallbacks + def _get_middleware_result(self, *mw_classes): + for mw_cls in mw_classes: + self.mwman._add_middleware(mw_cls()) + result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) + return result + + def assertAsyncGeneratorNotIterable(self, o): + with self.assertRaisesRegex(TypeError, + "'(async_generator|MutableAsyncChain)' object is not iterable"): + list(o) + + @defer.inlineCallbacks + def _test_simple_base(self, *mw_classes): + result = yield self._get_middleware_result(*mw_classes) + self.assertIsInstance(result, collections.abc.Iterable) + result_list = list(result) + self.assertEqual(len(result_list), self.RESULT_COUNT) + self.assertIsInstance(result_list[0], self.ITEM_TYPE) + + @defer.inlineCallbacks + def _test_asyncgen_base(self, *mw_classes): + result = yield self._get_middleware_result(*mw_classes) + self.assertIsInstance(result, collections.abc.AsyncIterator) + result_list = yield deferred_from_coro(collect_asyncgen(result)) + self.assertEqual(len(result_list), self.RESULT_COUNT) + self.assertIsInstance(result_list[0], self.ITEM_TYPE) + + @defer.inlineCallbacks + def _test_asyncgen_fail(self, *mw_classes): + result = yield self._get_middleware_result(*mw_classes) + self.assertIsInstance(result, collections.abc.Iterable) + self.assertAsyncGeneratorNotIterable(result) + + +class ProcessSpiderOutputSimpleMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + yield r + + +class ProcessSpiderOutputAsyncGenMiddleware: + async def process_spider_output(self, response, result, spider): + async for r in as_async_generator(result): + yield r + + +class ProcessSpiderOutputUniversalMiddleware: + def process_spider_output(self, response, result, spider): + @_process_iterable_universal + async def process(result): + async for r in result: + yield r + return process(result) + + +class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): + """ process_spider_output tests for simple callbacks""" + + ITEM_TYPE = dict + MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware + MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware + + def _scrape_func(self, *args, **kwargs): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + + def test_simple(self): + """ Simple mw """ + return self._test_simple_base(self.MW_SIMPLE) + + def test_asyncgen(self): + """ Asyncgen mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN) + + def test_simple_asyncgen(self): + """ Simple mw -> asyncgen mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_SIMPLE) + + def test_asyncgen_simple(self): + """ Asyncgen mw -> simple mw; cannot work """ + return self._test_asyncgen_fail(self.MW_SIMPLE, + self.MW_ASYNCGEN) + + def test_universal(self): + """ Universal mw """ + return self._test_simple_base(self.MW_UNIVERSAL) + + def test_universal_simple(self): + """ Universal mw -> simple mw """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_UNIVERSAL) + + def test_simple_universal(self): + """ Simple mw -> universal mw """ + return self._test_simple_base(self.MW_UNIVERSAL, + self.MW_SIMPLE) + + def test_universal_asyncgen(self): + """ Universal mw -> asyncgen mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_UNIVERSAL) + + def test_asyncgen_universal(self): + """ Asyncgen mw -> universal mw """ + return self._test_asyncgen_base(self.MW_UNIVERSAL, + self.MW_ASYNCGEN) + + +class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): + """ process_spider_output tests for async generator callbacks """ + + async def _scrape_func(self, *args, **kwargs): + for item in super()._scrape_func(): + yield item + + def test_simple(self): + """ Simple mw; cannot work """ + return self._test_asyncgen_fail(self.MW_SIMPLE) + + @defer.inlineCallbacks + def test_simple_asyncgen(self): + """ Simple mw -> asyncgen mw; cannot work """ + result = yield self._get_middleware_result( + self.MW_ASYNCGEN, + self.MW_SIMPLE) + self.assertIsInstance(result, collections.abc.AsyncIterable) + self.assertAsyncGeneratorNotIterable(result) + + def test_universal(self): + """ Universal mw """ + return self._test_asyncgen_base(self.MW_UNIVERSAL) + + def test_universal_simple(self): + """ Universal mw -> simple mw; cannot work """ + return self._test_asyncgen_fail(self.MW_SIMPLE, + self.MW_UNIVERSAL) + + def test_simple_universal(self): + """ Simple mw -> universal mw; cannot work """ + return self._test_asyncgen_fail(self.MW_UNIVERSAL, + self.MW_SIMPLE) + + +class ProcessStartRequestsSimpleMiddleware: + def process_start_requests(self, start_requests, spider): + for r in start_requests: + yield r + + +class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): + """ process_start_requests tests for simple start_requests""" + + ITEM_TYPE = Request + MW_SIMPLE = ProcessStartRequestsSimpleMiddleware + + def _start_requests(self): + for i in range(3): + yield Request(f'https://example.com/{i}', dont_filter=True) + + @defer.inlineCallbacks + def _get_middleware_result(self, *mw_classes): + for mw_cls in mw_classes: + self.mwman._add_middleware(mw_cls()) + start_requests = iter(self._start_requests()) + results = yield self.mwman.process_start_requests(start_requests, self.spider) + return results + + def test_simple(self): + """ Simple mw """ + self._test_simple_base(self.MW_SIMPLE) From 0638d6f01c41f12311b21f06fee5c71f5d08569f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 25 Mar 2021 21:58:29 +0500 Subject: [PATCH 040/237] Fix handling middlewares that change sync iterables into async. --- scrapy/core/spidermw.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 33e215971..230332673 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -import inspect +import collections.abc from itertools import islice from twisted.python.failure import Failure @@ -96,11 +96,10 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - if inspect.isasyncgen(result): - iter_class = MutableAsyncChain + if isinstance(result, collections.abc.AsyncIterator): + recovered = MutableAsyncChain() else: - iter_class = MutableChain - recovered = iter_class() + recovered = MutableChain() method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): @@ -121,16 +120,23 @@ class SpiderMiddlewareManager(MiddlewareManager): f"iterable, got {type(result)}") raise _InvalidOutput(msg) - return iter_class(result, recovered) + # check this again as the middlewares could change "result" from sync to async + if isinstance(result, collections.abc.AsyncIterator): + return MutableAsyncChain(result, recovered) + else: + return MutableChain(result, recovered) def _process_callback_output(self, response, spider, result): - if inspect.isasyncgen(result): - iter_class = MutableAsyncChain + if isinstance(result, collections.abc.AsyncIterator): + recovered = MutableAsyncChain() else: - iter_class = MutableChain - recovered = iter_class() + recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) - return iter_class(self._process_spider_output(response, spider, result), recovered) + result = self._process_spider_output(response, spider, result) + if isinstance(result, collections.abc.AsyncIterator): + return MutableAsyncChain(result, recovered) + else: + return MutableChain(result, recovered) def scrape_response(self, scrape_func, response, request, spider): def process_callback_output(result): From b5f501df7bc3917af3d144bb6556a763f381cd7e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Mar 2021 20:17:41 +0500 Subject: [PATCH 041/237] Remove some unneeded code from _AsyncCooperatorAdapter. --- scrapy/utils/defer.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 554edc38c..ca3d79fa6 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -136,8 +136,6 @@ class _AsyncCooperatorAdapter: self.anext_deferred = None result = self.callable(result, *self.callable_args, **self.callable_kwargs) d = self.waiting_deferreds.pop(0) - if d.called: - raise ValueError('Deferred in waiting_deferreds already called') if isinstance(result, defer.Deferred): result.chainDeferred(d) else: @@ -152,8 +150,6 @@ class _AsyncCooperatorAdapter: failure.trap(StopAsyncIteration) self.finished = True for d in self.waiting_deferreds: - if d.called: - raise ValueError('Deferred in waiting_deferreds already called') d.callback(None) def _call_anext(self): @@ -162,9 +158,6 @@ class _AsyncCooperatorAdapter: self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __iter__(self): - return self - def __next__(self): # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. From 6803779eb7e408b275da62f670aba9deaf3eade9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 26 Mar 2021 22:29:07 +0500 Subject: [PATCH 042/237] Add more tests for _process_iterable_universal. --- scrapy/utils/asyncgen.py | 2 +- tests/test_utils_asyncgen.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 39c94ad8a..a79552f76 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -57,6 +57,6 @@ def _process_iterable_universal(process_async): if hasattr(iterable, '__iter__'): # convert process_async to process_sync return process_sync(iterable, *args, **kwargs) - raise ValueError(f"Wrong iterable type {type(iterable)}") + raise TypeError(f"Wrong iterable type {type(iterable)}") return process diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 2f4181d3d..41993a934 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -2,6 +2,7 @@ from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.test import get_web_client_agent_req class AsyncgenUtilsTest(unittest.TestCase): @@ -26,6 +27,13 @@ async def process_iterable(iterable): yield i * 2 +@_process_iterable_universal +async def process_iterable_awaiting(iterable): + async for i in iterable: + yield i * 2 + await get_web_client_agent_req('http://example.com') + + class ProcessIterableUniversalTest(unittest.TestCase): def test_normal(self): @@ -38,3 +46,22 @@ class ProcessIterableUniversalTest(unittest.TestCase): iterable = as_async_generator([1, 2, 3]) results = await collect_asyncgen(process_iterable(iterable)) self.assertEqual(results, [2, 4, 6]) + + @deferred_f_from_coro_f + async def test_blocking(self): + iterable = [1, 2, 3] + with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"): + list(process_iterable_awaiting(iterable)) + + def test_invalid_iterable(self): + with self.assertRaisesRegex(TypeError, "Wrong iterable type"): + process_iterable(None) + + @deferred_f_from_coro_f + async def test_invalid_process(self): + @_process_iterable_universal + def process_iterable_invalid(iterable): + pass + + with self.assertRaisesRegex(ValueError, "process_async returned wrong type"): + list(process_iterable_invalid([])) From 849472535ef9490e8cc2a62cc7f1835b2bc3eaba Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 2 Apr 2021 20:20:35 +0500 Subject: [PATCH 043/237] Update docs. --- docs/topics/coroutines.rst | 65 +++++++++++++++++++++++++++++++ docs/topics/spider-middleware.rst | 14 ++++--- scrapy/utils/asyncgen.py | 1 + 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 9b50d9312..6a39dcb5e 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -108,3 +108,68 @@ Common use cases for asynchronous code include: :ref:`the screenshot pipeline example`). .. _aio-libs: https://github.com/aio-libs + +.. _async-spider-middlewares: + +Asynchronous spider middlewares +=============================== + +.. versionadded:: VERSION +.. note:: This currently applies to + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`. + +Middleware methods discussed here can take and return async iterables. They can +return the same type of iterable or they can take a normal one and return an +async one. If such method needs to return an async iterable it must be an async +generator, not just a coroutine that returns an iterable. + +.. autofunction:: scrapy.utils.asyncgen.as_async_generator + +In the simplest form that supports both sync and async input it can be written +like this:: + + from scrapy.utils.asyncgen import as_async_generator + + class ProcessSpiderOutputAsyncGenMiddleware: + async def process_spider_output(self, response, result, spider): + async for r in as_async_generator(result): + # ... do something with r + yield r + +If the middleware input (the callback result for ``process_spider_output``) is +an async iterable, all middlewares that process it must support it. The +built-in ones do, but the ones in your project and 3rd-party ones will need to +be updated to support it, as the code that expects a normal iterable will break +on an async one. If these middlewares receive an async iterable, they must +return one as well. On the other hand, if they receive a normal iterable, they +shouldn't break and ideally should return a normal iterable too. There can be +several possible implementations of this. + +The simplest one, always converting normal iterables to async ones, is provided +above. Because a result of a middleware method is passed to the same method of +the next middleware, it's only possible to mix middlewares with synchronous and +asynchronous implementations of the same method if all synchronous ones are +called first (which isn't always possible). + +Another option is to make separate methods for normal and async iterables and +choose one at run time:: + + from inspect import isasyncgen + + class ProcessSpiderOutputAsyncGenMiddleware: + def _normal_process_spider_output(self, response, result, spider): + # ... do something with normal result + + async def _async_process_spider_output(self, response, result, spider): + # ... do the same with async result + + def process_spider_output(self, response, result, spider): + if isasyncgen(result): + return self._async_process_spider_output(self, response, result, spider) + else: + return self._normal_process_spider_output(self, response, result, spider) + +If you are writing a middleware that you intend to publish or to use in many +projects, this is likely the best way to implement it. It may be possible to +extract common code from both methods to reduce code duplication, as in the +simplest case the only difference between them will be ``for`` vs ``async for``. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index d09693c16..d87b89292 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -105,18 +105,20 @@ object gives you access, for example, to the :ref:`settings `. This method is called with the results returned from the Spider, after it has processed the response. - :meth:`process_spider_output` must return an iterable (normal or - asynchronous) of :class:`~scrapy.http.Request` objects and - :ref:`item objects `. + :meth:`process_spider_output` must return an iterable of + :class:`~scrapy.http.Request` objects and :ref:`item objects + `. + + .. note:: When defined as a :ref:`coroutine `, this method needs + to be an async generator, not just return an iterable. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object :param result: the result returned by the spider - :type result: an iterable (normal or asynchronous) of - :class:`~scrapy.http.Request` objects and :ref:`item objects - ` + :type result: an iterable of :class:`~scrapy.http.Request` objects and + :ref:`item objects ` :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.spiders.Spider` object diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index a79552f76..6c0bb1d10 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -11,6 +11,7 @@ async def collect_asyncgen(result): async def as_async_generator(it): + """ Wraps an iterator (sync or async) into an async generator. """ if isinstance(it, collections.abc.AsyncIterator): async for r in it: yield r From 30ed7fa349214ad11b4d13c981cbd2fc63ddaf42 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 2 Apr 2021 22:20:56 +0500 Subject: [PATCH 044/237] Some cleanup, make sync middlewares fail earlier. --- scrapy/core/spidermw.py | 15 +++++++-------- tests/test_spidermiddleware.py | 18 ++++-------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 230332673..b24ccf6ae 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -20,10 +20,6 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__') -def _fname(f): - return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}" - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {_fname(method)} must return None " + msg = (f"Middleware {method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - if isinstance(result, collections.abc.AsyncIterator): + result_async = isinstance(result, collections.abc.AsyncIterator) + if result_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {_fname(method)} must return an " + msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) + if result_async and isinstance(result, collections.abc.Iterator): + raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") # check this again as the middlewares could change "result" from sync to async if isinstance(result, collections.abc.AsyncIterator): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2584dec21..0a6b96c0c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) return result - def assertAsyncGeneratorNotIterable(self, o): - with self.assertRaisesRegex(TypeError, - "'(async_generator|MutableAsyncChain)' object is not iterable"): - list(o) - @defer.inlineCallbacks def _test_simple_base(self, *mw_classes): result = yield self._get_middleware_result(*mw_classes) @@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): @defer.inlineCallbacks def _test_asyncgen_fail(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) - self.assertIsInstance(result, collections.abc.Iterable) - self.assertAsyncGeneratorNotIterable(result) + with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"): + yield self._get_middleware_result(*mw_classes) class ProcessSpiderOutputSimpleMiddleware: @@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): """ Simple mw; cannot work """ return self._test_asyncgen_fail(self.MW_SIMPLE) - @defer.inlineCallbacks def test_simple_asyncgen(self): """ Simple mw -> asyncgen mw; cannot work """ - result = yield self._get_middleware_result( - self.MW_ASYNCGEN, - self.MW_SIMPLE) - self.assertIsInstance(result, collections.abc.AsyncIterable) - self.assertAsyncGeneratorNotIterable(result) + return self._test_asyncgen_fail(self.MW_ASYNCGEN, + self.MW_SIMPLE) def test_universal(self): """ Universal mw """ From 7bd1d888d49d238622007e659e54af76e82bf1c1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 2 Apr 2021 23:06:29 +0500 Subject: [PATCH 045/237] More robust sync/async middleware mix checking. --- scrapy/core/spidermw.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b24ccf6ae..d0d292007 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -92,8 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output(self, response, spider, result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - result_async = isinstance(result, collections.abc.AsyncIterator) - if result_async: + last_result_async = isinstance(result, collections.abc.AsyncIterator) + if last_result_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -116,11 +116,11 @@ class SpiderMiddlewareManager(MiddlewareManager): msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) - if result_async and isinstance(result, collections.abc.Iterator): + if last_result_async and isinstance(result, collections.abc.Iterator): raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") + last_result_async = isinstance(result, collections.abc.AsyncIterator) - # check this again as the middlewares could change "result" from sync to async - if isinstance(result, collections.abc.AsyncIterator): + if last_result_async: return MutableAsyncChain(result, recovered) else: return MutableChain(result, recovered) From 61197d3dba53cd67c41e5d23e1f0c11a864f539b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Apr 2021 18:21:58 +0500 Subject: [PATCH 046/237] Add/update typing, cleanup iterator/iterable inconsistencies. --- scrapy/core/scraper.py | 7 +++--- scrapy/core/spidermw.py | 36 +++++++++++++++------------- scrapy/utils/asyncgen.py | 19 +++++++-------- scrapy/utils/defer.py | 52 ++++++++++++++++++++++++---------------- scrapy/utils/python.py | 14 +++++------ 5 files changed, 71 insertions(+), 57 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8ce57af2f..0630ce625 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,10 +1,8 @@ """This module implements the Scraper component which parses responses and extracts information from them""" -import collections import logging from collections import deque -from collections.abc import Iterable -from typing import Union +from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Union from itemadapter import is_item from twisted.internet import defer @@ -188,7 +186,8 @@ class Scraper: def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider): if not result: return defer_succeed(None) - if isinstance(result, collections.abc.AsyncIterable): + it: Union[Generator, AsyncGenerator] + if isinstance(result, AsyncIterable): it = aiter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 6fcbc7492..9dd6c462d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,9 +3,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -import collections.abc from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable +from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator from twisted.internet.defer import Deferred from twisted.python.failure import Failure @@ -61,8 +60,9 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable, - exception_processor_index: int, recover_to: MutableChain) -> Generator: + def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable], + exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain] + ) -> Union[Generator, AsyncGenerator]: @_process_iterable_universal async def _evaluate_async_iterable(iterable): try: @@ -100,11 +100,13 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def _process_spider_output(self, response: Response, spider: Spider, - result: Iterable, start_index: int = 0) -> MutableChain: + result: Union[Iterable, AsyncIterable], start_index: int = 0 + ) -> Union[MutableChain, MutableAsyncChain]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - last_result_async = isinstance(result, collections.abc.AsyncIterator) - if last_result_async: + recovered: Union[MutableChain, MutableAsyncChain] + last_result_is_async = isinstance(result, AsyncIterable) + if last_result_is_async: recovered = MutableAsyncChain() else: recovered = MutableChain() @@ -127,30 +129,32 @@ class SpiderMiddlewareManager(MiddlewareManager): msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) - if last_result_async and isinstance(result, collections.abc.Iterator): + if last_result_is_async and isinstance(result, Iterable): raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") - last_result_async = isinstance(result, collections.abc.AsyncIterator) + last_result_is_async = isinstance(result, AsyncIterable) - if last_result_async: + if last_result_is_async: return MutableAsyncChain(result, recovered) else: - return MutableChain(result, recovered) + return MutableChain(result, recovered) # type: ignore[arg-type] - def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain: - if isinstance(result, collections.abc.AsyncIterator): + def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: + recovered: Union[MutableChain, MutableAsyncChain] + if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = self._process_spider_output(response, spider, result) - if isinstance(result, collections.abc.AsyncIterator): + if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) else: - return MutableChain(result, recovered) + return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: - def process_callback_output(result: Iterable) -> MutableChain: + def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]: return self._process_callback_output(response, spider, result) def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 92118ddb9..ae9a79989 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,6 @@ -import collections import functools import inspect -from collections.abc import AsyncIterable +from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union async def collect_asyncgen(result: AsyncIterable): @@ -11,9 +10,9 @@ async def collect_asyncgen(result: AsyncIterable): return results -async def as_async_generator(it): - """ Wraps an iterator (sync or async) into an async generator. """ - if isinstance(it, collections.abc.AsyncIterator): +async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator: + """ Wraps an iterable (sync or async) into an async generator. """ + if isinstance(it, AsyncIterable): async for r in it: yield r else: @@ -22,7 +21,7 @@ async def as_async_generator(it): # https://stackoverflow.com/a/66170760/113586 -def _process_iterable_universal(process_async): +def _process_iterable_universal(process_async: Callable): """ Takes a function that takes an async iterable, args and kwargs. Returns a function that takes any iterable, args and kwargs. @@ -33,7 +32,7 @@ def _process_iterable_universal(process_async): # If this stops working, all internal uses can be just replaced with manually-written # process_sync functions. - def process_sync(iterable, *args, **kwargs): + def process_sync(iterable: Iterable, *args, **kwargs) -> Generator: agen = process_async(as_async_generator(iterable), *args, **kwargs) if not inspect.isasyncgen(agen): raise ValueError(f"process_async returned wrong type {type(agen)}") @@ -52,11 +51,11 @@ def _process_iterable_universal(process_async): f"you can't use {_process_iterable_universal.__name__} with it.") @functools.wraps(process_async) - def process(iterable, *args, **kwargs): - if inspect.isasyncgen(iterable): + def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]: + if isinstance(iterable, AsyncIterable): # call process_async directly return process_async(iterable, *args, **kwargs) - if hasattr(iterable, '__iter__'): + if isinstance(iterable, Iterable): # convert process_async to process_sync return process_sync(iterable, *args, **kwargs) raise TypeError(f"Wrong iterable type {type(iterable)}") diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index f81faf6dd..39c8a85e9 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -3,9 +3,21 @@ Helper functions for dealing with Twisted deferreds """ import asyncio import inspect -from collections.abc import Coroutine +from asyncio import Future from functools import wraps -from typing import Any, Callable, Generator, Iterable +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Callable, + Coroutine, + Generator, + Iterable, + Iterator, + List, + Optional, + Union +) from twisted.internet import defer from twisted.internet.defer import Deferred, DeferredList, ensureDeferred @@ -80,8 +92,8 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named) return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter: - """ A class that wraps an async iterator into a normal iterator suitable +class _AsyncCooperatorAdapter(Iterator): + """ A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more generic interface. @@ -125,30 +137,30 @@ class _AsyncCooperatorAdapter: Cooperator/CooperativeTask and use it instead of this adapter to achieve the same goal. """ - def __init__(self, aiterator, callable, *callable_args, **callable_kwargs): - self.aiterator = aiterator + def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs): + self.aiterator = aiterable.__aiter__() self.callable = callable self.callable_args = callable_args self.callable_kwargs = callable_kwargs self.finished = False - self.waiting_deferreds = [] - self.anext_deferred = None + self.waiting_deferreds: List[Deferred] = [] + self.anext_deferred: Optional[Deferred] = None - def _callback(self, result): + def _callback(self, result: Any) -> None: # This gets called when the result from aiterator.__anext__() is available. # It calls the callable on it and sends the result to the oldest waiting Deferred # (by chaining if the result is a Deferred too or by firing if not). self.anext_deferred = None result = self.callable(result, *self.callable_args, **self.callable_kwargs) d = self.waiting_deferreds.pop(0) - if isinstance(result, defer.Deferred): + if isinstance(result, Deferred): result.chainDeferred(d) else: d.callback(None) if self.waiting_deferreds: self._call_anext() - def _errback(self, failure): + def _errback(self, failure: Failure) -> None: # This gets called on any exceptions in aiterator.__anext__(). # It handles StopAsyncIteration by stopping the iteration and reraises all others. self.anext_deferred = None @@ -157,29 +169,29 @@ class _AsyncCooperatorAdapter: for d in self.waiting_deferreds: d.callback(None) - def _call_anext(self): + def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self): + def __next__(self) -> Deferred: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d = defer.Deferred() + d = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() return d -def parallel_async(async_iterable, count, callable, *args, **named): - """ Like parallel but for async iterables """ +def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList: + """ Like parallel but for async iterators """ coop = Cooperator() work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + dl = DeferredList([coop.coiterate(work) for _ in range(count)]) return dl @@ -232,7 +244,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator: errback(failure.Failure(), *a, **kw) -async def aiter_errback(aiterable, errback, *a, **kw): +async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator: """Wraps an async iterable calling an errback if an error is caught while iterating it. Similar to scrapy.utils.defer.iter_errback() """ @@ -290,13 +302,13 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: return defer.succeed(result) -def deferred_to_future(d): +def deferred_to_future(d: Deferred) -> Future: """ Wraps a Deferred into a Future. Requires the asyncio reactor. """ return d.asFuture(asyncio.get_event_loop()) -def maybe_deferred_to_future(d): +def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: """ Converts a Deferred to something that can be awaited in a callback or other user coroutine. If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8b823d174..d086347bc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,9 +8,9 @@ import re import sys import warnings import weakref -from collections.abc import Iterable from functools import partial, wraps from itertools import chain +from typing import AsyncIterable, Iterable, Union from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator @@ -345,7 +345,7 @@ class MutableChain(Iterable): def __init__(self, *args: Iterable): self.data = chain.from_iterable(args) - def extend(self, *iterables: Iterable): + def extend(self, *iterables: Iterable) -> None: self.data = chain(self.data, chain.from_iterable(iterables)) def __iter__(self): @@ -359,22 +359,22 @@ class MutableChain(Iterable): return self.__next__() -async def _async_chain(*iterables): +async def _async_chain(*iterables: Union[Iterable, AsyncIterable]): for it in iterables: async for o in as_async_generator(it): yield o -class MutableAsyncChain: +class MutableAsyncChain(AsyncIterable): """ Similar to MutableChain but for async iterables """ - def __init__(self, *args): + def __init__(self, *args: Union[Iterable, AsyncIterable]): self.data = _async_chain(*args) - def extend(self, *aiterables): - self.data = _async_chain(self.data, _async_chain(*aiterables)) + def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None: + self.data = _async_chain(self.data, _async_chain(*iterables)) def __aiter__(self): return self From 9db01a483c729369b272235bfb6e1c66ff62d1f7 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 14 Apr 2021 19:02:38 +0500 Subject: [PATCH 047/237] Update scrapy/core/spidermw.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/core/spidermw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 9dd6c462d..d1fedae07 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ from itertools import islice -from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator +from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union from twisted.internet.defer import Deferred from twisted.python.failure import Failure From 5bacc4822e80a78a2a287a5d1e417c0d3b7ddbdf Mon Sep 17 00:00:00 2001 From: Gustavo Bordin Date: Sun, 16 May 2021 18:40:30 -0300 Subject: [PATCH 048/237] changing dunder-str to dunder-repr --- scrapy/core/http2/stream.py | 6 ++---- scrapy/http/request/__init__.py | 4 +--- scrapy/http/response/__init__.py | 4 +--- scrapy/settings/__init__.py | 4 +--- scrapy/spiders/__init__.py | 4 +--- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index c2a4b702f..a3a9e5e1d 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -150,12 +150,10 @@ class Stream: self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) - - def __str__(self) -> str: + + def __repr__(self): return f'Stream(id={self.stream_id!r})' - __repr__ = __str__ - @property def _log_warnsize(self) -> bool: """Checks if we have received data which exceeds the download warnsize diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ad884feac..a1857c604 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -109,11 +109,9 @@ class Request(object_ref): def encoding(self): return self._encoding - def __str__(self): + def __repr__(self): return f"<{self.method} {self.url}>" - __repr__ = __str__ - def copy(self): """Return a copy of this Request""" return self.replace() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 185a9bb67..eb80c5214 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -87,11 +87,9 @@ class Response(object_ref): body = property(_get_body, obsolete_setter(_set_body, 'body')) - def __str__(self): + def __repr__(self): return f"<{self.status} {self.url}>" - __repr__ = __str__ - def copy(self): """Return a copy of this Response""" return self.replace() diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 1fe1e6fd1..69d0476eb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -51,11 +51,9 @@ class SettingsAttribute: self.value = value self.priority = priority - def __str__(self): + def __repr__(self): return f"" - __repr__ = __str__ - class BaseSettings(MutableMapping): """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c13ba4b3c..1c079ff27 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -106,11 +106,9 @@ class Spider(object_ref): if callable(closed): return closed(reason) - def __str__(self): + def __repr__(self): return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>" - __repr__ = __str__ - # Top-level imports from scrapy.spiders.crawl import CrawlSpider, Rule From 09a07c9b4ad02959e7416c25984fa715df7bd71f Mon Sep 17 00:00:00 2001 From: Gustavo Bordin Date: Tue, 18 May 2021 19:07:28 -0300 Subject: [PATCH 049/237] removed whitespaces --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a3a9e5e1d..780191505 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -150,7 +150,7 @@ class Stream: self.close(StreamCloseReason.CANCELLED) self._deferred_response = Deferred(_cancel) - + def __repr__(self): return f'Stream(id={self.stream_id!r})' From de69d967f90bc70bc07f734b93a4e7b73f2a4aa9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 11 Jun 2021 16:12:25 +0500 Subject: [PATCH 050/237] Fix async spider examples. --- docs/topics/coroutines.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 6a39dcb5e..67f1a4098 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -77,14 +77,16 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code:: - class MySpider(Spider): + class MySpiderDeferred(Spider): # ... - async def parse_with_deferred(self, response): + async def parse(self, response): additional_response = await treq.get('https://additional.url') additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests - async def parse_with_asyncio(self, response): + class MySpiderAsyncio(Spider): + # ... + async def parse(self, response): async with aiohttp.ClientSession() as session: async with session.get('https://additional.url') as additional_response: additional_data = await additional_response.text() From 7306a81188f81964ad85f4936ce29e3aa0084447 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 19 Jul 2021 20:09:11 +0500 Subject: [PATCH 051/237] Disable builtin middlewares in spider middleware tests. --- tests/test_spidermiddleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 0a6b96c0c..b0ca2f62e 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -19,7 +19,7 @@ class SpiderMiddlewareTestCase(TestCase): def setUp(self): self.request = Request('http://example.com/index.html') self.response = Response(self.request.url, request=self.request) - self.crawler = get_crawler(Spider) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}}) self.spider = self.crawler._create_spider('foo') self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) From e5b057cfd4472e970b9b51757a1b823b2f585b09 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Oct 2021 19:06:51 +0500 Subject: [PATCH 052/237] Don't use a HTTP request in a case that will not be awaited. --- tests/test_utils_asyncgen.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 41993a934..d9e6bc2eb 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,3 +1,4 @@ +from twisted.internet.defer import Deferred from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal @@ -31,7 +32,10 @@ async def process_iterable(iterable): async def process_iterable_awaiting(iterable): async for i in iterable: yield i * 2 - await get_web_client_agent_req('http://example.com') + d = Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, 42) + await d class ProcessIterableUniversalTest(unittest.TestCase): From a642b73e1a1ba355bae9d5b9d1e87a9da0696293 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 Oct 2021 19:21:49 +0500 Subject: [PATCH 053/237] Remove an unused import. --- tests/test_utils_asyncgen.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index d9e6bc2eb..7abe17c22 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -3,7 +3,6 @@ from twisted.trial import unittest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.test import get_web_client_agent_req class AsyncgenUtilsTest(unittest.TestCase): From f789547551ae0eb79f41c9de44525bf597a0ffa5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Jan 2022 18:28:32 +0500 Subject: [PATCH 054/237] Implement spider middleware iterable upgrade/downgrade. --- docs/topics/coroutines.rst | 104 ++++--- docs/topics/spider-middleware.rst | 9 + scrapy/core/scraper.py | 3 +- scrapy/core/spidermw.py | 135 +++++++-- scrapy/middleware.py | 7 +- scrapy/spidermiddlewares/depth.py | 71 ++--- scrapy/spidermiddlewares/offsite.py | 40 +-- scrapy/spidermiddlewares/referer.py | 22 +- scrapy/spidermiddlewares/urllength.py | 33 +-- scrapy/utils/asyncgen.py | 47 +-- scrapy/utils/python.py | 4 +- tests/test_spidermiddleware.py | 302 +++++++++++++++++--- tests/test_spidermiddleware_output_chain.py | 17 ++ tests/test_utils_asyncgen.py | 52 +--- 14 files changed, 553 insertions(+), 293 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index c514fc00a..073b6bd9a 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -35,11 +35,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. + method of :ref:`spider middlewares `. See + :ref:`async-spider-middlewares`. .. versionadded:: VERSION - .. note:: This method needs to be an async generator, not just a coroutine that - returns an iterable. Usage ===== @@ -106,8 +105,8 @@ Common use cases for asynchronous code include: * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see - :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` + (see :ref:`the screenshot pipeline example`). .. _aio-libs: https://github.com/aio-libs @@ -119,59 +118,72 @@ Asynchronous spider middlewares .. versionadded:: VERSION .. note:: This currently applies to :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`. + In the future it will also apply to + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`. Middleware methods discussed here can take and return async iterables. They can return the same type of iterable or they can take a normal one and return an async one. If such method needs to return an async iterable it must be an async generator, not just a coroutine that returns an iterable. -.. autofunction:: scrapy.utils.asyncgen.as_async_generator +As the result of a middleware method is passed to the same method of the next +middleware, it needs to be adapted if the second method expects a different +type. Scrapy will do this transparently: -In the simplest form that supports both sync and async input it can be written -like this:: +* A normal iterable is wrapped into an async one which shouldn't cause any side + effects. +* An async iterable is downgraded to a normal one by waiting until all results + are available and wrapping them in a normal iterable. This is problematic + because it pauses the normal middleware processing for this iterable and + because all results can be skipped if exceptions are raised during + processing. This case emits a warning and will be deprecated and then removed + in a later Scrapy version. +* Async iterables returned from + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` + won't be downgraded, an exception will be raised if that is needed. - from scrapy.utils.asyncgen import as_async_generator +As downgrading is undesirable, here is the proposed way to avoid it. If all +middlewares, including 3rd-party ones, support async iterables as input, no +downgrading will happen. But removing normal iterable support (making the +method a coroutine) from a middleware published as a separate project or used +internally in projects for older Scrapy versions breaks backwards +compatibility. So, as an interim measure (it will be deprecated and then +removed in a later Scrapy version), a middleware can provide both sync and +async methods in the following form:: - class ProcessSpiderOutputAsyncGenMiddleware: - async def process_spider_output(self, response, result, spider): - async for r in as_async_generator(result): + class UniversalSpiderMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: # ... do something with r yield r -If the middleware input (the callback result for ``process_spider_output``) is -an async iterable, all middlewares that process it must support it. The -built-in ones do, but the ones in your project and 3rd-party ones will need to -be updated to support it, as the code that expects a normal iterable will break -on an async one. If these middlewares receive an async iterable, they must -return one as well. On the other hand, if they receive a normal iterable, they -shouldn't break and ideally should return a normal iterable too. There can be -several possible implementations of this. + async def process_spider_output_async(self, response, result, spider): + async for r in result: + # ... do something with r + yield r -The simplest one, always converting normal iterables to async ones, is provided -above. Because a result of a middleware method is passed to the same method of -the next middleware, it's only possible to mix middlewares with synchronous and -asynchronous implementations of the same method if all synchronous ones are -called first (which isn't always possible). +In this case normal and async iterables will be passed to the respective +methods without any wrapping or downgrading, and in older versions of Scrapy +the coroutine method will just be ignored. When the backwards compatibility is +no longer needed the non-coroutine method can be dropped and the coroutine one +renamed to the normal name. It may be possible to extract common code from both +methods to reduce code duplication, as in the simplest case the only difference +between them will be ``for`` vs ``async for``. -Another option is to make separate methods for normal and async iterables and -choose one at run time:: +So, to recap: - from inspect import isasyncgen - - class ProcessSpiderOutputAsyncGenMiddleware: - def _normal_process_spider_output(self, response, result, spider): - # ... do something with normal result - - async def _async_process_spider_output(self, response, result, spider): - # ... do the same with async result - - def process_spider_output(self, response, result, spider): - if isasyncgen(result): - return self._async_process_spider_output(self, response, result, spider) - else: - return self._normal_process_spider_output(self, response, result, spider) - -If you are writing a middleware that you intend to publish or to use in many -projects, this is likely the best way to implement it. It may be possible to -extract common code from both methods to reduce code duplication, as in the -simplest case the only difference between them will be ``for`` vs ``async for``. +* If you don't intend to use async callbacks or middlewares containing async + code in your project, nothing should change for you yet. At some point in the + future some of the 3rd-party middlewares you use may drop backwards + compatibility, which shouldn't lead to immediate problems but may be a sign + to start converting your code to ``async def`` too. +* If you maintain a middleware that can be used with projects you can't control + (e.g. one you published for other people to use, or one that needs to support + some old project that can't be modernized), we recommend adding a + ``process_spider_output_async`` method so that the amount of unnecessary + iterable conversions is reduced but no compatibility is broken. +* If you use async callbacks, try to make sure all middlewares support them. + Note that you can modernize 3rd-party middlewares by subclassing them. +* If you want to write and publish a middleware that requires async code, you + should write in the docs that the minimum support Scrapy version is VERSION + (maybe even check this at the run time, using :attr:`scrapy.__version__`). diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index f8d4a356f..edfc2e4bb 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -123,6 +123,15 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider whose result is being processed :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_output_async(response, result, spider) + + .. versionadded:: VERSION + + If exists, this methid will be called instead of + :meth:`process_spider_output` when ``result`` is an async iterable. + If this method exists, it must be a coroutine while + :meth:`process_spider_output` must not be a coroutine. + .. method:: process_spider_exception(response, exception, spider) This method is called when a spider or :meth:`process_spider_output` diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e64cfae0a..e1fdd8d13 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -192,7 +192,8 @@ class Scraper: spider=spider ) - def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred: + def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request, + response: Response, spider: Spider) -> Deferred: if not result: return defer_succeed(None) it: Union[Generator, AsyncGenerator] diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 009ece06d..6075670b0 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,22 +3,27 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +import logging +from inspect import isasyncgenfunction from itertools import islice -from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union, cast +from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider from scrapy.exceptions import _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.utils.asyncgen import _process_iterable_universal +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import mustbe_deferred +from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import MutableAsyncChain, MutableChain +logger = logging.getLogger(__name__) + + ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any] @@ -30,6 +35,10 @@ class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' + def __init__(self, *middlewares): + super().__init__(*middlewares) + self.downgrade_warning_done = False + @classmethod def _get_mwlist_from_settings(cls, settings): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) @@ -40,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].appendleft(mw.process_start_requests) - process_spider_output = getattr(mw, 'process_spider_output', None) + process_spider_output = self._get_async_method_pair(mw, 'process_spider_output') self.methods['process_spider_output'].appendleft(process_spider_output) process_spider_exception = getattr(mw, 'process_spider_exception', None) self.methods['process_spider_exception'].appendleft(process_spider_exception) @@ -64,8 +73,19 @@ class SpiderMiddlewareManager(MiddlewareManager): def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable], exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain] ) -> Union[Generator, AsyncGenerator]: - @_process_iterable_universal - async def _evaluate_async_iterable(iterable): + + def process_sync(iterable: Iterable): + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = self._process_spider_exception(response, spider, Failure(ex), + exception_processor_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + + async def process_async(iterable: AsyncIterable): try: async for r in iterable: yield r @@ -75,7 +95,10 @@ class SpiderMiddlewareManager(MiddlewareManager): if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) - return _evaluate_async_iterable(iterable) + + if isinstance(iterable, AsyncIterable): + return process_async(iterable) + return process_sync(iterable) def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure, start_index: int = 0) -> Union[Failure, MutableChain]: @@ -87,11 +110,22 @@ class SpiderMiddlewareManager(MiddlewareManager): for method_index, method in enumerate(method_list, start=start_index): if method is None: continue + method = cast(Callable, method) result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return self._process_spider_output(response, spider, result, method_index + 1) + dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1) + # _process_spider_output() returns a Deferred only because of downgrading so this can be + # simplified when downgrading is removed. + if dfd.called: + # the result is available immediately if _process_spider_output didn't do downgrading + return dfd.result + else: + # we forbid waiting here because otherwise we would need to return a deferred from + # _process_spider_exception too, which complicates the architecture + msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + raise _InvalidOutput(msg) elif result is None: continue else: @@ -100,9 +134,13 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) return _failure + # This method cannot be made async def, as _process_spider_exception relies on the Deferred result + # being available immediately which doesn't work when it's a wrapped coroutine. + # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed. + @inlineCallbacks def _process_spider_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable], start_index: int = 0 - ) -> Union[MutableChain, MutableAsyncChain]: + ) -> Deferred: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered: Union[MutableChain, MutableAsyncChain] @@ -112,11 +150,43 @@ class SpiderMiddlewareManager(MiddlewareManager): else: recovered = MutableChain() + # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. + # 1. def foo. Sync iterables are passed as is, async ones are downgraded. + # 2. async def foo. Sync iterables are upgraded, async ones are passed as is. + # 3. def foo + async def foo_async. Iterables are passed to the respective method. + # Storing methods and method tuples in the same list is weird but we should be able to roll this back + # when we drop this compatibility feature. + method_list = islice(self.methods['process_spider_output'], start_index, None) - for method_index, method in enumerate(method_list, start=start_index): - if method is None: + for method_index, method_pair in enumerate(method_list, start=start_index): + if method_pair is None: continue + need_upgrade = need_downgrade = False + if isinstance(method_pair, tuple): + # This tuple handling is only needed until _async compatibility methods are removed. + method_sync, method_async = method_pair + method = method_async if last_result_is_async else method_sync + else: + method = method_pair + if not last_result_is_async and isasyncgenfunction(method): + need_upgrade = True + elif last_result_is_async and not isasyncgenfunction(method): + need_downgrade = True try: + if need_upgrade: + # Iterable -> AsyncIterable + result = as_async_generator(result) + elif need_downgrade: + if not self.downgrade_warning_done: + logger.warning(f"Async iterable passed to {method.__qualname__} " + f"was downgraded to a non-async one") + self.downgrade_warning_done = True + assert isinstance(result, AsyncIterable) + # AsyncIterable -> Iterable + result = yield deferred_from_coro(collect_asyncgen(result)) + if isinstance(recovered, AsyncIterable): + recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered)) + recovered = MutableChain(recovered_collected) # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: @@ -130,8 +200,6 @@ class SpiderMiddlewareManager(MiddlewareManager): msg = (f"Middleware {method.__qualname__} must return an " f"iterable, got {type(result)}") raise _InvalidOutput(msg) - if last_result_is_async and isinstance(result, Iterable): - raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable") last_result_is_async = isinstance(result, AsyncIterable) if last_result_is_async: @@ -139,31 +207,58 @@ class SpiderMiddlewareManager(MiddlewareManager): else: return MutableChain(result, recovered) # type: ignore[arg-type] - def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] - ) -> Union[MutableChain, MutableAsyncChain]: + async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: recovered: Union[MutableChain, MutableAsyncChain] if isinstance(result, AsyncIterable): recovered = MutableAsyncChain() else: recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) - result = self._process_spider_output(response, spider, result) + result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result)) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) else: + if isinstance(recovered, AsyncIterable): + recovered_collected = await collect_asyncgen(recovered) + recovered = MutableChain(recovered_collected) return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: - def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]: - return self._process_callback_output(response, spider, result) + async def process_callback_output(result: Union[Iterable, AsyncIterable] + ) -> Union[MutableChain, MutableAsyncChain]: + return await self._process_callback_output(response, spider, result) def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]: return self._process_spider_exception(response, spider, _failure) dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider) - dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) + dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception) return dfd def process_start_requests(self, start_requests, spider: Spider) -> Deferred: return self._process_chain('process_start_requests', start_requests, spider) + + # This method is only needed until _async compatibility methods are removed. + @staticmethod + def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]: + normal_method = getattr(mw, methodname, None) + methodname_async = methodname + "_async" + async_method = getattr(mw, methodname_async, None) + if not async_method: + return normal_method + if not normal_method: + logger.error(f"Middleware {mw.__qualname__} has {methodname_async} " + f"without {methodname}, skipping this method.") + return None + if not isasyncgenfunction(async_method): + logger.error(f"{async_method.__qualname__} is not " + f"an async generator function, skipping this method.") + return normal_method + if isasyncgenfunction(normal_method): + logger.error(f"{normal_method.__qualname__} is an async " + f"generator function while {methodname_async} exists, " + f"skipping both methods.") + return None + return normal_method, async_method diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2eb1d8609..8d7e5a602 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict, Optional, cast, Iterable +from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -21,8 +21,9 @@ class MiddlewareManager: def __init__(self, *middlewares): self.middlewares = middlewares - # Optional because process_spider_output and process_spider_exception can be None - self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque) + # Only process_spider_output and process_spider_exception can be None. + # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. + self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 973404b2b..29634c3ad 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,7 +7,6 @@ See documentation in docs/topics/spider-middleware.rst import logging from scrapy.http import Request -from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -29,36 +28,42 @@ class DepthMiddleware: return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): - def _filter(request): - if isinstance(request, Request): - depth = response.meta['depth'] + 1 - request.meta['depth'] = depth - if self.prio: - request.priority -= depth * self.prio - if self.maxdepth and depth > self.maxdepth: - logger.debug( - "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", - {'maxdepth': self.maxdepth, 'requrl': request.url}, - extra={'spider': spider} - ) - return False - else: - if self.verbose_stats: - self.stats.inc_value(f'request_depth_count/{depth}', - spider=spider) - self.stats.max_value('request_depth_max', depth, - spider=spider) + # base case (depth=0) + if 'depth' not in response.meta: + response.meta['depth'] = 0 + if self.verbose_stats: + self.stats.inc_value('request_depth_count/0', spider=spider) + + return (r for r in result or () if self._filter(r, response, spider)) + + async def process_spider_output_async(self, response, result, spider): + # base case (depth=0) + if 'depth' not in response.meta: + response.meta['depth'] = 0 + if self.verbose_stats: + self.stats.inc_value('request_depth_count/0', spider=spider) + + async for r in result or (): + if self._filter(r, response, spider): + yield r + + def _filter(self, request, response, spider): + if not isinstance(request, Request): return True - - @_process_iterable_universal - async def process(result): - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) - - async for r in result or (): - if _filter(r): - yield r - return process(result) + depth = response.meta['depth'] + 1 + request.meta['depth'] = depth + if self.prio: + request.priority -= depth * self.prio + if self.maxdepth and depth > self.maxdepth: + logger.debug( + "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", + {'maxdepth': self.maxdepth, 'requrl': request.url}, + extra={'spider': spider} + ) + return False + if self.verbose_stats: + self.stats.inc_value(f'request_depth_count/{depth}', + spider=spider) + self.stats.max_value('request_depth_max', depth, + spider=spider) + return True diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 074ec7a4e..448bc1367 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,7 +9,6 @@ import warnings from scrapy import signals from scrapy.http import Request -from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) @@ -27,24 +26,27 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - @_process_iterable_universal - async def process(result): - async for x in result: - if isinstance(x, Request): - if x.dont_filter or self.should_follow(x, spider): - yield x - else: - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - else: - yield x - return process(result) + return (r for r in result or () if self._filter(r, spider)) + + async def process_spider_output_async(self, response, result, spider): + async for r in result or (): + if self._filter(r, spider): + yield r + + def _filter(self, request, spider) -> bool: + if not isinstance(request, Request): + return True + if request.dont_filter or self.should_follow(request, spider): + return True + domain = urlparse_cached(request).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': request}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + return False def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index e0a22592f..8027beb92 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,7 +11,6 @@ from w3lib.url import safe_url_string from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response -from scrapy.utils.asyncgen import _process_iterable_universal from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url @@ -334,19 +333,18 @@ class RefererMiddleware: return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): - def _set_referer(r): - if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response.url, r.url) - if referrer is not None: - r.headers.setdefault('Referer', referrer) - return r + return (self._set_referer(r, response) for r in result or ()) - @_process_iterable_universal - async def process(result): - async for r in result or (): - yield _set_referer(r) + async def process_spider_output_async(self, response, result, spider): + async for r in result or (): + yield self._set_referer(r, response) - return process(result) + def _set_referer(self, r, response): + if isinstance(r, Request): + referrer = self.policy(response, r).referrer(response.url, r.url) + if referrer is not None: + r.headers.setdefault('Referer', referrer) + return r def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 63b1d36bd..7ad64d2af 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,7 +8,6 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured -from scrapy.utils.asyncgen import _process_iterable_universal logger = logging.getLogger(__name__) @@ -26,22 +25,20 @@ class UrlLengthMiddleware: return cls(maxlength) def process_spider_output(self, response, result, spider): - def _filter(request): - if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.info( - "Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider} - ) - spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) - return False - else: - return True + return (r for r in result or () if self._filter(r, spider)) - @_process_iterable_universal - async def process(result): - async for r in result or (): - if _filter(r): - yield r + async def process_spider_output_async(self, response, result, spider): + async for r in result or (): + if self._filter(r, spider): + yield r - return process(result) + def _filter(self, request, spider): + if isinstance(request, Request) and len(request.url) > self.maxlength: + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) + return False + return True diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index ae9a79989..9f794de92 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,6 +1,4 @@ -import functools -import inspect -from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, Union async def collect_asyncgen(result: AsyncIterable): @@ -18,46 +16,3 @@ async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerat else: for r in it: yield r - - -# https://stackoverflow.com/a/66170760/113586 -def _process_iterable_universal(process_async: Callable): - """ Takes a function that takes an async iterable, args and kwargs. Returns - a function that takes any iterable, args and kwargs. - - Requires that process_async only awaits on the iterable and synchronous functions, - so it's better to use this only in the Scrapy code itself. - """ - - # If this stops working, all internal uses can be just replaced with manually-written - # process_sync functions. - - def process_sync(iterable: Iterable, *args, **kwargs) -> Generator: - agen = process_async(as_async_generator(iterable), *args, **kwargs) - if not inspect.isasyncgen(agen): - raise ValueError(f"process_async returned wrong type {type(agen)}") - sent = None - while True: - try: - gen = agen.asend(sent) - gen.send(None) - except StopIteration as e: - sent = yield e.value - except StopAsyncIteration: - return - else: - gen.throw(RuntimeError, - f"Synchronously-called function '{process_async.__name__}' has blocked, " - f"you can't use {_process_iterable_universal.__name__} with it.") - - @functools.wraps(process_async) - def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]: - if isinstance(iterable, AsyncIterable): - # call process_async directly - return process_async(iterable, *args, **kwargs) - if isinstance(iterable, Iterable): - # convert process_async to process_sync - return process_sync(iterable, *args, **kwargs) - raise TypeError(f"Wrong iterable type {type(iterable)}") - - return process diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d086347bc..11c089ac2 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import warnings import weakref from functools import partial, wraps from itertools import chain -from typing import AsyncIterable, Iterable, Union +from typing import AsyncGenerator, AsyncIterable, Iterable, Union from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator @@ -359,7 +359,7 @@ class MutableChain(Iterable): return self.__next__() -async def _async_chain(*iterables: Union[Iterable, AsyncIterable]): +async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: for it in iterables: async for o in as_async_generator(it): yield o diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index b0ca2f62e..f9f2b6642 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,8 @@ import collections.abc +from typing import Optional from unittest import mock +from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -8,8 +10,8 @@ from twisted.python.failure import Failure from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput -from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager @@ -115,33 +117,40 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects + @staticmethod + def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None): + if start_index is None: + start_index = 10 + return {i: c for c, i in enumerate(mw_classes, start=start_index)} + @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes): - for mw_cls in mw_classes: - self.mwman._add_middleware(mw_cls()) + def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting}) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) return result @defer.inlineCallbacks - def _test_simple_base(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) + def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None): + with LogCapture() as log: + result = yield self._get_middleware_result(*mw_classes, start_index=start_index) self.assertIsInstance(result, collections.abc.Iterable) result_list = list(result) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) + self.assertEqual("downgraded to a non-async" in str(log), downgrade) @defer.inlineCallbacks - def _test_asyncgen_base(self, *mw_classes): - result = yield self._get_middleware_result(*mw_classes) + def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None): + with LogCapture() as log: + result = yield self._get_middleware_result(*mw_classes, start_index=start_index) self.assertIsInstance(result, collections.abc.AsyncIterator) result_list = yield deferred_from_coro(collect_asyncgen(result)) self.assertEqual(len(result_list), self.RESULT_COUNT) self.assertIsInstance(result_list[0], self.ITEM_TYPE) - - @defer.inlineCallbacks - def _test_asyncgen_fail(self, *mw_classes): - with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"): - yield self._get_middleware_result(*mw_classes) + self.assertEqual("downgraded to a non-async" in str(log), downgrade) class ProcessSpiderOutputSimpleMiddleware: @@ -152,17 +161,36 @@ class ProcessSpiderOutputSimpleMiddleware: class ProcessSpiderOutputAsyncGenMiddleware: async def process_spider_output(self, response, result, spider): - async for r in as_async_generator(result): + async for r in result: yield r class ProcessSpiderOutputUniversalMiddleware: def process_spider_output(self, response, result, spider): - @_process_iterable_universal - async def process(result): - async for r in result: - yield r - return process(result) + for r in result: + yield r + + async def process_spider_output_async(self, response, result, spider): + async for r in result: + yield r + + +class ProcessSpiderExceptionSimpleIterableMiddleware: + def process_spider_exception(self, response, exception, spider): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + + +class ProcessSpiderExceptionAsyncIterableMiddleware: + async def process_spider_exception(self, response, exception, spider): + yield {'foo': 1} + d = defer.Deferred() + from twisted.internet import reactor + reactor.callLater(0, d.callback, None) + await maybe_deferred_to_future(d) + yield {'foo': 2} + yield {'foo': 3} class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): @@ -183,18 +211,19 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): return self._test_simple_base(self.MW_SIMPLE) def test_asyncgen(self): - """ Asyncgen mw """ + """ Asyncgen mw; upgrade """ return self._test_asyncgen_base(self.MW_ASYNCGEN) def test_simple_asyncgen(self): - """ Simple mw -> asyncgen mw """ + """ Simple mw -> asyncgen mw; upgrade """ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) def test_asyncgen_simple(self): - """ Asyncgen mw -> simple mw; cannot work """ - return self._test_asyncgen_fail(self.MW_SIMPLE, - self.MW_ASYNCGEN) + """ Asyncgen mw -> simple mw; upgrade then downgrade """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_ASYNCGEN, + downgrade=True) def test_universal(self): """ Universal mw """ @@ -211,12 +240,12 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): self.MW_SIMPLE) def test_universal_asyncgen(self): - """ Universal mw -> asyncgen mw """ + """ Universal mw -> asyncgen mw; upgrade """ return self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) def test_asyncgen_universal(self): - """ Asyncgen mw -> universal mw """ + """ Asyncgen mw -> universal mw; upgrade """ return self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) @@ -229,27 +258,31 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): yield item def test_simple(self): - """ Simple mw; cannot work """ - return self._test_asyncgen_fail(self.MW_SIMPLE) + """ Simple mw; downgrade """ + return self._test_simple_base(self.MW_SIMPLE, + downgrade=True) def test_simple_asyncgen(self): - """ Simple mw -> asyncgen mw; cannot work """ - return self._test_asyncgen_fail(self.MW_ASYNCGEN, - self.MW_SIMPLE) + """ Simple mw -> asyncgen mw; downgrade then upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_SIMPLE, + downgrade=True) def test_universal(self): """ Universal mw """ return self._test_asyncgen_base(self.MW_UNIVERSAL) def test_universal_simple(self): - """ Universal mw -> simple mw; cannot work """ - return self._test_asyncgen_fail(self.MW_SIMPLE, - self.MW_UNIVERSAL) + """ Universal mw -> simple mw; downgrade """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_UNIVERSAL, + downgrade=True) def test_simple_universal(self): - """ Simple mw -> universal mw; cannot work """ - return self._test_asyncgen_fail(self.MW_UNIVERSAL, - self.MW_SIMPLE) + """ Simple mw -> universal mw; downgrade """ + return self._test_simple_base(self.MW_UNIVERSAL, + self.MW_SIMPLE, + downgrade=True) class ProcessStartRequestsSimpleMiddleware: @@ -269,13 +302,198 @@ class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase): yield Request(f'https://example.com/{i}', dont_filter=True) @defer.inlineCallbacks - def _get_middleware_result(self, *mw_classes): - for mw_cls in mw_classes: - self.mwman._add_middleware(mw_cls()) + def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting}) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) start_requests = iter(self._start_requests()) results = yield self.mwman.process_start_requests(start_requests, self.spider) return results def test_simple(self): """ Simple mw """ - self._test_simple_base(self.MW_SIMPLE) + return self._test_simple_base(self.MW_SIMPLE) + + +class UniversalMiddlewareNoSync: + async def process_spider_output_async(self, response, result, spider): + yield + + +class UniversalMiddlewareBothSync: + def process_spider_output(self, response, result, spider): + yield + + def process_spider_output_async(self, response, result, spider): + yield + + +class UniversalMiddlewareBothAsync: + async def process_spider_output(self, response, result, spider): + yield + + async def process_spider_output_async(self, response, result, spider): + yield + + +class UniversalMiddlewareManagerTest(TestCase): + def setUp(self): + self.mwman = SpiderMiddlewareManager() + + def test_simple_mw(self): + mw = ProcessSpiderOutputSimpleMiddleware + self.mwman._add_middleware(mw) + self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + + def test_async_mw(self): + mw = ProcessSpiderOutputAsyncGenMiddleware + self.mwman._add_middleware(mw) + self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + + def test_universal_mw(self): + mw = ProcessSpiderOutputUniversalMiddleware + self.mwman._add_middleware(mw) + self.assertEqual(self.mwman.methods['process_spider_output'][0], + (mw.process_spider_output, mw.process_spider_output_async)) + + def test_universal_mw_no_sync(self): + with LogCapture() as log: + self.mwman._add_middleware(UniversalMiddlewareNoSync) + self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async" + " without process_spider_output", str(log)) + self.assertEqual(self.mwman.methods['process_spider_output'][0], None) + + def test_universal_mw_both_sync(self): + mw = UniversalMiddlewareBothSync + with LogCapture() as log: + self.mwman._add_middleware(mw) + self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async " + "is not an async generator function", str(log)) + self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output) + + def test_universal_mw_both_async(self): + with LogCapture() as log: + self.mwman._add_middleware(UniversalMiddlewareBothAsync) + self.assertIn("UniversalMiddlewareBothAsync.process_spider_output " + "is an async generator function while process_spider_output_async exists", + str(log)) + self.assertEqual(self.mwman.methods['process_spider_output'][0], None) + + +class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): + ITEM_TYPE = dict + MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware + MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware + + def _scrape_func(self, *args, **kwargs): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + + @defer.inlineCallbacks + def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): + setting = self._construct_mw_setting(*mw_classes, start_index=start_index) + self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting}) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) + result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider) + return result + + def test_just_builtin(self): + return self._test_simple_base() + + def test_builtin_simple(self): + return self._test_simple_base(self.MW_SIMPLE, start_index=1000) + + def test_builtin_async(self): + """ Upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + + def test_builtin_universal(self): + return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) + + def test_simple_builtin(self): + return self._test_simple_base(self.MW_SIMPLE) + + def test_async_builtin(self): + """ Upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN) + + def test_universal_builtin(self): + return self._test_simple_base(self.MW_UNIVERSAL) + + +class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest): + async def _scrape_func(self, *args, **kwargs): + for item in super()._scrape_func(): + yield item + + def test_just_builtin(self): + return self._test_asyncgen_base() + + def test_builtin_simple(self): + """ Downgrade """ + return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) + + def test_builtin_async(self): + return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) + + def test_builtin_universal(self): + return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) + + def test_simple_builtin(self): + """ Downgrade """ + return self._test_simple_base(self.MW_SIMPLE, downgrade=True) + + def test_async_builtin(self): + return self._test_asyncgen_base(self.MW_ASYNCGEN) + + def test_universal_builtin(self): + return self._test_asyncgen_base(self.MW_UNIVERSAL) + + +class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase): + ITEM_TYPE = dict + MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware + MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware + MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware + MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware + + def _scrape_func(self, *args, **kwargs): + 1 / 0 + + @defer.inlineCallbacks + def _test_asyncgen_nodowngrade(self, *mw_classes): + with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"): + yield self._get_middleware_result(*mw_classes) + + def test_exc_simple(self): + """ Simple exc mw """ + return self._test_simple_base(self.MW_EXC_SIMPLE) + + def test_exc_async(self): + """ Async exc mw """ + return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) + + def test_exc_simple_simple(self): + """ Simple exc mw -> simple output mw """ + return self._test_simple_base(self.MW_SIMPLE, + self.MW_EXC_SIMPLE) + + def test_exc_async_async(self): + """ Async exc mw -> async output mw """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_EXC_ASYNCGEN) + + def test_exc_simple_async(self): + """ Simple exc mw -> async output mw; upgrade """ + return self._test_asyncgen_base(self.MW_ASYNCGEN, + self.MW_EXC_SIMPLE) + + def test_exc_async_simple(self): + """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """ + return self._test_asyncgen_nodowngrade(self.MW_SIMPLE, + self.MW_EXC_ASYNCGEN) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 088c14ca8..dac246fb6 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -28,6 +28,7 @@ class RecoveryMiddleware: class RecoverySpider(Spider): name = 'RecoverySpider' custom_settings = { + 'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': { RecoveryMiddleware: 10, }, @@ -107,6 +108,13 @@ class GeneratorCallbackSpider(Spider): raise ImportError() +class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider): + async def parse(self, response): + yield {'test': 1} + yield {'test': 2} + raise ImportError() + + # ================================================================================ # (2.1) exceptions from a spider callback (generator, middleware right after callback) class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider): @@ -360,6 +368,15 @@ class TestSpiderMiddleware(TestCase): self.assertIn("Middleware: ImportError exception caught", str(log2)) self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks + def test_async_generator_callback(self): + """ + Same as test_generator_callback but with an async callback. + """ + log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks def test_generator_callback_right_after_callback(self): """ diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 7abe17c22..9ae66c57c 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,7 +1,6 @@ -from twisted.internet.defer import Deferred from twisted.trial import unittest -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f @@ -19,52 +18,3 @@ class AsyncgenUtilsTest(unittest.TestCase): ag = as_async_generator(range(42)) results = await collect_asyncgen(ag) self.assertEqual(results, list(range(42))) - - -@_process_iterable_universal -async def process_iterable(iterable): - async for i in iterable: - yield i * 2 - - -@_process_iterable_universal -async def process_iterable_awaiting(iterable): - async for i in iterable: - yield i * 2 - d = Deferred() - from twisted.internet import reactor - reactor.callLater(0, d.callback, 42) - await d - - -class ProcessIterableUniversalTest(unittest.TestCase): - - def test_normal(self): - iterable = iter([1, 2, 3]) - results = list(process_iterable(iterable)) - self.assertEqual(results, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_async(self): - iterable = as_async_generator([1, 2, 3]) - results = await collect_asyncgen(process_iterable(iterable)) - self.assertEqual(results, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_blocking(self): - iterable = [1, 2, 3] - with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"): - list(process_iterable_awaiting(iterable)) - - def test_invalid_iterable(self): - with self.assertRaisesRegex(TypeError, "Wrong iterable type"): - process_iterable(None) - - @deferred_f_from_coro_f - async def test_invalid_process(self): - @_process_iterable_universal - def process_iterable_invalid(iterable): - pass - - with self.assertRaisesRegex(ValueError, "process_async returned wrong type"): - list(process_iterable_invalid([])) From e079bffdab11402d1936103ba453e43e97925079 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Jan 2022 19:21:07 +0500 Subject: [PATCH 055/237] Disable logging-fstring-interpolation in pylint. --- pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/pylintrc b/pylintrc index 2cdd6321e..0d29dc709 100644 --- a/pylintrc +++ b/pylintrc @@ -49,6 +49,7 @@ disable=abstract-method, keyword-arg-before-vararg, line-too-long, logging-format-interpolation, + logging-fstring-interpolation, logging-not-lazy, lost-exception, method-hidden, From 3ecbea4b876ed084f30bb4063e1993dd9c3cdb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 16:06:49 +0100 Subject: [PATCH 056/237] CrawlerProcess: initiate the reactor only once --- scrapy/crawler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a638254f1..9939a19eb 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -290,6 +290,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) + self._initiated_reactor = False def _signal_shutdown(self, signum, _): from twisted.internet import reactor @@ -310,7 +311,9 @@ class CrawlerProcess(CrawlerRunner): def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, init_reactor=True) + init_reactor = not self._initiated_reactor + self._initiated_reactor = True + return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start(self, stop_after_crawl=True, install_signal_handlers=True): """ From 96fc4dae15181695c58040389fa502857a2b0df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 16:14:39 +0100 Subject: [PATCH 057/237] CrawlerProcess: test a multi-spider scenario --- tests/CrawlerProcess/multi.py | 16 ++++++++++++++++ tests/test_crawler.py | 6 ++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/CrawlerProcess/multi.py diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py new file mode 100644 index 000000000..aaa1af5c5 --- /dev/null +++ b/tests/CrawlerProcess/multi.py @@ -0,0 +1,16 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8f6227109..957525382 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -302,6 +302,12 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_multi(self): + log = self.run_script('multi.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) From 3bf6baeaa705ddc2d3417206f3db816dabadf5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 17:03:41 +0100 Subject: [PATCH 058/237] =?UTF-8?q?initiated=20=E2=86=92=20initialized?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/crawler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9939a19eb..d669d93a8 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -290,7 +290,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initiated_reactor = False + self._initialized_reactor = False def _signal_shutdown(self, signum, _): from twisted.internet import reactor @@ -311,8 +311,8 @@ class CrawlerProcess(CrawlerRunner): def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - init_reactor = not self._initiated_reactor - self._initiated_reactor = True + init_reactor = not self._initialized_reactor + self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start(self, stop_after_crawl=True, install_signal_handlers=True): From 62a00812669aa0906aa0d4e9a4c2e87be3b74975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 7 Mar 2022 12:00:44 +0100 Subject: [PATCH 059/237] engine: prevent slot method call after unsetting the slot --- scrapy/core/engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f9de7ee23..6602f661d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -136,7 +136,9 @@ class ExecutionEngine: self.paused = False def _next_request(self) -> None: - assert self.slot is not None # typing + if self.slot is None: + return + assert self.spider is not None # typing if self.paused: @@ -184,7 +186,8 @@ class ExecutionEngine: d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) - d.addBoth(lambda _: self.slot.nextcall.schedule()) + slot = self.slot + d.addBoth(lambda _: slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) From c1d4be8cb5ed610fef4a2af60f9b654faab7a243 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Tue, 15 Mar 2022 07:30:30 -0300 Subject: [PATCH 060/237] =?UTF-8?q?Restore=20=E2=80=98-o=20-=E2=80=99=20su?= =?UTF-8?q?pport=20(#5445)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/cmdline.py | 17 +++++++++++++---- tests/test_commands.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 491c4beab..5ee1f0f44 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -14,6 +14,15 @@ from scrapy.utils.project import inside_project, get_project_settings from scrapy.utils.python import garbage_collect +class ScrapyArgumentParser(argparse.ArgumentParser): + def _parse_optional(self, arg_string): + # if starts with -: it means that is a parameter not a argument + if arg_string[:2] == '-:': + return None + + return super()._parse_optional(arg_string) + + def _iter_command_classes(module_name): # TODO: add `name` attribute to commands and and merge this function with # scrapy.utils.spider.iter_spider_classes @@ -131,10 +140,10 @@ def execute(argv=None, settings=None): sys.exit(2) cmd = cmds[cmdname] - parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, - usage=f"scrapy {cmdname} {cmd.syntax()}", - conflict_handler='resolve', - description=cmd.long_desc()) + parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter, + usage=f"scrapy {cmdname} {cmd.syntax()}", + conflict_handler='resolve', + description=cmd.long_desc()) settings.setdict(cmd.default_settings, priority='command') cmd.settings = settings cmd.add_options(parser) diff --git a/tests/test_commands.py b/tests/test_commands.py index 7cd19b29a..b5e6c2b8b 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -770,6 +770,21 @@ class MySpider(scrapy.Spider): log = self.get_log(spider_code, args=args) self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + def test_output_stdout(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + def start_requests(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return [] +""" + args = ['-o', '-:json'] + log = self.get_log(spider_code, args=args) + self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log) + @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): From b59a69be1790f138afe89df3dfed17ac48384d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 15 Mar 2022 12:10:41 +0100 Subject: [PATCH 061/237] Test that a low CLOSEPIDER_TIMEOUT does not raise an exception --- tests/test_engine.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_engine.py b/tests/test_engine.py index fa7d0c8d4..b8fd341f6 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,9 +12,11 @@ module with the ``runserver`` argument:: import os import re +import subprocess import sys import warnings from collections import defaultdict +from threading import Timer from urllib.parse import urlparse import attr @@ -502,6 +504,37 @@ class EngineTest(unittest.TestCase): self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") + def test_short_timeout(self): + args = ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'fetch', + '-s', + 'CLOSESPIDER_TIMEOUT=0.001', + '-s', + 'LOG_LEVEL=DEBUG', + 'http://toscrape.com', + ) + p = subprocess.Popen( + args, + stderr=subprocess.PIPE, + ) + + def kill_proc(): + p.kill() + p.communicate() + assert False, 'Command took too much time to complete' + + timer = Timer(15, kill_proc) + try: + timer.start() + _, stderr = p.communicate() + finally: + timer.cancel() + + self.assertNotIn(b'Traceback', stderr) + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': From 78ba4b033b016be7fbb22bfa9e6d5d389380e6d4 Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Wed, 16 Mar 2022 15:14:24 +0100 Subject: [PATCH 062/237] fixed detection of extension like ".tar.gz" in URL --- scrapy/utils/url.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index a6a2a9e8b..bae5a9433 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,7 +5,6 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ -import posixpath import re from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse @@ -31,8 +30,8 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): - return posixpath.splitext(parse_url(url).path)[1].lower() in extensions - + """Return True if the url ends with one of the extensions provided""" + return any(parse_url(url).path.lower().endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already From fd08bb6cd99f16c9ce433583d4698801dc7e0ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 16 Mar 2022 14:34:57 +0100 Subject: [PATCH 063/237] Refactor the asynchronous process_spider_output documentation --- docs/topics/coroutines.rst | 157 +++++++++++++++++------------- docs/topics/spider-middleware.rst | 22 +++-- 2 files changed, 102 insertions(+), 77 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 073b6bd9a..361cd5e60 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -17,9 +17,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.Request` callbacks. + If you are using any custom or third-party :ref:`spider middleware + `, see :ref:`sync-async-spider-middleware`. + .. versionchanged:: VERSION - Output of async callbacks is now processed asynchronously instead of collecting - all of it first. + Output of async callbacks is now processed asynchronously instead of + collecting all of it first. - The :meth:`process_item` method of :ref:`item pipelines `. @@ -34,18 +37,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. -- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. See - :ref:`async-spider-middlewares`. +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares `. + + It must be defined as an :term:`asynchronous generator`. The input + ``result`` parameter is an :term:`asynchronous iterable`. + + See also :ref:`sync-async-spider-middleware` and + :ref:`universal-spider-middleware`. .. versionadded:: VERSION -Usage -===== +General usage +============= -There are several use cases for coroutines in Scrapy. Code that would -return Deferreds when written for previous Scrapy versions, such as downloader -middlewares and signal handlers, can be rewritten to be shorter and cleaner:: +There are several use cases for coroutines in Scrapy. + +Code that would return Deferreds when written for previous Scrapy versions, +such as downloader middlewares and signal handlers, can be rewritten to be +shorter and cleaner:: from itemadapter import ItemAdapter @@ -110,46 +121,73 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs -.. _async-spider-middlewares: -Asynchronous spider middlewares -=============================== +.. _sync-async-spider-middleware: + +Mixing synchronous and asynchronous spider middlewares +====================================================== .. versionadded:: VERSION -.. note:: This currently applies to - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`. - In the future it will also apply to - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`. -Middleware methods discussed here can take and return async iterables. They can -return the same type of iterable or they can take a normal one and return an -async one. If such method needs to return an async iterable it must be an async -generator, not just a coroutine that returns an iterable. +The output of a :class:`~scrapy.Request` callback is passed as the ``result`` +parameter to the +:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method +of the first :ref:`spider middleware ` from the +:ref:`list of active spider middlewares `. +Then the output of that ``process_spider_output`` method is passed to the +``process_spider_output`` method of the next spider middleware, and so on for +every active spider middleware. -As the result of a middleware method is passed to the same method of the next -middleware, it needs to be adapted if the second method expects a different -type. Scrapy will do this transparently: +Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods +in this chain of calls. -* A normal iterable is wrapped into an async one which shouldn't cause any side - effects. -* An async iterable is downgraded to a normal one by waiting until all results - are available and wrapping them in a normal iterable. This is problematic - because it pauses the normal middleware processing for this iterable and - because all results can be skipped if exceptions are raised during - processing. This case emits a warning and will be deprecated and then removed - in a later Scrapy version. -* Async iterables returned from - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` - won't be downgraded, an exception will be raised if that is needed. +However, if any of the ``process_spider_output`` methods is defined as a +synchronous method, and the previous ``Request`` callback or +``process_spider_output`` method is a coroutine, there are some drawbacks to +the asynchronous-to-synchronous conversion that Scrapy does so that the +synchronous ``process_spider_output`` method gets a synchronous iterable as its +``result`` parameter: -As downgrading is undesirable, here is the proposed way to avoid it. If all -middlewares, including 3rd-party ones, support async iterables as input, no -downgrading will happen. But removing normal iterable support (making the -method a coroutine) from a middleware published as a separate project or used -internally in projects for older Scrapy versions breaks backwards -compatibility. So, as an interim measure (it will be deprecated and then -removed in a later Scrapy version), a middleware can provide both sync and -async methods in the following form:: +- The whole output of the previous ``Request`` callback or + ``process_spider_output`` method is awaited at this point. + +- If an exception raises while awaiting the output of the previous + ``Request`` callback or ``process_spider_output`` method, none of that + output will be processed. + +Asynchronous-to-synchronous conversions are supported for backward +compatibility, but they are deprecated and will stop working in a future +version of Scrapy. + +To avoid asynchronous-to-synchronous conversion, when defining ``Request`` +callbacks as coroutine methods or when using spider middlewares whose +``process_spider_output`` method is an :term:`asynchronous generator`, all +active spider middlewares must either have their ``process_spider_output`` +method defined as an asynchronous generator or :ref:`define a +process_spider_output_async method `. + +.. note:: When using third-party spider middlewares that only define a + synchronous ``process_spider_output`` method, consider + :ref:`making them universal ` through + :ref:`subclassing `. + + +.. _universal-spider-middleware: + +Universal spider middleware +=========================== + +.. versionadded:: VERSION + +To allow writing a spider middleware that supports asynchronous execution of +its ``process_spider_output`` method in Scrapy VERSION and later (avoiding +:ref:`asynchronous-to-synchronous conversions `) +while maintaining support for older Scrapy versions, you may define +``process_spider_output`` as a synchronous method and define an +:term:`asynchronous generator` version of that method with an alternative name: +``process_spider_output_async``. + +For example:: class UniversalSpiderMiddleware: def process_spider_output(self, response, result, spider): @@ -162,28 +200,13 @@ async methods in the following form:: # ... do something with r yield r -In this case normal and async iterables will be passed to the respective -methods without any wrapping or downgrading, and in older versions of Scrapy -the coroutine method will just be ignored. When the backwards compatibility is -no longer needed the non-coroutine method can be dropped and the coroutine one -renamed to the normal name. It may be possible to extract common code from both -methods to reduce code duplication, as in the simplest case the only difference -between them will be ``for`` vs ``async for``. +.. note:: This is an interim measure to allow, for a time, to write code that + works in Scrapy VERSION and later without requiring + asynchronous-to-synchronous conversions, and works in earlier Scrapy + versions as well. -So, to recap: - -* If you don't intend to use async callbacks or middlewares containing async - code in your project, nothing should change for you yet. At some point in the - future some of the 3rd-party middlewares you use may drop backwards - compatibility, which shouldn't lead to immediate problems but may be a sign - to start converting your code to ``async def`` too. -* If you maintain a middleware that can be used with projects you can't control - (e.g. one you published for other people to use, or one that needs to support - some old project that can't be modernized), we recommend adding a - ``process_spider_output_async`` method so that the amount of unnecessary - iterable conversions is reduced but no compatibility is broken. -* If you use async callbacks, try to make sure all middlewares support them. - Note that you can modernize 3rd-party middlewares by subclassing them. -* If you want to write and publish a middleware that requires async code, you - should write in the docs that the minimum support Scrapy version is VERSION - (maybe even check this at the run time, using :attr:`scrapy.__version__`). + In some future version of Scrapy, however, this feature will be + deprecated and, eventually, in a later version of Scrapy, this + feature will be removed, and all spider middlewares will be expected + to define their ``process_spider_output`` method as an asynchronous + generator. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index edfc2e4bb..787545ed2 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -98,10 +98,6 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output(response, result, spider) - .. versionchanged:: VERSION - Since VERSION this can take and return an :term:`python:asynchronous - iterable`. - This method is called with the results returned from the Spider, after it has processed the response. @@ -109,8 +105,15 @@ object gives you access, for example, to the :ref:`settings `. :class:`~scrapy.Request` objects and :ref:`item objects `. - .. note:: When defined as a :ref:`coroutine `, this method needs - to be an async generator, not just return an iterable. + .. versionchanged:: VERSION + This method may be defined as an :term:`asynchronous generator`, in + which case ``result`` is an :term:`asynchronous iterable`. + + Consider defining this method as an :term:`asynchronous generator`, + which will be a requirement in a future version of Scrapy. However, if + you wish your spider middleware to work with Scrapy versions earlier + than Scrapy VERSION, :ref:`make your spider middleware universal + ` instead. :param response: the response which generated this output from the spider @@ -127,10 +130,9 @@ object gives you access, for example, to the :ref:`settings `. .. versionadded:: VERSION - If exists, this methid will be called instead of - :meth:`process_spider_output` when ``result`` is an async iterable. - If this method exists, it must be a coroutine while - :meth:`process_spider_output` must not be a coroutine. + If defined, this method must be an :term:`asynchronous generator`, + which will be called instead of :meth:`process_spider_output` if + ``result`` is an :term:`asynchronous iterable`. .. method:: process_spider_exception(response, exception, spider) From c961438d5d9998344460d930ea502fae40553043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 16 Mar 2022 18:45:56 +0100 Subject: [PATCH 064/237] tests: cover scenarios of bad results from process_spider_output --- scrapy/core/spidermw.py | 19 ++++++++---- tests/test_spidermiddleware.py | 54 +++++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 6075670b0..1aa02f29f 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -4,7 +4,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ import logging -from inspect import isasyncgenfunction +from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast @@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - msg = (f"Middleware {method.__qualname__} must return None " + msg = (f"{method.__qualname__} must return None " f"or raise an exception, got {type(result)}") raise _InvalidOutput(msg) except _InvalidOutput: @@ -129,7 +129,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - msg = (f"Middleware {method.__qualname__} must return None " + msg = (f"{method.__qualname__} must return None " f"or an iterable, got {type(result)}") raise _InvalidOutput(msg) return _failure @@ -197,8 +197,17 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered) else: - msg = (f"Middleware {method.__qualname__} must return an " - f"iterable, got {type(result)}") + if iscoroutine(result): + result.close() # Silence warning about not awaiting + msg = ( + f"{method.__qualname__} must be an asynchronous " + f"generator (i.e. use yield)" + ) + else: + msg = ( + f"{method.__qualname__} must return an iterable, got " + f"{type(result)}" + ) raise _InvalidOutput(msg) last_result_is_async = isinstance(result, AsyncIterable) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index f9f2b6642..ed0912b82 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -123,6 +123,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase): start_index = 10 return {i: c for c, i in enumerate(mw_classes, start=start_index)} + def _scrape_func(self, *args, **kwargs): + yield {'foo': 1} + yield {'foo': 2} + yield {'foo': 3} + @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) @@ -201,11 +206,6 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def _scrape_func(self, *args, **kwargs): - yield {'foo': 1} - yield {'foo': 2} - yield {'foo': 3} - def test_simple(self): """ Simple mw """ return self._test_simple_base(self.MW_SIMPLE) @@ -285,6 +285,45 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple): downgrade=True) +class ProcessSpiderOutputNonIterableMiddleware: + def process_spider_output(self, response, result, spider): + return + + +class ProcessSpiderOutputCoroutineMiddleware: + async def process_spider_output(self, response, result, spider): + results = [] + for r in result: + results.append(r) + return results + + +class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): + + @defer.inlineCallbacks + def test_non_iterable(self): + with self.assertRaisesRegex( + _InvalidOutput, + ( + "\.process_spider_output must return an iterable, got " + ), + ): + yield self._get_middleware_result( + ProcessSpiderOutputNonIterableMiddleware, + ) + + @defer.inlineCallbacks + def test_coroutine(self): + with self.assertRaisesRegex( + _InvalidOutput, + "\.process_spider_output must be an asynchronous generator", + ): + yield self._get_middleware_result( + ProcessSpiderOutputCoroutineMiddleware, + ) + + class ProcessStartRequestsSimpleMiddleware: def process_start_requests(self, start_requests, spider): for r in start_requests: @@ -387,11 +426,6 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase): MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware - def _scrape_func(self, *args, **kwargs): - yield {'foo': 1} - yield {'foo': 2} - yield {'foo': 3} - @defer.inlineCallbacks def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None): setting = self._construct_mw_setting(*mw_classes, start_index=start_index) From b78e6915c6b259b31d3c37cae1529e85e797c964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 16 Mar 2022 20:17:25 +0100 Subject: [PATCH 065/237] Clarify that without async-to-sync conversions items yielded before an exception are processed --- docs/topics/coroutines.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 361cd5e60..efc4566a0 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -155,6 +155,9 @@ synchronous ``process_spider_output`` method gets a synchronous iterable as its ``Request`` callback or ``process_spider_output`` method, none of that output will be processed. + This contrasts with the regular behavior, where all items yielded before + an exception raises are processed. + Asynchronous-to-synchronous conversions are supported for backward compatibility, but they are deprecated and will stop working in a future version of Scrapy. From 5b4b8b6fb12874d4a0a11c261341639c9af95b10 Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Wed, 16 Mar 2022 22:32:05 +0100 Subject: [PATCH 066/237] added test for new url_has_any_extension function --- tests/test_utils_url.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 144c7bd76..58e2be622 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,6 +1,8 @@ import unittest +from scrapy.linkextractors import IGNORED_EXTENSIONS from scrapy.spiders import Spider +from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, @@ -8,9 +10,9 @@ from scrapy.utils.url import ( strip_url, url_is_from_any_domain, url_is_from_spider, + url_has_any_extension, ) - __doctests__ = ['scrapy.utils.url'] @@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + def test_url_has_any_extension(self): + deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)} + self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions)) + self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions)) + self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions)) + class AddHttpIfNoScheme(unittest.TestCase): From 0905d42e33871e976760d880316210d4953cd5df Mon Sep 17 00:00:00 2001 From: Yann Defretin Date: Thu, 17 Mar 2022 11:19:09 +0100 Subject: [PATCH 067/237] refactored url_has_any_extension function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/utils/url.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index bae5a9433..4d5e9ae82 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -31,7 +31,8 @@ def url_is_from_spider(url, spider): def url_has_any_extension(url, extensions): """Return True if the url ends with one of the extensions provided""" - return any(parse_url(url).path.lower().endswith(ext) for ext in extensions) + lowercase_path = parse_url(url).path.lower() + return any(lowercase_path.endswith(ext) for ext in extensions) def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already From b95c634b861bacc2b2ee3beeb5fcf64ab8607eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 17 Mar 2022 22:23:25 +0100 Subject: [PATCH 068/237] Document how to enforce Scrapy versions on Scrapy components --- docs/index.rst | 13 +++-- docs/topics/asyncio.rst | 24 +++++++++ docs/topics/components.rst | 84 +++++++++++++++++++++++++++++++ docs/topics/coroutines.rst | 6 +-- docs/topics/spider-middleware.rst | 8 +-- 5 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 docs/topics/components.rst diff --git a/docs/index.rst b/docs/index.rst index 75e08f537..6e22db884 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -229,10 +229,11 @@ Extending Scrapy topics/downloader-middleware topics/spider-middleware topics/extensions - topics/api topics/signals topics/scheduler topics/exporters + topics/components + topics/api :doc:`topics/architecture` @@ -247,9 +248,6 @@ Extending Scrapy :doc:`topics/extensions` Extend Scrapy with your custom functionality -:doc:`topics/api` - Use it on extensions and middlewares to extend Scrapy functionality - :doc:`topics/signals` See all available signals and how to work with them. @@ -259,6 +257,13 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/components` + Learn the common API and some good practices when building custom Scrapy + components. + +:doc:`topics/api` + Use it on extensions and middlewares to extend Scrapy functionality + All the rest ============ diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 3a6941a2c..dbee7146d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this: down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. + + +.. _enforce-asyncio-requirement: + +Enforcing asyncio as a requirement +================================== + +If you are writing a :ref:`component ` that requires asyncio +to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +:ref:`enforce it as a requirement `. For +example:: + + from scrapy.utils.reactor import is_asyncio_reactor_installed + + class MyComponent: + + def __init__(self): + if not is_asyncio_reactor_installed(): + raise ValueError( + f"{MyComponent.__qualname__} requires the asyncio Twisted " + f"reactor. Make sure you have it configured in the " + f"TWISTED_REACTOR setting. See the asyncio documentation " + f"of Scrapy for more information." + ) diff --git a/docs/topics/components.rst b/docs/topics/components.rst new file mode 100644 index 000000000..1fff2d61a --- /dev/null +++ b/docs/topics/components.rst @@ -0,0 +1,84 @@ +.. _topics-components: + +========== +Components +========== + +A Scrapy component is any class whose objects are created using +:func:`scrapy.utils.misc.create_instance`. + +That includes the classes that you may assign to the following settings: + +- :setting:`DNS_RESOLVER` + +- :setting:`DOWNLOAD_HANDLERS` + +- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` + +- :setting:`DOWNLOADER_MIDDLEWARES` + +- :setting:`DUPEFILTER_CLASS` + +- :setting:`EXTENSIONS` + +- :setting:`FEED_EXPORTERS` + +- :setting:`FEED_STORAGES` + +- :setting:`ITEM_PIPELINES` + +- :setting:`SCHEDULER` + +- :setting:`SCHEDULER_DISK_QUEUE` + +- :setting:`SCHEDULER_MEMORY_QUEUE` + +- :setting:`SCHEDULER_PRIORITY_QUEUE` + +- :setting:`SPIDER_MIDDLEWARES` + +Third-party Scrapy components may also let you define additional Scrapy +components, usually configurable through :ref:`settings `, to +modify their behavior. + +.. _enforce-component-requirements: + +Enforcing component requirements +================================ + +Sometimes, your components may only be intended to work under certain +conditions. For example, the may require a minimum version of Scrapy to work as +intended, or they may require certain settings to have specific values. + +In addition to describing those conditions in the documentation of your +component, it is a good practice to raise an exception from the ``__init__`` +method of your component if those conditions are not met at run time. + +In the case of :ref:`downloader middlewares `, +:ref:`extensions `, :ref:`item pipelines +`, and :ref:`spider middlewares +`, you should raise +:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a +parameter to the exception so that it is printed in the logs, for the user to +see. For other components, feel free to raise whatever other exception feels +right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy +version mismatch, while :exc:`ValueError` may be better if the issue is the +value of a setting. + +If your requirement is a minimum Scrapy version, you may use +:attr:`scrapy.__version__` to enforce your requirement. For example:: + + from pkg_resources import parse_version + + import scrapy + + class MyComponent: + + def __init__(self): + if parse_version(scrapy.__version__) < parse_version('VERSION'): + raise RuntimeError( + f"{MyComponent.__qualname__} requires Scrapy VERSION or " + f"later, which allow defining the process_spider_output " + f"method of spider middlewares as an asynchronous " + f"generator." + ) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index efc4566a0..55d013c06 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -162,7 +162,7 @@ Asynchronous-to-synchronous conversions are supported for backward compatibility, but they are deprecated and will stop working in a future version of Scrapy. -To avoid asynchronous-to-synchronous conversion, when defining ``Request`` +To avoid asynchronous-to-synchronous conversions, when defining ``Request`` callbacks as coroutine methods or when using spider middlewares whose ``process_spider_output`` method is an :term:`asynchronous generator`, all active spider middlewares must either have their ``process_spider_output`` @@ -177,8 +177,8 @@ process_spider_output_async method `. .. _universal-spider-middleware: -Universal spider middleware -=========================== +Universal spider middlewares +============================ .. versionadded:: VERSION diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 787545ed2..816cb5e03 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -111,9 +111,11 @@ object gives you access, for example, to the :ref:`settings `. Consider defining this method as an :term:`asynchronous generator`, which will be a requirement in a future version of Scrapy. However, if - you wish your spider middleware to work with Scrapy versions earlier - than Scrapy VERSION, :ref:`make your spider middleware universal - ` instead. + you plan on sharing your spider middleware with other people, consider + either :ref:`enforcing Scrapy VERSION ` + as a minimum requirement of your spider middleware, or :ref:`making + your spider middleware universal ` so that + it works with Scrapy versions earlier than Scrapy VERSION. :param response: the response which generated this output from the spider From 4af22bf157a5d25703cf5142097ba1c277f0d0d4 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 8 Apr 2022 14:26:23 +0500 Subject: [PATCH 069/237] Pin mitmproxy to < 8 for now (#5459) --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fcd3563b2..aba94d79d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,10 +12,11 @@ deps = -rtests/requirements.txt # mitmproxy does not support PyPy # mitmproxy does not support Windows when running Python < 3.7 - # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe + # Python 3.9+ requires mitmproxy >= 5.3.0 # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0 #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy' - mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' + # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454 + mitmproxy >= 4.0.4, < 8; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy' mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy' From bae3f8745589b09e0ea75fa463a5423546ca442c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Apr 2022 12:04:02 +0200 Subject: [PATCH 070/237] Cover a backward-incompatible Request serialization change in the 2.6 release notes --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 5d92067b5..e4e2bce3c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -113,6 +113,9 @@ Backward-incompatible changes meet expectations, :exc:`TypeError` is now raised at startup time. Before, other exceptions would be raised at run time. (:issue:`3559`) +- The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects + is now named ``encoding``, in line with all other fields (:issue:`5130`) + Deprecation removals ~~~~~~~~~~~~~~~~~~~~ From aead27bcbdf7c2a4d959dcb357c7f12cc8411739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Apr 2022 15:06:22 +0200 Subject: [PATCH 071/237] Add release notes for 2.6.2 (#5448) --- docs/news.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index e4e2bce3c..2e0b43455 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,23 @@ Release notes ============= +.. _release-2.6.2: + +Scrapy 2.6.2 (2022-03-15) +------------------------- + +Fixes additional regressions introduced in 2.6.0: + +- :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple + spiders (:issue:`5435`, :issue:`5436`) + +- Fixed an exception that was being logged after the spider finished under + certain conditions (:issue:`5437`, :issue:`5440`) + +- The ``--output``/``-o`` command-line parameter supports again a value + starting with a hyphen (:issue:`5444`, :issue:`5445`) + + .. _release-2.6.1: Scrapy 2.6.1 (2022-03-01) From 636127ec1ea2b8949438015c2167ab5d009ff1bf Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Apr 2022 12:01:24 -0700 Subject: [PATCH 072/237] tests that all CLI help messages don't throw errors --- tests/test_commands.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index b5e6c2b8b..76d5f3935 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -930,3 +930,17 @@ class MySpider(scrapy.Spider): args = ['-o', 'example1.json', '-O', 'example2.json'] log = self.get_log(spider_code, args=args) self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) + + +class HelpMessageTest(CommandTest): + + def setUp(self): + super().setUp() + self.commands = ["parse", "startproject", "view", "crawl", "edit", + "list", "fetch", "settings", "shell", "runspider", + "version", "genspider", "check", "bench"] + + def test_help_messages(self): + for command in self.commands: + _, out, _ = self.proc(command, "-h") + self.assertIn("Usage", out) From b0f5503cb8d0590aab4d9c91b0ee660d98d1e4a6 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 16 Apr 2022 18:17:47 -0700 Subject: [PATCH 073/237] Fixes Issue #5481 --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a3f6b96f4..6365fbdd0 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand): parser.add_argument("--cbkwargs", dest="cbkwargs", help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help="maximum depth for parsing requests [default: %default]") + help=f"maximum depth for parsing requests [default: {self.max_level}]") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") From 56c9098d6af2a57cc11927959d47253999cddd46 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 16 Apr 2022 18:33:12 -0700 Subject: [PATCH 074/237] changed default depth to 1 --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 6365fbdd0..a798ef945 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand): parser.add_argument("--cbkwargs", dest="cbkwargs", help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help=f"maximum depth for parsing requests [default: {self.max_level}]") + help="maximum depth for parsing requests [default: 1]") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") From 915c288205e2b9a0bdbbe18cc67cd23ba5bb4de3 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 17 Apr 2022 10:49:50 -0700 Subject: [PATCH 075/237] edit --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a798ef945..8e52d0d76 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -51,7 +51,7 @@ class Command(BaseRunSpiderCommand): parser.add_argument("--cbkwargs", dest="cbkwargs", help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_argument("-d", "--depth", dest="depth", type=int, default=1, - help="maximum depth for parsing requests [default: 1]") + help="maximum depth for parsing requests [default: %(default)s]") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one") From 7de9ed5bd42052f6491c939513adde6f6243ce2f Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Sun, 1 May 2022 01:34:35 +0300 Subject: [PATCH 076/237] Deprecate scrapy.pipelines.images.NoimagesDrop --- scrapy/pipelines/images.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 9c99dc69e..df4575a41 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -15,10 +15,12 @@ from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings +from scrapy.utils.decorators import deprecated from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes +@deprecated() class NoimagesDrop(DropItem): """Product with no images exception""" From 9f659bd63c04656f1c0601d6338f9050d5049fe8 Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Sun, 1 May 2022 13:36:15 +0300 Subject: [PATCH 077/237] Fix deprecation of scrapy.pipelines.images.NoimagesDrop --- scrapy/pipelines/images.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index df4575a41..c7a04a0cf 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -5,25 +5,28 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +import warnings from contextlib import suppress from io import BytesIO from itemadapter import ItemAdapter -from scrapy.exceptions import DropItem, NotConfigured +from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.pipelines.files import FileException, FilesPipeline # TODO: from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings -from scrapy.utils.decorators import deprecated from scrapy.utils.misc import md5sum from scrapy.utils.python import to_bytes -@deprecated() class NoimagesDrop(DropItem): """Product with no images exception""" + def __init__(self, *args, **kwargs): + warnings.warn("The NoimagesDrop class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) + super().__init__(*args, **kwargs) + class ImageException(FileException): """General image error exception""" From cc16af35af8b20e04b42ca84431820b2b092f379 Mon Sep 17 00:00:00 2001 From: PluT00 <314lut00@gmail.com> Date: Tue, 3 May 2022 11:29:21 +0300 Subject: [PATCH 078/237] Add deprecation warning test for scrapy.pipelines.images.NoimagesDrop --- tests/test_pipeline_images.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c69cd0e4a..613190f9c 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,14 +4,16 @@ import random from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +from warnings import catch_warnings import attr from itemadapter import ItemAdapter from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item -from scrapy.pipelines.images import ImagesPipeline +from scrapy.pipelines.images import ImagesPipeline, NoimagesDrop from scrapy.settings import Settings from scrapy.utils.python import to_bytes @@ -413,6 +415,22 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): expected_value) +class NoimagesDropTestCase(unittest.TestCase): + + def test_deprecation_warning(self): + arg = str() + with catch_warnings(record=True) as warnings: + NoimagesDrop(arg) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedNoimagesDrop(NoimagesDrop): + pass + SubclassedNoimagesDrop(arg) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + + def _create_image(format, *a, **kw): buf = io.BytesIO() Image.new(*a, **kw).save(buf, format) From 84c29a286f6b9bc94a5318ca68cd4fc21244443a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 20 May 2022 06:45:38 +0200 Subject: [PATCH 079/237] Unset the release date of still-unreleased 2.6.2 (#5503) --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 2e0b43455..ca9aeb783 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.6.2: -Scrapy 2.6.2 (2022-03-15) +Scrapy 2.6.2 (2022-0?-??) ------------------------- Fixes additional regressions introduced in 2.6.0: From 1c1cd5d8eae48eade88560426499846d217a555f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 20 May 2022 07:05:26 +0200 Subject: [PATCH 080/237] Update the 2.6.2 release notes --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index ca9aeb783..ffeb50390 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,6 +19,9 @@ Fixes additional regressions introduced in 2.6.0: - The ``--output``/``-o`` command-line parameter supports again a value starting with a hyphen (:issue:`5444`, :issue:`5445`) +- The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`, + :issue:`5482`) + .. _release-2.6.1: From ddfd192b704dddfefe2dd78345de239995a40159 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 11 Jun 2022 23:51:34 +0430 Subject: [PATCH 081/237] add tests for multiple headers with same name --- tests/test_http_headers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 64ff7a73d..0c51fd701 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -38,6 +38,13 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2']) assert h.getlist('X-Forwarded-For') is not hlist + def test_multivalue_for_one_header(self): + h = Headers((("a", "b"), ("a", "c"))) + self.assertEqual(h["a"], b"c") + self.assertEqual(h.get("a"), b"c") + self.assertEqual(h.getlist("a"), [b"b", b"c"]) + assert h.getlist("a") is not ["b", "c"] + def test_encode_utf8(self): h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() From 6a0bcf97cc6016cb966b92170709bc7518cf62c2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 11 Jun 2022 23:52:21 +0430 Subject: [PATCH 082/237] Merge values of multiple headers with same name (#5515) --- scrapy/http/headers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 1a2b99b0a..a9471d721 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,6 +1,7 @@ from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode +from collections.abc import Mapping class Headers(CaselessDict): @@ -10,6 +11,13 @@ class Headers(CaselessDict): self.encoding = encoding super().__init__(seq) + def update(self, seq): + seq = seq.items() if isinstance(seq, Mapping) else seq + iseq = {} + for k, v in seq: + iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v)) + super().update(iseq) + def normkey(self, key): """Normalize key to bytes""" return self._tobytes(key.title()) @@ -86,4 +94,5 @@ class Headers(CaselessDict): def __copy__(self): return self.__class__(self) + copy = __copy__ From a135d6caf050f4b7b5af28d4cccc5d5ef51dbaf6 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 13 Jun 2022 15:24:30 +0430 Subject: [PATCH 083/237] Move Mapping import line up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/http/headers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index a9471d721..9c03fe54f 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,8 @@ +from collections.abc import Mapping + from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode -from collections.abc import Mapping class Headers(CaselessDict): From 892c2a46554bdf80d49d3f28cc012c49cd1e19ca Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 13 Jun 2022 23:46:42 +0430 Subject: [PATCH 084/237] delete unnecessary test --- tests/test_http_headers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 0c51fd701..1ca936247 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -43,7 +43,6 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h["a"], b"c") self.assertEqual(h.get("a"), b"c") self.assertEqual(h.getlist("a"), [b"b", b"c"]) - assert h.getlist("a") is not ["b", "c"] def test_encode_utf8(self): h = Headers({'key': '\xa3'}, encoding='utf-8') From 3e994bda45ac4f95cec0517c4b1be68a38314914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 16 Jun 2022 20:53:14 +0200 Subject: [PATCH 085/237] Update for Python 3.7+ --- docs/topics/exporters.rst | 10 +--------- scrapy/exporters.py | 2 +- scrapy/settings/__init__.py | 9 ++++----- tests/test_exporters.py | 5 ++--- tests/test_feedexport.py | 10 +++------- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7580011ac..3c36ef002 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -205,7 +205,7 @@ BaseItemExporter ['field1', 'field2'] - - A dict [3]_ where keys are fields and values are output names:: + - A dict where keys are fields and values are output names:: {'field1': 'Field 1', 'field2': 'Field 2'} @@ -214,14 +214,6 @@ BaseItemExporter all their possible fields, exporters that do not support exporting a different subset of fields per item will only export the fields found in the first item exported. - .. [3] Dicts preserve insertion order since `Python 3.7`_ - (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version - of Python, use an OrderedDict_ to enforce a specific field order. - - .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html - .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation - .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html - .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict .. attribute:: export_empty_fields diff --git a/scrapy/exporters.py b/scrapy/exporters.py index ad12f26d6..76cbe4d4b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -2,13 +2,13 @@ Item Exporters are used to export/serialize items into different formats. """ -from collections import Mapping import csv import io import marshal import pickle import pprint import warnings +from collections.abc import Mapping from xml.sax.saxutils import XMLGenerator from itemadapter import is_item, ItemAdapter diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 2bbe38481..6cacc63e1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,6 +1,5 @@ import json import copy -from collections import OrderedDict from collections.abc import MutableMapping from importlib import import_module from pprint import pformat @@ -199,7 +198,7 @@ class BaseSettings(MutableMapping): return dict(value) def getdictorlist(self, name, default=None): - """Get a setting value as either an ``OrderedDict`` or a list. + """Get a setting value as either a :class:`dict` or a :class:`list`. If the setting is already a dict or a list, a copy of it will be returned. @@ -209,7 +208,7 @@ class BaseSettings(MutableMapping): For example, settings populated from the command line will return: - - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to + - ``{'key1': 'value1', 'key2': 'value2'}`` if set to ``'{"key1": "value1", "key2": "value2"}'`` - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'`` @@ -222,10 +221,10 @@ class BaseSettings(MutableMapping): """ value = self.get(name, default) if value is None: - return OrderedDict() + return {} if isinstance(value, str): try: - return json.loads(value, object_pairs_hook=OrderedDict) + return json.loads(value) except ValueError: return value.split(',') return copy.deepcopy(value) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 6ba7428f6..096cd3116 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,7 +4,6 @@ import marshal import pickle import tempfile import unittest -from collections import OrderedDict from io import BytesIO from datetime import datetime from warnings import catch_warnings, filterwarnings @@ -114,11 +113,11 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, 'John\xa3') ie = self._get_exporter( - fields_to_export=OrderedDict([('name', u'名稱')]) + fields_to_export={'name': '名稱'} ) self.assertEqual( list(ie._get_serialized_fields(self.i)), - [(u'名稱', u'John\xa3')] + [('名稱', 'John\xa3')] ) def test_field_custom_serializer(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 83aabbdc7..9098e035d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -11,7 +11,7 @@ import sys import tempfile import warnings from abc import ABC, abstractmethod -from collections import defaultdict, OrderedDict +from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger @@ -998,9 +998,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items_field_names(self): items = [{'foo': 'bar'}] - header = OrderedDict(( - ("foo", "Foo"), - )) + header = {'foo': 'Foo'} rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': header} yield self.assertExported(items, list(header.values()), rows, @@ -1023,9 +1021,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items_json_field_names(self): items = [{'foo': 'bar'}] - header = OrderedDict(( - ("foo", "Foo"), - )) + header = {'foo': 'Foo'} rows = [{'Foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': json.dumps(header)} yield self.assertExported(items, list(header.values()), rows, From 1b9ed22becf03311ec014dc9b7e0c09ce87b612c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:27:17 +0200 Subject: [PATCH 086/237] Remove Python < 3.7 leftover --- tests/test_feedexport.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9098e035d..946c94bd4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1004,8 +1004,6 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, list(header.values()), rows, settings=settings) - @pytest.mark.skipif(sys.version_info < (3, 7), - reason='Only official in Python 3.7+') @defer.inlineCallbacks def test_export_items_dict_field_names(self): items = [{'foo': 'bar'}] From 24f382fa459434cccfa4c0a8884a48d09d75e243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:31:45 +0200 Subject: [PATCH 087/237] test_feedexport: remove ordered=False --- tests/test_feedexport.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 946c94bd4..4006b5957 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -657,8 +657,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) + def assertExported(self, items, header, rows, settings=None): + yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -719,7 +719,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { @@ -730,10 +730,7 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) - if ordered: - self.assertEqual(reader.fieldnames, header) - else: - self.assertEqual(set(reader.fieldnames), set(header)) + self.assertEqual(reader.fieldnames, header) self.assertEqual(rows, got_rows) @@ -886,7 +883,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, ordered=False) + yield self.assertExported(items, header, rows) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -958,7 +955,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) + yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -968,7 +965,7 @@ class FeedExportTest(FeedExportTestBase): header = ["foo"] rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows, ordered=False) + yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks @@ -1146,7 +1143,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) + yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -2065,7 +2062,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { From 3729c6d26698ae6b8a7ef297606a1c7630d82619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:33:34 +0200 Subject: [PATCH 088/237] Remove unused import and redundant import --- tests/test_feedexport.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 4006b5957..8ef221b70 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -21,7 +21,6 @@ from unittest import mock from urllib.parse import urljoin, quote from urllib.request import pathname2url -import pytest import lxml.etree from testfixtures import LogCapture from twisted.internet import defer @@ -731,7 +730,6 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) self.assertEqual(reader.fieldnames, header) - self.assertEqual(rows, got_rows) @defer.inlineCallbacks @@ -1815,7 +1813,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): @defer.inlineCallbacks def test_lzma_plugin_filters(self): - import sys if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 raise unittest.SkipTest("lzma filters doesn't work in PyPy") From 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 08:55:45 +0200 Subject: [PATCH 089/237] Revert "test_feedexport: remove ordered=False" This reverts commit 24f382fa459434cccfa4c0a8884a48d09d75e243. --- tests/test_feedexport.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8ef221b70..fe90501fb 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None): - yield self.assertExportedCsv(items, header, rows, settings) + def assertExported(self, items, header, rows, settings=None, ordered=True): + yield self.assertExportedCsv(items, header, rows, settings, ordered) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} settings.update({ 'FEEDS': { @@ -729,7 +729,11 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) - self.assertEqual(reader.fieldnames, header) + if ordered: + self.assertEqual(reader.fieldnames, header) + else: + self.assertEqual(set(reader.fieldnames), set(header)) + self.assertEqual(rows, got_rows) @defer.inlineCallbacks @@ -881,7 +885,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows) + yield self.assertExported(items, header, rows, ordered=False) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -953,7 +957,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv) + yield self.assertExportedCsv(items, header, rows_csv, ordered=False) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -963,7 +967,7 @@ class FeedExportTest(FeedExportTestBase): header = ["foo"] rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows) + yield self.assertExportedCsv(items, header, rows, ordered=False) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks @@ -1141,7 +1145,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) + yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -2059,7 +2063,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} settings.update({ 'FEEDS': { From bc285f393ca8ff33ef715f98ef3367c973d23ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 09:00:39 +0200 Subject: [PATCH 090/237] Revert "Revert "test_feedexport: remove ordered=False"" This reverts commit 516e2d6ec0da77b8e0c01eb5188311b5fbeaa22e. --- tests/test_feedexport.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index fe90501fb..8ef221b70 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -656,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase): return data @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) + def assertExported(self, items, header, rows, settings=None): + yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) @@ -718,7 +718,7 @@ class FeedExportTest(FeedExportTestBase): return content @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { @@ -729,11 +729,7 @@ class FeedExportTest(FeedExportTestBase): reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) - if ordered: - self.assertEqual(reader.fieldnames, header) - else: - self.assertEqual(set(reader.fieldnames), set(header)) - + self.assertEqual(reader.fieldnames, header) self.assertEqual(rows, got_rows) @defer.inlineCallbacks @@ -885,7 +881,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'} ] header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, ordered=False) + yield self.assertExported(items, header, rows) @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): @@ -957,7 +953,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] - yield self.assertExportedCsv(items, header, rows_csv, ordered=False) + yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -967,7 +963,7 @@ class FeedExportTest(FeedExportTestBase): header = ["foo"] rows = [{'foo': 'bar'}] settings = {'FEED_EXPORT_FIELDS': []} - yield self.assertExportedCsv(items, header, rows, ordered=False) + yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) @defer.inlineCallbacks @@ -1145,7 +1141,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False) + yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks @@ -2063,7 +2059,7 @@ class BatchDeliveriesTest(FeedExportTestBase): self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { From ec5cf3e9cea3c66aca4cf1aad576f33edca3ad1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 09:10:18 +0200 Subject: [PATCH 091/237] test_feedexport: solve ordered comparison issues --- tests/test_feedexport.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8ef221b70..ec48f8d4a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -726,11 +726,9 @@ class FeedExportTest(FeedExportTestBase): }, }) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_unicode(data['csv']).splitlines()) - got_rows = list(reader) - self.assertEqual(reader.fieldnames, header) - self.assertEqual(rows, got_rows) + self.assertEqual(reader.fieldnames, list(header)) + self.assertEqual(rows, list(reader)) @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): @@ -1141,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase): {'egg': 'spam', 'foo': 'bar'} ] rows_jl = items - yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv) + yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) @defer.inlineCallbacks From 4ef71829b22b7362d08d6897090595138107852f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 10:37:27 +0200 Subject: [PATCH 092/237] If TWISTED_REACTOR is None, reuse any pre-installed reactor (#5528) --- docs/topics/settings.rst | 7 ++- scrapy/crawler.py | 3 +- scrapy/utils/reactor.py | 2 +- tests/CrawlerProcess/reactor_default.py | 17 +++++ .../reactor_default_twisted_reactor_select.py | 20 ++++++ tests/CrawlerProcess/reactor_select.py | 19 ++++++ ..._select_subclass_twisted_reactor_select.py | 31 +++++++++ .../reactor_select_twisted_reactor_select.py | 22 +++++++ tests/test_crawler.py | 63 +++++++++++++++++-- 9 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 tests/CrawlerProcess/reactor_default.py create mode 100644 tests/CrawlerProcess/reactor_default_twisted_reactor_select.py create mode 100644 tests/CrawlerProcess/reactor_select.py create mode 100644 tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py create mode 100644 tests/CrawlerProcess/reactor_select_twisted_reactor_select.py diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4e105642d..f3b28c4c4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1638,9 +1638,10 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will install the default reactor defined by Twisted for the -current platform. This is to maintain backward compatibility and avoid possible -problems caused by using a non-default reactor. +means that Scrapy will use the existing reactor if one is already installed, or +install the default reactor defined by Twisted for the current platform. This +is to maintain backward compatibility and avoid possible problems caused by +using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d669d93a8..dcf0c2146 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -78,8 +78,7 @@ class Crawler: if reactor_class: install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) else: - from twisted.internet import default - default.install() + from twisted.internet import reactor # noqa: F401 log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 96395543c..bc543b230 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -83,7 +83,7 @@ def verify_installed_reactor(reactor_path): path.""" from twisted.internet import reactor reactor_class = load_object(reactor_path) - if not isinstance(reactor, reactor_class): + if not reactor.__class__ == reactor_class: msg = ("The installed reactor " f"({reactor.__module__}.{reactor.__class__.__name__}) does not " f"match the requested one ({reactor_path})") diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py new file mode 100644 index 000000000..5a21a3717 --- /dev/null +++ b/tests/CrawlerProcess/reactor_default.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import reactor + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() + diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py new file mode 100644 index 000000000..c476722ef --- /dev/null +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -0,0 +1,20 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import reactor + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) + +process.crawl(NoRequestsSpider) +process.start() + + diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py new file mode 100644 index 000000000..eac6e2f89 --- /dev/null +++ b/tests/CrawlerProcess/reactor_select.py @@ -0,0 +1,19 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import selectreactor +selectreactor.install() + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() + + diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py new file mode 100644 index 000000000..47f480605 --- /dev/null +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -0,0 +1,31 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet.main import installReactor +from twisted.internet.selectreactor import SelectReactor + + +class SelectReactorSubclass(SelectReactor): + pass + + +reactor = SelectReactorSubclass() +installReactor(reactor) + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) + +process.crawl(NoRequestsSpider) +process.start() + + + + diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py new file mode 100644 index 000000000..e0d2dab26 --- /dev/null +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +from twisted.internet import selectreactor +selectreactor.install() + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) + +process.crawl(NoRequestsSpider) +process.start() + + + diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 957525382..1ff2e8a67 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -308,6 +308,57 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) self.assertNotIn("ReactorAlreadyInstalledError", log) + def test_reactor_default(self): + log = self.run_script('reactor_default.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + + def test_reactor_default_twisted_reactor_select(self): + log = self.run_script('reactor_default_twisted_reactor_select.py') + if platform.system() == 'Windows': + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + self.assertIn('Spider closed (finished)', log) + else: + self.assertNotIn('Spider closed (finished)', log) + self.assertIn( + ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ), + log, + ) + + def test_reactor_select(self): + log = self.run_script('reactor_select.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + + def test_reactor_select_twisted_reactor_select(self): + log = self.run_script('reactor_select_twisted_reactor_select.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + + def test_reactor_select_subclass_twisted_reactor_select(self): + log = self.run_script('reactor_select_subclass_twisted_reactor_select.py') + self.assertNotIn('Spider closed (finished)', log) + self.assertIn( + ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ), + log, + ) + def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -340,33 +391,33 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertNotIn("TimeoutError", log) self.assertNotIn("twisted.internet.error.DNSLookupError", log) - def test_reactor_select(self): + def test_twisted_reactor_select(self): log = self.run_script("twisted_reactor_select.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows") - def test_reactor_poll(self): + def test_twisted_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) - def test_reactor_asyncio(self): + def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_reactor_asyncio_custom_settings(self): + def test_twisted_reactor_asyncio_custom_settings(self): log = self.run_script("twisted_reactor_custom_settings.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_reactor_asyncio_custom_settings_same(self): + def test_twisted_reactor_asyncio_custom_settings_same(self): log = self.run_script("twisted_reactor_custom_settings_same.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_reactor_asyncio_custom_settings_conflict(self): + def test_twisted_reactor_asyncio_custom_settings_conflict(self): log = self.run_script("twisted_reactor_custom_settings_conflict.py") self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) From 54bfb9649bdec565f9798cc41643ed1bae25bd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 17 Jun 2022 11:51:02 +0200 Subject: [PATCH 093/237] Cover #5525 in the 2.6.2 release notes (#5535) --- docs/conf.py | 1 + docs/news.rst | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 55aa72d5a..9a0afe73e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -295,6 +295,7 @@ intersphinx_mapping = { 'twisted': ('https://twistedmatrix.com/documents/current', None), 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), } +intersphinx_disabled_reftypes = [] # Options for sphinx-hoverxref options diff --git a/docs/news.rst b/docs/news.rst index ffeb50390..7993b4b4f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,14 +5,19 @@ Release notes .. _release-2.6.2: -Scrapy 2.6.2 (2022-0?-??) -------------------------- +Scrapy 2.6.2 (to be determined) +------------------------------- Fixes additional regressions introduced in 2.6.0: - :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple spiders (:issue:`5435`, :issue:`5436`) +- Installing a Twisted reactor before Scrapy does (e.g. importing + :mod:`twisted.internet.reactor` somewhere at the module level) no longer + prevents Scrapy from starting, as long as a different reactor is not + specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`) + - Fixed an exception that was being logged after the spider finished under certain conditions (:issue:`5437`, :issue:`5440`) From e3e69d1209407c72a6478936bdbfd32cc22e9432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 20 Jun 2022 11:46:13 +0200 Subject: [PATCH 094/237] Pin documentation requirements (#5536) --- docs/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a0930ba1e..9f9aef711 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx>=3.0 -sphinx-hoverxref>=0.2b1 -sphinx-notfound-page>=0.4 -sphinx-rtd-theme>=0.5.2 \ No newline at end of file +sphinx==5.0.2 +sphinx-hoverxref==1.1.1 +sphinx-notfound-page==0.8 +sphinx-rtd-theme==1.0.0 From 387326fad42c4709933851108f2370d3105b8dd1 Mon Sep 17 00:00:00 2001 From: Vardhaman <83634399+cyai@users.noreply.github.com> Date: Thu, 23 Jun 2022 14:40:49 +0530 Subject: [PATCH 095/237] MAINT: Updated f-string format Updated the code with the f-string method for better and cleaner understanding. --- scrapy/downloadermiddlewares/cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 3afa06077..c592acb57 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -104,8 +104,8 @@ class CookiesMiddleware: for key in ("name", "value", "path", "domain"): if cookie.get(key) is None: if key in ("name", "value"): - msg = "Invalid cookie found in request {}: {} ('{}' is missing)" - logger.warning(msg.format(request, cookie, key)) + msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" + logger.warning(msg) return continue if isinstance(cookie[key], (bool, float, int, str)): From c4c816624fc4fd5fbc1866c507b661e92704136a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 30 Jun 2022 10:42:01 -0300 Subject: [PATCH 096/237] chore: Deprecate the `scrapy.downloadermiddlewares.decompression` module --- scrapy/downloadermiddlewares/decompression.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 0fcf8fb8c..98f18a836 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -9,10 +9,19 @@ import tarfile import zipfile from io import BytesIO from tempfile import mktemp +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes +warnings.warn( + 'scrapy.downloadermiddlewares.decompression is deprecated', + ScrapyDeprecationWarning, + stacklevel=2, +) + + logger = logging.getLogger(__name__) From fe08a119d965b2291e44801901e15b58a1f959ad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 30 Jun 2022 10:46:00 -0300 Subject: [PATCH 097/237] chore: import only used function --- scrapy/downloadermiddlewares/decompression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 98f18a836..e01e9cc76 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -9,13 +9,13 @@ import tarfile import zipfile from io import BytesIO from tempfile import mktemp -import warnings +from warnings import warn from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes -warnings.warn( +warn( 'scrapy.downloadermiddlewares.decompression is deprecated', ScrapyDeprecationWarning, stacklevel=2, From 09c3a4ad082dd6fc431be65975292d2eba369ad6 Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Tue, 12 Jul 2022 12:41:46 +0200 Subject: [PATCH 098/237] Fix doc: `scrapy.exporter` to `scrapy.exporters` --- docs/topics/exporters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 3c36ef002..9360ecf37 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -117,7 +117,7 @@ after your custom code. Example:: - from scrapy.exporter import XmlItemExporter + from scrapy.exporters import XmlItemExporter class ProductXmlExporter(XmlItemExporter): From 1c7ed4f2e59a94651d6d9d136cab78821cd3e80e Mon Sep 17 00:00:00 2001 From: Rotzbua Date: Thu, 6 Jan 2022 22:21:56 +0100 Subject: [PATCH 099/237] [doc] Remove incompatible web service project * Abandoned since 2017 * Not compatible with Python3 --- docs/index.rst | 4 ---- docs/topics/webservice.rst | 11 ----------- 2 files changed, 15 deletions(-) delete mode 100644 docs/topics/webservice.rst diff --git a/docs/index.rst b/docs/index.rst index 75e08f537..40c6cb485 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -130,7 +130,6 @@ Built-in services topics/stats topics/email topics/telnetconsole - topics/webservice :doc:`topics/logging` Learn how to use Python's builtin logging on Scrapy. @@ -144,9 +143,6 @@ Built-in services :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. -:doc:`topics/webservice` - Monitor and control a crawler using a web service. - Solving specific problems ========================= diff --git a/docs/topics/webservice.rst b/docs/topics/webservice.rst deleted file mode 100644 index 2c4052c04..000000000 --- a/docs/topics/webservice.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _topics-webservice: - -=========== -Web Service -=========== - -webservice has been moved into a separate project. - -It is hosted at: - - https://github.com/scrapy-plugins/scrapy-jsonrpc From 2f13f23d927900de0a89197ded0ee7aed387e351 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Fri, 15 Jul 2022 18:16:23 +0900 Subject: [PATCH 100/237] Fix typo in sep-014.rst requets -> requests --- sep/sep-014.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-014.rst b/sep/sep-014.rst index 0859e3f7c..2521aa0e5 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -590,11 +590,11 @@ Request Generator def generate_requests(self, response): """ - Extract and process new requets from response + Extract and process new requests from response """ requests = [] for ext in self._request_extractors: - requets.extend(ext.extract_requests(response)) + requests.extend(ext.extract_requests(response)) for proc in self._request_processors: requests = proc(requests) From 9b33b82a8b802c3906c2f1eaf1b88efee9b2fb09 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 17 Jul 2022 15:50:40 +0500 Subject: [PATCH 101/237] Fixed intersphinx references --- docs/conf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 378b01804..3241295af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -291,9 +291,9 @@ intersphinx_mapping = { 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current', None), - 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + 'tox': ('https://tox.wiki/en/latest/', None), + 'twisted': ('https://docs.twisted.org/en/stable/', None), + 'twistedapi': ('https://docs.twisted.org/en/stable/api/', None), 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), } intersphinx_disabled_reftypes = [] From 26c70318cb14806a07ee09d0283e9d5d306490e9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 17 Jul 2022 16:47:20 +0500 Subject: [PATCH 102/237] make Scrapy testing suite more robust in environments where non-existing hosts are resolvable --- tests/__init__.py | 10 ++++++++++ tests/test_command_shell.py | 4 +++- tests/test_crawl.py | 4 ++++ tests/test_downloader_handlers.py | 5 ++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..bb62851dc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests """ import os +import socket # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server @@ -25,6 +26,15 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') +# In some environments accessing a non-existing host doesn't raise an +# error. In such cases we're going to skip tests which rely on it. +try: + socket.getaddrinfo('non-existing-host', 80) + NON_EXISTING_RESOLVABLE = True +except socket.gaierror: + NON_EXISTING_RESOLVABLE = False + + def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 16c9559b5..33189e9be 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -6,7 +6,7 @@ from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from tests import tests_datadir +from tests import tests_datadir, NON_EXISTING_RESOLVABLE class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @@ -109,6 +109,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_dns_failures(self): + if NON_EXISTING_RESOLVABLE: + raise unittest.SkipTest("Non-existing hosts are resolvable") url = 'www.somedomainthatdoesntexi.st' errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 7bda3bef2..f9ffcd6bb 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -3,6 +3,7 @@ import logging from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse +import unittest from pytest import mark from testfixtures import LogCapture @@ -17,6 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode +from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, @@ -137,6 +139,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_dns_error(self): + if NON_EXISTING_RESOLVABLE: + raise unittest.SkipTest("Non-existing hosts are resolvable") crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a non-existent domain diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 72f52121e..883960084 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -4,7 +4,7 @@ import shutil import sys import tempfile from typing import Optional, Type -from unittest import mock +from unittest import mock, SkipTest from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal @@ -32,6 +32,7 @@ from scrapy.spiders import Spider from scrapy.utils.misc import create_instance from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, skip_if_no_boto +from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import ( Echo, ForeverTakingResource, @@ -791,6 +792,8 @@ class Http11ProxyTestCase(HttpProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): """ Test TunnelingTCP4ClientEndpoint """ + if NON_EXISTING_RESOLVABLE: + raise SkipTest("Non-existing hosts are resolvable") http_proxy = self.getURL('') domain = 'https://no-such-domain.nosuch' request = Request( From e248360e6e3dbb36fab185caf131707195fa6a26 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 18 Jul 2022 23:49:08 +0500 Subject: [PATCH 103/237] remove compatibility code from tests for the case dataclasses module is not available It was Python 3.6 compat code, and Python 3.6 support is dropped. --- tests/test_exporters.py | 32 ++++++++++++++--------------- tests/test_loader.py | 23 +++++++-------------- tests/test_pipeline_files.py | 37 ++++++++++++---------------------- tests/test_pipeline_images.py | 38 ++++++++++++----------------------- tests/test_utils_serialize.py | 18 +++++++---------- 5 files changed, 55 insertions(+), 93 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 096cd3116..69ac928c3 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,6 +4,7 @@ import marshal import pickle import tempfile import unittest +import dataclasses from io import BytesIO from datetime import datetime from warnings import catch_warnings, filterwarnings @@ -21,31 +22,30 @@ from scrapy.exporters import ( ) +def custom_serializer(value): + return str(int(value) + 2) + + class TestItem(Item): name = Field() age = Field() -def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): name = Field() age = Field(serializer=custom_serializer) -try: - from dataclasses import make_dataclass, field -except ImportError: - TestDataClass = None - CustomFieldDataclass = None -else: - TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) - CustomFieldDataclass = make_dataclass( - "CustomFieldDataclass", - [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] - ) +@dataclasses.dataclass +class TestDataClass: + name: str + age: int + + +@dataclasses.dataclass +class CustomFieldDataclass: + name: str + age: int = dataclasses.field(metadata={"serializer": custom_serializer}) class BaseItemExporterTest(unittest.TestCase): @@ -54,8 +54,6 @@ class BaseItemExporterTest(unittest.TestCase): custom_field_item_class = CustomFieldItem def setUp(self): - if self.item_class is None: - raise unittest.SkipTest("item class is None") self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() diff --git a/tests/test_loader.py b/tests/test_loader.py index f7ab1f236..c0937b349 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,4 +1,5 @@ import unittest +import dataclasses import attr from itemadapter import ItemAdapter @@ -10,13 +11,6 @@ from scrapy.loader import ItemLoader from scrapy.selector import Selector -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - # test items class NameItem(Item): name = Field() @@ -41,6 +35,11 @@ class AttrsNameItem: name = attr.ib(default="") +@dataclasses.dataclass +class TestDataClass: + name: list = dataclasses.field(default_factory=list) + + # test item loaders class NameItemLoader(ItemLoader): default_item_class = TestItem @@ -187,16 +186,8 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase item_class = AttrsNameItem -@unittest.skipIf(not make_dataclass, "dataclasses module is not available") class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "TestDataClass", - [("name", list, dataclass_field(default_factory=list))], - ) + item_class = TestDataClass class BaseNoInputReprocessingLoader(ItemLoader): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 4228173ed..5d381c018 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -7,6 +7,7 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import mock, skipIf from urllib.parse import urlparse +import dataclasses import attr from itemadapter import ItemAdapter @@ -32,13 +33,6 @@ from scrapy.utils.test import ( ) -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -226,24 +220,19 @@ class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest item_class = FilesPipelineTestItem -@skipIf(not make_dataclass, "dataclasses module is not available") -class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): +@dataclasses.dataclass +class FilesPipelineTestDataClass: + name: str + # default fields + file_urls: list = dataclasses.field(default_factory=list) + files: list = dataclasses.field(default_factory=list) + # overridden fields + custom_file_urls: list = dataclasses.field(default_factory=list) + custom_files: list = dataclasses.field(default_factory=list) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "FilesPipelineTestDataClass", - [ - ("name", str), - # default fields - ("file_urls", list, dataclass_field(default_factory=list)), - ("files", list, dataclass_field(default_factory=list)), - # overridden fields - ("custom_file_urls", list, dataclass_field(default_factory=list)), - ("custom_files", list, dataclass_field(default_factory=list)), - ], - ) + +class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = FilesPipelineTestDataClass @attr.s diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index dd94d296b..e6f5bea21 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,6 +4,7 @@ import random from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf +import dataclasses import attr from itemadapter import ItemAdapter @@ -16,13 +17,6 @@ from scrapy.settings import Settings from scrapy.utils.python import to_bytes -try: - from dataclasses import make_dataclass, field as dataclass_field -except ImportError: - make_dataclass = None - dataclass_field = None - - try: from PIL import Image except ImportError: @@ -203,25 +197,19 @@ class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unitte item_class = ImagesPipelineTestItem -@skipIf(not make_dataclass, "dataclasses module is not available") -class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): - item_class = None +@dataclasses.dataclass +class ImagesPipelineTestDataClass: + name: str + # default fields + image_urls: list = dataclasses.field(default_factory=list) + images: list = dataclasses.field(default_factory=list) + # overridden fields + custom_image_urls: list = dataclasses.field(default_factory=list) + custom_images: list = dataclasses.field(default_factory=list) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - if make_dataclass: - self.item_class = make_dataclass( - "FilesPipelineTestDataClass", - [ - ("name", str), - # default fields - ("image_urls", list, dataclass_field(default_factory=list)), - ("images", list, dataclass_field(default_factory=list)), - # overridden fields - ("custom_image_urls", list, dataclass_field(default_factory=list)), - ("custom_images", list, dataclass_field(default_factory=list)), - ], - ) + +class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase): + item_class = ImagesPipelineTestDataClass @attr.s diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py index daf022aee..a51de1877 100644 --- a/tests/test_utils_serialize.py +++ b/tests/test_utils_serialize.py @@ -1,6 +1,7 @@ import datetime import json import unittest +import dataclasses from decimal import Decimal import attr @@ -10,12 +11,6 @@ from scrapy.http import Request, Response from scrapy.utils.serialize import ScrapyJSONEncoder -try: - from dataclasses import make_dataclass -except ImportError: - make_dataclass = None - - class JsonEncoderTestCase(unittest.TestCase): def setUp(self): @@ -56,12 +51,13 @@ class JsonEncoderTestCase(unittest.TestCase): self.assertIn(r.url, rs) self.assertIn(str(r.status), rs) - @unittest.skipIf(not make_dataclass, "No dataclass support") def test_encode_dataclass_item(self): - TestDataClass = make_dataclass( - "TestDataClass", - [("name", str), ("url", str), ("price", int)], - ) + @dataclasses.dataclass + class TestDataClass: + name: str + url: str + price: int + item = TestDataClass(name="Product", url="http://product.org", price=1) encoded = self.encoder.encode(item) self.assertEqual( From 105468959363ee50b597038ac30fd32d3ea1b1f2 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 18 Jul 2022 23:53:30 +0500 Subject: [PATCH 104/237] remove unused imports thanks flake8! --- tests/test_pipeline_files.py | 2 +- tests/test_pipeline_images.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 5d381c018..d641e7a43 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -5,7 +5,7 @@ from datetime import datetime from io import BytesIO from shutil import rmtree from tempfile import mkdtemp -from unittest import mock, skipIf +from unittest import mock from urllib.parse import urlparse import dataclasses diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index e6f5bea21..0082e7a4e 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -3,7 +3,6 @@ import io import random from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf import dataclasses import attr From b103664bf45b079e5488b13a0737866de1b7dc50 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 19 Jul 2022 20:39:26 +0500 Subject: [PATCH 105/237] Address 2/3 of warnings from tests (#5561) --- pytest.ini | 3 + tests/test_contracts.py | 4 +- tests/test_crawl.py | 100 +++---- tests/test_crawler.py | 36 ++- tests/test_downloadermiddleware_httpauth.py | 14 +- tests/test_downloadermiddleware_httpproxy.py | 12 +- tests/test_downloadermiddleware_stats.py | 7 +- tests/test_dupefilters.py | 27 +- tests/test_engine.py | 228 ++++++++-------- tests/test_engine_stop_download_bytes.py | 32 ++- tests/test_engine_stop_download_headers.py | 32 ++- tests/test_feedexport.py | 269 +++++++------------ tests/test_logformatter.py | 6 +- tests/test_pipeline_crawl.py | 12 +- tests/test_request_attribute_binding.py | 21 +- tests/test_request_cb_kwargs.py | 5 +- tests/test_scheduler.py | 3 +- tests/test_scheduler_base.py | 15 +- tests/test_spiderloader/__init__.py | 5 +- tests/test_utils_project.py | 27 +- tests/test_utils_request.py | 13 +- tests/test_utils_response.py | 21 +- 22 files changed, 424 insertions(+), 468 deletions(-) diff --git a/pytest.ini b/pytest.ini index ae2ed2029..af0f2fb6e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,3 +21,6 @@ addopts = markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed +filterwarnings = + ignore:scrapy.downloadermiddlewares.decompression is deprecated + ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d0f4a68c2..136056f50 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -5,11 +5,11 @@ from twisted.python import failure from twisted.trial import unittest from scrapy import FormRequest -from scrapy.crawler import CrawlerRunner from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field +from scrapy.utils.test import get_crawler from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, @@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase): TestSameUrlSpider.parse_first.__doc__ = contract_doc TestSameUrlSpider.parse_second.__doc__ = contract_doc - crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + crawler = get_crawler(TestSameUrlSpider) yield crawler.crawl() self.assertEqual(crawler.spider.visited, 2) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f9ffcd6bb..59c271868 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,6 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode +from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( @@ -49,14 +50,13 @@ class CrawlTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() - self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_follow_all(self): - crawler = self.runner.create_crawler(FollowAllSpider) + crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @@ -79,7 +79,7 @@ class CrawlTestCase(TestCase): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} - crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) + crawler = get_crawler(FollowAllSpider, settings) yield crawler.crawl(**crawl_kwargs) times = crawler.spider.times total_time = times[-1] - times[0] @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): # of ``total`` and ``delay`` values that are too small for the test # code above to have any meaning. settings["DOWNLOAD_DELAY"] = 0 - crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) + crawler = get_crawler(FollowAllSpider, settings) yield crawler.crawl(**crawl_kwargs) times = crawler.spider.times total_time = times[-1] - times[0] @@ -102,7 +102,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): - crawler = self.runner.create_crawler(DelaySpider) + crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): - crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) + crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) @@ -125,14 +125,14 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(log) @@ -141,7 +141,7 @@ class CrawlTestCase(TestCase): def test_retry_dns_error(self): if NON_EXISTING_RESOLVABLE: raise unittest.SkipTest("Non-existing hosts are resolvable") - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a non-existent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) @@ -150,7 +150,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as log: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(log.records), 1) @@ -161,7 +161,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as log: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(log.records), 1) @@ -172,7 +172,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) + crawler = get_crawler(BrokenStartRequestsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) self.assertTrue( crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), @@ -181,7 +181,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) + crawler = get_crawler(DuplicateStartRequestsSpider, settings) yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) @@ -210,7 +210,7 @@ Connection: close foo body with multiples lines '''}) - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver) self.assertEqual(str(log).count("Got response 200"), 1) @@ -218,7 +218,7 @@ with multiples lines @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(log) @@ -226,7 +226,7 @@ with multiples lines @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(log) @@ -245,7 +245,7 @@ with multiples lines req0.meta['next'] = req1 req1.meta['next'] = req2 req2.meta['next'] = req3 - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) @@ -271,7 +271,7 @@ with multiples lines def cb(response): est.append(get_engine_status(crawler.engine)) - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) @@ -286,7 +286,7 @@ with multiples lines def cb(response): est.append(format_engine_status(crawler.engine)) - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) est = est[0].split("\n")[2:-2] # remove header & footer @@ -317,7 +317,7 @@ with multiples lines def start_requests(self): raise TestError - crawler = self.runner.create_crawler(FaultySpider) + crawler = get_crawler(FaultySpider) yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @@ -328,26 +328,28 @@ with multiples lines "tests.pipelines.ZeroDivisionErrorPipeline": 300, } } - crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider, settings) yield self.assertFailure( - self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), + crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks def test_crawlerrunner_accepts_crawler(self): - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) + runner = CrawlerRunner() with LogCapture() as log: - yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) - self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) + runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: - yield self.runner.join() + yield runner.join() self._assert_retried(log) self.assertIn("Got response 200", str(log)) @@ -358,7 +360,6 @@ class CrawlSpiderTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() - self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -370,7 +371,7 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - crawler = self.runner.create_crawler(spider_cls) + crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -378,10 +379,9 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_crawlspider_with_parse(self): - self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) - + crawler = get_crawler(CrawlSpiderWithParseMethod) with LogCapture() as log: - yield self.runner.join() + yield crawler.crawl(mockserver=self.mockserver) self.assertIn("[parse] status 200 (foo: None)", str(log)) self.assertIn("[parse] status 201 (foo: None)", str(log)) @@ -389,10 +389,9 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_crawlspider_with_errback(self): - self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver) - + crawler = get_crawler(CrawlSpiderWithErrback) with LogCapture() as log: - yield self.runner.join() + yield crawler.crawl(mockserver=self.mockserver) self.assertIn("[parse] status 200 (foo: None)", str(log)) self.assertIn("[parse] status 201 (foo: None)", str(log)) @@ -403,18 +402,19 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_async_def_parse(self): - self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: - yield self.runner.join() + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse(self): - runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"}) - runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + crawler = get_crawler(AsyncDefAsyncioSpider, { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + }) with LogCapture() as log: - yield runner.join() + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @mark.only_asyncio() @@ -433,7 +433,7 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider) + crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -479,14 +479,14 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_response_ssl_certificate_none(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta['responses'][0].certificate) @defer.inlineCallbacks def test_response_ssl_certificate(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta['responses'][0].certificate @@ -497,7 +497,7 @@ class CrawlSpiderTestCase(TestCase): @mark.xfail(reason="Responses with no body return early and contain no certificate") @defer.inlineCallbacks def test_response_ssl_certificate_empty_response(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200", is_secure=True) yield crawler.crawl(seed=url, mockserver=self.mockserver) cert = crawler.spider.meta['responses'][0].certificate @@ -507,7 +507,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_dns_server_ip_address_none(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta['responses'][0].ip_address @@ -515,7 +515,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_dns_server_ip_address(self): - crawler = self.runner.create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url('/echo?body=test') expected_netloc, _ = urlparse(url).netloc.split(':') yield crawler.crawl(seed=url, mockserver=self.mockserver) @@ -525,7 +525,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_bytes_received_stop_download_callback(self): - crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) + crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) @@ -534,7 +534,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_bytes_received_stop_download_errback(self): - crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) + crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) self.assertIsInstance(crawler.spider.meta["failure"], Failure) @@ -549,7 +549,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_headers_received_stop_download_callback(self): - crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider) + crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) @@ -557,7 +557,7 @@ class CrawlSpiderTestCase(TestCase): @defer.inlineCallbacks def test_headers_received_stop_download_errback(self): - crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider) + crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("response")) self.assertIsInstance(crawler.spider.meta["failure"], Failure) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f7aa769e4..d67abed7c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -13,11 +13,13 @@ from twisted.trial import unittest import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv @@ -34,9 +36,6 @@ class BaseCrawlerTest(unittest.TestCase): class CrawlerTestCase(BaseCrawlerTest): - def setUp(self): - self.crawler = Crawler(DefaultSpider, Settings()) - def test_populate_spidercls_settings(self): spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} project_settings = {'TEST1': 'project', 'TEST3': 'project'} @@ -46,7 +45,9 @@ class CrawlerTestCase(BaseCrawlerTest): settings = Settings() settings.setdict(project_settings, priority='project') - crawler = Crawler(CustomSettingsSpider, settings) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + crawler = Crawler(CustomSettingsSpider, settings) self.assertEqual(crawler.settings.get('TEST1'), 'spider') self.assertEqual(crawler.settings.get('TEST2'), 'spider') @@ -56,12 +57,14 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertTrue(crawler.settings.frozen) def test_crawler_accepts_dict(self): - crawler = Crawler(DefaultSpider, {'foo': 'bar'}) + crawler = get_crawler(DefaultSpider, {'foo': 'bar'}) self.assertEqual(crawler.settings['foo'], 'bar') self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') def test_crawler_accepts_None(self): - crawler = Crawler(DefaultSpider) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED') def test_crawler_rejects_spider_objects(self): @@ -77,7 +80,7 @@ class SpiderSettingsTestCase(unittest.TestCase): 'AUTOTHROTTLE_ENABLED': True } - crawler = Crawler(MySpider, {}) + crawler = get_crawler(MySpider) enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] self.assertIn(AutoThrottle, enabled_exts) @@ -91,7 +94,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): class MySpider(scrapy.Spider): name = 'spider' - Crawler(MySpider, {}) + get_crawler(MySpider) assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): @@ -111,7 +114,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): configure_logging() self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) - crawler = Crawler(MySpider, {}) + crawler = get_crawler(MySpider) self.assertEqual(get_scrapy_root_handler().level, logging.INFO) info_count = crawler.stats.get_value('log_count/INFO') logging.debug('debug message') @@ -148,7 +151,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): } configure_logging() - Crawler(MySpider, {}) + get_crawler(MySpider) logging.debug('debug message') with open(log_file, 'rb') as fo: @@ -229,22 +232,25 @@ class NoRequestsSpider(scrapy.Spider): @mark.usefixtures('reactor_pytest') class CrawlerRunnerHasSpider(unittest.TestCase): + def _runner(self): + return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): - runner = CrawlerRunner() + runner = self._runner() yield runner.crawl(NoRequestsSpider) self.assertEqual(runner.bootstrap_failed, False) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): - runner = CrawlerRunner() + runner = self._runner() yield runner.crawl(NoRequestsSpider) yield runner.crawl(NoRequestsSpider) self.assertEqual(runner.bootstrap_failed, False) @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed(self): - runner = CrawlerRunner() + runner = self._runner() try: yield runner.crawl(ExceptionSpider) @@ -257,7 +263,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): @defer.inlineCallbacks def test_crawler_runner_bootstrap_failed_for_several(self): - runner = CrawlerRunner() + runner = self._runner() try: yield runner.crawl(ExceptionSpider) @@ -275,12 +281,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", }) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 0362e2018..b9f3e24a4 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,7 +1,9 @@ import unittest +import pytest from w3lib.http import basic_auth_header +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.spiders import Spider @@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.spider = TestSpiderLegacy('foo') def test_auth(self): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) + with pytest.warns(ScrapyDeprecationWarning, + match="Using HttpAuthMiddleware without http_auth_domain is deprecated"): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) # initial request, sets the domain and sends the header req = Request('http://example.com/') @@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase): self.assertNotIn('Authorization', req.headers) def test_auth_already_set(self): - mw = HttpAuthMiddleware() - mw.spider_opened(self.spider) + with pytest.warns(ScrapyDeprecationWarning, + match="Using HttpAuthMiddleware without http_auth_domain is deprecated"): + mw = HttpAuthMiddleware() + mw.spider_opened(self.spider) req = Request('http://example.com/', headers=dict(Authorization='Digest 123')) assert mw.process_request(req, self.spider) is None diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7c97bf32a..4ac85c1ec 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,13 +1,13 @@ import os -from functools import partial + +import pytest from twisted.trial.unittest import TestCase from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.crawler import Crawler -from scrapy.settings import Settings +from scrapy.utils.test import get_crawler spider = Spider('foo') @@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase): os.environ = self._oldenv def test_not_enabled(self): - settings = Settings({'HTTPPROXY_ENABLED': False}) - crawler = Crawler(Spider, settings) - self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) + crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False}) + with pytest.raises(NotConfigured): + HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 9e75f0a50..7d88ba4d2 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,7 +1,9 @@ +import warnings from itertools import product from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.response import response_httprepr @@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase): for test_response in test_responses: self.crawler.stats.set_value('downloader/response_bytes', 0) self.mw.process_response(self.req, test_response, self.spider) - self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response))) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + resp_size = len(response_httprepr(test_response)) + self.assertStatsEqual('downloader/response_bytes', resp_size) def test_process_exception(self): self.mw.process_exception(self.req, MyException(), self.spider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index b7df2554a..8a37a8ebe 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -10,7 +10,6 @@ from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes -from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -29,8 +28,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod def from_crawler(cls, crawler): - debug = crawler.settings.getbool('DUPEFILTER_DEBUG') - df = cls(job_dir(crawler.settings), debug) + df = super().from_crawler(crawler) df.method = 'from_crawler' return df @@ -38,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): class FromSettingsRFPDupeFilter(RFPDupeFilter): @classmethod - def from_settings(cls, settings): - debug = settings.getbool('DUPEFILTER_DEBUG') - df = cls(job_dir(settings), debug) + def from_settings(cls, settings, *, fingerprinter=None): + df = super().from_settings(settings, fingerprinter=fingerprinter) df.method = 'from_settings' return df @@ -53,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -61,14 +59,16 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter} + 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': DirectDupeFilter} + settings = {'DUPEFILTER_CLASS': DirectDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -171,7 +171,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -197,7 +198,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter} + 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -230,7 +232,8 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug_default_dupefilter(self): with LogCapture() as log: - settings = {'DUPEFILTER_DEBUG': True} + settings = {'DUPEFILTER_DEBUG': True, + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) diff --git a/tests/test_engine.py b/tests/test_engine.py index fa7d0c8d4..1bd802bcf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,10 +13,11 @@ module with the ``runserver`` argument:: import os import re import sys -import warnings from collections import defaultdict from urllib.parse import urlparse +from dataclasses import dataclass +import pytest import attr from itemadapter import ItemAdapter from pydispatch import dispatcher @@ -50,6 +51,13 @@ class AttrsItem: price = attr.ib(default=0) +@dataclass +class DataClassItem: + name: str = "" + url: str = "" + price: int = 0 + + class TestSpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] @@ -92,17 +100,8 @@ class AttrsItemsSpider(TestSpider): item_cls = AttrsItem -try: - from dataclasses import make_dataclass -except ImportError: - DataClassItemsSpider = None -else: - TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)]) - - class DataClassItemsSpider(DictItemsSpider): # type: ignore[no-redef] - def parse_item(self, response): - item = super().parse_item(response) - return TestDataClass(**item) +class DataClassItemsSpider(TestSpider): + item_cls = DataClassItem class ItemZeroDivisionErrorSpider(TestSpider): @@ -188,7 +187,7 @@ class CrawlerRun: return self.deferred def stop(self): - self.port.stopListening() + self.port.stopListening() # FIXME: wait for this Deferred for name, signal in vars(signals).items(): if not name.startswith('_'): disconnect_all(signal) @@ -239,79 +238,77 @@ class EngineTest(unittest.TestCase): def test_crawler(self): for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): - if spider is None: - continue - self.run = CrawlerRun(spider) - yield self.run.run() - self._assert_visited_urls() - self._assert_scheduled_requests(count=9) - self._assert_downloaded_responses(count=9) - self._assert_scraped_items() - self._assert_signals_caught() - self._assert_bytes_received() + run = CrawlerRun(spider) + yield run.run() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_scraped_items(run) + self._assert_signals_caught(run) + self._assert_bytes_received(run) @defer.inlineCallbacks def test_crawler_dupefilter(self): - self.run = CrawlerRun(TestDupeFilterSpider) - yield self.run.run() - self._assert_scheduled_requests(count=8) - self._assert_dropped_requests() + run = CrawlerRun(TestDupeFilterSpider) + yield run.run() + self._assert_scheduled_requests(run, count=8) + self._assert_dropped_requests(run) @defer.inlineCallbacks def test_crawler_itemerror(self): - self.run = CrawlerRun(ItemZeroDivisionErrorSpider) - yield self.run.run() - self._assert_items_error() + run = CrawlerRun(ItemZeroDivisionErrorSpider) + yield run.run() + self._assert_items_error(run) @defer.inlineCallbacks def test_crawler_change_close_reason_on_idle(self): - self.run = CrawlerRun(ChangeCloseReasonSpider) - yield self.run.run() - self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, - self.run.signals_caught[signals.spider_closed]) + run = CrawlerRun(ChangeCloseReasonSpider) + yield run.run() + self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'}, + run.signals_caught[signals.spider_closed]) - def _assert_visited_urls(self): + def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] - urls_visited = {rp[0].url for rp in self.run.respplug} - urls_expected = {self.run.geturl(p) for p in must_be_visited} + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" - def _assert_scheduled_requests(self, count=None): - self.assertEqual(count, len(self.run.reqplug)) + def _assert_scheduled_requests(self, run: CrawlerRun, count=None): + self.assertEqual(count, len(run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] - urls_requested = {rq[0].url for rq in self.run.reqplug} - urls_expected = {self.run.geturl(p) for p in paths_expected} + urls_requested = {rq[0].url for rq in run.reqplug} + urls_expected = {run.geturl(p) for p in paths_expected} assert urls_expected <= urls_requested - scheduled_requests_count = len(self.run.reqplug) - dropped_requests_count = len(self.run.reqdropped) - responses_count = len(self.run.respplug) + scheduled_requests_count = len(run.reqplug) + dropped_requests_count = len(run.reqdropped) + responses_count = len(run.respplug) self.assertEqual(scheduled_requests_count, dropped_requests_count + responses_count) - self.assertEqual(len(self.run.reqreached), + self.assertEqual(len(run.reqreached), responses_count) - def _assert_dropped_requests(self): - self.assertEqual(len(self.run.reqdropped), 1) + def _assert_dropped_requests(self, run: CrawlerRun): + self.assertEqual(len(run.reqdropped), 1) - def _assert_downloaded_responses(self, count): + def _assert_downloaded_responses(self, run: CrawlerRun, count): # response tests - self.assertEqual(count, len(self.run.respplug)) - self.assertEqual(count, len(self.run.reqreached)) + self.assertEqual(count, len(run.respplug)) + self.assertEqual(count, len(run.reqreached)) - for response, _ in self.run.respplug: - if self.run.getpath(response.url) == '/item999.html': + for response, _ in run.respplug: + if run.getpath(response.url) == '/item999.html': self.assertEqual(404, response.status) - if self.run.getpath(response.url) == '/redirect': + if run.getpath(response.url) == '/redirect': self.assertEqual(302, response.status) - def _assert_items_error(self): - self.assertEqual(2, len(self.run.itemerror)) - for item, response, spider, failure in self.run.itemerror: + def _assert_items_error(self, run: CrawlerRun): + self.assertEqual(2, len(run.itemerror)) + for item, response, spider, failure in run.itemerror: self.assertEqual(failure.value.__class__, ZeroDivisionError) - self.assertEqual(spider, self.run.spider) + self.assertEqual(spider, run.spider) self.assertEqual(item['url'], response.url) if 'item1.html' in item['url']: @@ -321,9 +318,9 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_scraped_items(self): - self.assertEqual(2, len(self.run.itemresp)) - for item, response in self.run.itemresp: + def _assert_scraped_items(self, run: CrawlerRun): + self.assertEqual(2, len(run.itemresp)) + for item, response in run.itemresp: item = ItemAdapter(item) self.assertEqual(item['url'], response.url) if 'item1.html' in item['url']: @@ -333,26 +330,26 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_headers_received(self): - for headers in self.run.headers.values(): + def _assert_headers_received(self, run: CrawlerRun): + for headers in run.headers.values(): self.assertIn(b"Server", headers) self.assertIn(b"TwistedWeb", headers[b"Server"]) self.assertIn(b"Date", headers) self.assertIn(b"Content-Type", headers) - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): + def _assert_bytes_received(self, run: CrawlerRun): + self.assertEqual(9, len(run.bytes)) + for request, data in run.bytes.items(): joined_data = b"".join(data) - if self.run.getpath(request.url) == "/": + if run.getpath(request.url) == "/": self.assertEqual(joined_data, get_testdata("test_site", "index.html")) - elif self.run.getpath(request.url) == "/item1.html": + elif run.getpath(request.url) == "/item1.html": self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) - elif self.run.getpath(request.url) == "/item2.html": + elif run.getpath(request.url) == "/item2.html": self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) - elif self.run.getpath(request.url) == "/redirected": + elif run.getpath(request.url) == "/redirected": self.assertEqual(joined_data, b"Redirected here") - elif self.run.getpath(request.url) == '/redirect': + elif run.getpath(request.url) == '/redirect': self.assertEqual( joined_data, b"\n\n" @@ -364,7 +361,7 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) - elif self.run.getpath(request.url) == "/tem999.html": + elif run.getpath(request.url) == "/tem999.html": self.assertEqual( joined_data, b"\n\n" @@ -375,27 +372,27 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) - elif self.run.getpath(request.url) == "/numbers": + elif run.getpath(request.url) == "/numbers": # signal was fired multiple times self.assertTrue(len(data) > 1) # bytes were received in order numbers = [str(x).encode("utf8") for x in range(2**18)] self.assertEqual(joined_data, b"".join(numbers)) - def _assert_signals_caught(self): - assert signals.engine_started in self.run.signals_caught - assert signals.engine_stopped in self.run.signals_caught - assert signals.spider_opened in self.run.signals_caught - assert signals.spider_idle in self.run.signals_caught - assert signals.spider_closed in self.run.signals_caught - assert signals.headers_received in self.run.signals_caught + def _assert_signals_caught(self, run: CrawlerRun): + assert signals.engine_started in run.signals_caught + assert signals.engine_stopped in run.signals_caught + assert signals.spider_opened in run.signals_caught + assert signals.spider_idle in run.signals_caught + assert signals.spider_closed in run.signals_caught + assert signals.headers_received in run.signals_caught - self.assertEqual({'spider': self.run.spider}, - self.run.signals_caught[signals.spider_opened]) - self.assertEqual({'spider': self.run.spider}, - self.run.signals_caught[signals.spider_idle]) - self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, - self.run.signals_caught[signals.spider_closed]) + self.assertEqual({'spider': run.spider}, + run.signals_caught[signals.spider_opened]) + self.assertEqual({'spider': run.spider}, + run.signals_caught[signals.spider_idle]) + self.assertEqual({'spider': run.spider, 'reason': 'finished'}, + run.signals_caught[signals.spider_closed]) @defer.inlineCallbacks def test_close_downloader(self): @@ -407,28 +404,29 @@ class EngineTest(unittest.TestCase): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") - ) - yield e.stop() + try: + yield self.assertFailure(e.start(), RuntimeError).addBoth( + lambda exc: self.assertEqual(str(exc), "Engine already running") + ) + finally: + yield e.stop() @defer.inlineCallbacks def test_close_spiders_downloader(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.open_spiders is deprecated, " + "please use ExecutionEngine.spider instead"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) self.assertEqual(len(e.open_spiders), 1) yield e.close() self.assertEqual(len(e.open_spiders), 0) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - ) @defer.inlineCallbacks def test_close_engine_spiders_downloader(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.open_spiders is deprecated, " + "please use ExecutionEngine.spider instead"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() @@ -436,61 +434,47 @@ class EngineTest(unittest.TestCase): yield e.close() self.assertFalse(e.running) self.assertEqual(len(e.open_spiders), 0) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead", - ) @defer.inlineCallbacks def test_crawl_deprecated_spider_arg(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="Passing a 'spider' argument to " + "ExecutionEngine.crawl is deprecated"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) e.start() e.crawl(Request("data:,"), spider) yield e.close() - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated", - ) @defer.inlineCallbacks def test_download_deprecated_spider_arg(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="Passing a 'spider' argument to " + "ExecutionEngine.download is deprecated"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) e.start() e.download(Request("data:,"), spider) yield e.close() - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "Passing a 'spider' argument to ExecutionEngine.download is deprecated", - ) @defer.inlineCallbacks def test_deprecated_schedule(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.schedule is deprecated, please use " + "ExecutionEngine.crawl or ExecutionEngine.download instead"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) spider = TestSpider() yield e.open_spider(spider, []) e.start() e.schedule(Request("data:,"), spider) yield e.close() - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual( - str(warning_list[0].message), - "ExecutionEngine.schedule is deprecated, please use " - "ExecutionEngine.crawl or ExecutionEngine.download instead", - ) @defer.inlineCallbacks def test_deprecated_has_capacity(self): - with warnings.catch_warnings(record=True) as warning_list: + with pytest.warns(ScrapyDeprecationWarning, + match="ExecutionEngine.has_capacity is deprecated"): e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) self.assertTrue(e.has_capacity()) spider = TestSpider() @@ -499,8 +483,6 @@ class EngineTest(unittest.TestCase): e.start() yield e.close() self.assertTrue(e.has_capacity()) - self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) - self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") if __name__ == "__main__": diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index 0ba69e096..933e4067d 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): - if spider is None: - continue - self.run = BytesReceivedCrawlerRun(spider) + run = BytesReceivedCrawlerRun(spider) with LogCapture() as log: - yield self.run.run() + yield run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for " + f"Download stopped for " "from signal handler BytesReceivedCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for " + f"Download stopped for " "from signal handler BytesReceivedCrawlerRun.bytes_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for " + f"Download stopped for " "from signal handler BytesReceivedCrawlerRun.bytes_received")) - self._assert_visited_urls() - self._assert_scheduled_requests(count=9) - self._assert_downloaded_responses(count=9) - self._assert_signals_caught() - self._assert_headers_received() - self._assert_bytes_received() + self._assert_visited_urls(run) + self._assert_scheduled_requests(run, count=9) + self._assert_downloaded_responses(run, count=9) + self._assert_signals_caught(run) + self._assert_headers_received(run) + self._assert_bytes_received(run) - def _assert_bytes_received(self): - self.assertEqual(9, len(self.run.bytes)) - for request, data in self.run.bytes.items(): + def _assert_bytes_received(self, run: CrawlerRun): + self.assertEqual(9, len(run.bytes)) + for request, data in run.bytes.items(): joined_data = b"".join(data) self.assertTrue(len(data) == 1) # signal was fired only once - if self.run.getpath(request.url) == "/numbers": + if run.getpath(request.url) == "/numbers": # Received bytes are not the complete response. The exact amount depends # on the buffer size, which can vary, so we only check that the amount # of received bytes is strictly less than the full response. diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index fad6643ad..8975d0e3f 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest): @defer.inlineCallbacks def test_crawler(self): for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider): - if spider is None: - continue - self.run = HeadersReceivedCrawlerRun(spider) + run = HeadersReceivedCrawlerRun(spider) with LogCapture() as log: - yield self.run.run() + yield run.run() log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for from" + f"Download stopped for from" " signal handler HeadersReceivedCrawlerRun.headers_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for from signal" + f"Download stopped for from signal" " handler HeadersReceivedCrawlerRun.headers_received")) log.check_present(("scrapy.core.downloader.handlers.http11", "DEBUG", - f"Download stopped for from" + f"Download stopped for from" " signal handler HeadersReceivedCrawlerRun.headers_received")) - self._assert_visited_urls() - self._assert_downloaded_responses(count=6) - self._assert_signals_caught() - self._assert_bytes_received() - self._assert_headers_received() + self._assert_visited_urls(run) + self._assert_downloaded_responses(run, count=6) + self._assert_signals_caught(run) + self._assert_bytes_received(run) + self._assert_headers_received(run) - def _assert_bytes_received(self): - self.assertEqual(0, len(self.run.bytes)) + def _assert_bytes_received(self, run: CrawlerRun): + self.assertEqual(0, len(run.bytes)) - def _assert_visited_urls(self): + def _assert_visited_urls(self, run: CrawlerRun): must_be_visited = ["/", "/redirect", "/redirected"] - urls_visited = {rp[0].url for rp in self.run.respplug} - urls_expected = {self.run.geturl(p) for p in must_be_visited} + urls_visited = {rp[0].url for rp in run.respplug} + urls_expected = {run.geturl(p) for p in must_be_visited} assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ec48f8d4a..ecd1b59d3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -22,6 +22,7 @@ from urllib.parse import urljoin, quote from urllib.request import pathname2url import lxml.etree +import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial import unittest @@ -30,7 +31,6 @@ from zope.interface import implementer from zope.interface.verify import verifyObject import scrapy -from scrapy.crawler import CrawlerRunner from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( @@ -697,9 +697,9 @@ class FeedExportTest(FeedExportTestBase): content = {} try: with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): @@ -1554,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): content = {} try: with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for file_path, feed_options in FEEDS.items(): if not os.path.exists(str(file_path)): @@ -2026,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase): content = defaultdict(list) try: with MockServer() as s: - runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] - yield runner.crawl(spider_cls) + crawler = get_crawler(spider_cls, settings) + yield crawler.crawl() for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) @@ -2048,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -2064,7 +2064,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -2080,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -2098,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -2123,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -2140,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') + batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -2166,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=Settings(settings)) + yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ @@ -2382,9 +2382,9 @@ class BatchDeliveriesTest(FeedExportTestBase): yield item with MockServer() as server: - runner = CrawlerRunner(Settings(settings)) TestSpider.start_urls = [server.url('/')] - yield runner.crawl(TestSpider) + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl() self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1) for stub in CustomS3FeedStorage.stubs[:-1]: @@ -2434,25 +2434,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': StdoutFeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") - with warnings.catch_warnings(record=True) as w: + with pytest.warns(ScrapyDeprecationWarning, + match="StdoutFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "StdoutFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class FileFeedStorageWithoutFeedOptions(FileFeedStorage): @@ -2476,25 +2467,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': FileFeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) spider = scrapy.Spider("default") - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="FileFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "FileFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class S3FeedStorageWithoutFeedOptions(S3FeedStorage): @@ -2524,26 +2506,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': S3FeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="S3FeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "S3FeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) def test_from_crawler(self): settings_dict = { @@ -2552,26 +2526,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " - "does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage): @@ -2601,26 +2567,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': FTPFeedStorageWithoutFeedOptions }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="FTPFeedStorageWithoutFeedOptions does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "FTPFeedStorageWithoutFeedOptions does not support " - "the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) def test_from_crawler(self): settings_dict = { @@ -2629,50 +2587,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase): 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler }, } - crawler = get_crawler(settings_dict=settings_dict) - feed_exporter = FeedExporter.from_crawler(crawler) + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings_dict) + feed_exporter = FeedExporter.from_crawler(crawler) + spider = scrapy.Spider("default") spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support " + "the 'feed_options' keyword argument."): feed_exporter.open_spider(spider) - messages = tuple(str(item.message) for item in w - if item.category is ScrapyDeprecationWarning) - self.assertEqual( - messages, - ( - ( - "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler " - "does not support the 'feed_options' keyword argument. Add a " - "'feed_options' parameter to its signature to remove " - "this warning. This parameter will become mandatory " - "in a future version of Scrapy." - ), - ) - ) class URIParamsTest: spider_name = "uri_params_spider" + deprecated_options = False def build_settings(self, uri='file:///tmp/foobar', uri_params=None): raise NotImplementedError + def _crawler_feed_exporter(self, settings): + if self.deprecated_options: + with pytest.warns(ScrapyDeprecationWarning, + match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"): + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + else: + crawler = get_crawler(settings_dict=settings) + feed_exporter = FeedExporter.from_crawler(crawler) + return crawler, feed_exporter + def test_default(self): settings = self.build_settings( uri='file:///tmp/%(name)s', ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) self.assertEqual( feed_exporter.slots[0].uri, @@ -2687,28 +2645,13 @@ class URIParamsTest: uri='file:///tmp/%(name)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with pytest.warns(ScrapyDeprecationWarning, + match="Modifying the params dictionary in-place"): feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual( - messages, - ( - ( - 'Modifying the params dictionary in-place in the ' - 'function defined in the FEED_URI_PARAMS setting or ' - 'in the uri_params key of the FEEDS setting is ' - 'deprecated. The function must return a new ' - 'dictionary instead.' - ), - ) - ) self.assertEqual( feed_exporter.slots[0].uri, @@ -2723,18 +2666,14 @@ class URIParamsTest: uri='file:///tmp/%(name)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) with self.assertRaises(KeyError): feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) def test_params_as_is(self): def uri_params(params, spider): @@ -2744,17 +2683,12 @@ class URIParamsTest: uri='file:///tmp/%(name)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) self.assertEqual( feed_exporter.slots[0].uri, @@ -2769,17 +2703,12 @@ class URIParamsTest: uri='file:///tmp/%(foo)s', uri_params=uri_params, ) - crawler = get_crawler(settings_dict=settings) - feed_exporter = FeedExporter.from_crawler(crawler) + crawler, feed_exporter = self._crawler_feed_exporter(settings) spider = scrapy.Spider(self.spider_name) spider.crawler = crawler - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) feed_exporter.open_spider(spider) - messages = tuple( - str(item.message) for item in w - if item.category is ScrapyDeprecationWarning - ) - self.assertEqual(messages, tuple()) self.assertEqual( feed_exporter.slots[0].uri, @@ -2788,6 +2717,7 @@ class URIParamsTest: class URIParamsSettingTest(URIParamsTest, unittest.TestCase): + deprecated_options = True def build_settings(self, uri='file:///tmp/foobar', uri_params=None): extra_settings = {} @@ -2800,6 +2730,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase): class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase): + deprecated_options = False def build_settings(self, uri='file:///tmp/foobar', uri_params=None): options = { diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 6381f895b..f3bb23bda 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -5,8 +5,8 @@ from twisted.internet import defer from twisted.python.failure import Failure from twisted.trial.unittest import TestCase as TwistedTestCase -from scrapy.crawler import CrawlerRunner from scrapy.exceptions import DropItem +from scrapy.utils.test import get_crawler from scrapy.http import Request, Response from scrapy.item import Item, Field from scrapy.logformatter import LogFormatter @@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): @defer.inlineCallbacks def test_show_messages(self): - crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider) + crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc)) @@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase): def test_skip_messages(self): settings = self.base_settings.copy() settings['LOG_FORMATTER'] = SkipMessagesLogFormatter - crawler = CrawlerRunner(settings).create_crawler(ItemSpider) + crawler = get_crawler(ItemSpider, settings) with LogCapture() as lc: yield crawler.crawl(mockserver=self.mockserver) self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc)) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index f49fda701..e46532a1c 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'ITEM_PIPELINES': {self.pipeline_class: 1}, self.store_setting_key: self.tmpmediastore, } @@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, **kwargs): - crawler = self.runner.create_crawler(spider_class, **kwargs) + def _create_crawler(self, spider_class, runner=None, **kwargs): + if runner is None: + runner = self.runner + crawler = runner.create_crawler(spider_class, **kwargs) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler @@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_allowed(self): settings = dict(self.settings) settings.update({'MEDIA_ALLOW_REDIRECTS': True}) - self.runner = CrawlerRunner(settings) - - crawler = self._create_crawler(RedirectedMediaDownloadSpider) + runner = CrawlerRunner(settings) + crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 25d9657d5..0406d906f 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -2,8 +2,8 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy import Request, signals -from scrapy.crawler import CrawlerRunner from scrapy.http.response import Response +from scrapy.utils.test import get_crawler from testfixtures import LogCapture @@ -71,7 +71,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_response_200(self): url = self.mockserver.url("/status?n=200") - crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.request.url, url) @@ -80,7 +80,7 @@ class CrawlTestCase(TestCase): def test_response_error(self): for status in ("404", "500"): url = self.mockserver.url(f"/status?n={status}") - crawler = CrawlerRunner().create_crawler(SingleRequestSpider) + crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] response = failure.value.response @@ -90,12 +90,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_downloader_middleware_raise_exception(self): url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { RaiseExceptionRequestMiddleware: 590, }, }) - crawler = runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) failure = crawler.spider.meta["failure"] self.assertEqual(failure.request.url, url) @@ -117,12 +116,11 @@ class CrawlTestCase(TestCase): signal_params["request"] = request url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { ProcessResponseMiddleware: 595, } }) - crawler = runner.create_crawler(SingleRequestSpider) crawler.signals.connect(signal_handler, signal=signals.response_received) with LogCapture() as log: @@ -147,13 +145,12 @@ class CrawlTestCase(TestCase): The spider callback should receive the overridden response.request """ url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { RaiseExceptionRequestMiddleware: 590, CatchExceptionOverrideRequestMiddleware: 595, }, }) - crawler = runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") @@ -168,13 +165,12 @@ class CrawlTestCase(TestCase): The spider callback should receive the original response.request """ url = self.mockserver.url("/status?n=200") - runner = CrawlerRunner(settings={ + crawler = get_crawler(SingleRequestSpider, { "DOWNLOADER_MIDDLEWARES": { RaiseExceptionRequestMiddleware: 590, CatchExceptionDoNotOverrideRequestMiddleware: 595, }, }) - crawler = runner.create_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) response = crawler.spider.meta["responses"][0] self.assertEqual(response.body, b"Caught ZeroDivisionError") @@ -186,12 +182,11 @@ class CrawlTestCase(TestCase): Downloader middleware which returns a response with a specific 'request' attribute, with an alternative callback """ - runner = CrawlerRunner(settings={ + crawler = get_crawler(AlternativeCallbacksSpider, { "DOWNLOADER_MIDDLEWARES": { AlternativeCallbacksMiddleware: 595, } }) - crawler = runner.create_crawler(AlternativeCallbacksSpider) with LogCapture() as log: url = self.mockserver.url("/status?n=200") diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 473a93e69..002a04358 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -3,7 +3,7 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.http import Request -from scrapy.crawler import CrawlerRunner +from scrapy.utils.test import get_crawler from tests.spiders import MockServerSpider from tests.mockserver import MockServer @@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() - self.runner = CrawlerRunner() def tearDown(self): self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_callback_kwargs(self): - crawler = self.runner.create_crawler(KeywordArgumentsSpider) + crawler = get_crawler(KeywordArgumentsSpider) with LogCapture() as log: yield crawler.crawl(mockserver=self.mockserver) self.assertTrue(all(crawler.spider.checks)) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2d4bfa165..ac66056ba 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -52,6 +52,7 @@ class MockCrawler(Crawler): SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', + REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION', ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase): SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', CONCURRENT_REQUESTS_PER_IP=1, ) - crawler = Crawler(Spider, settings) + crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) spider = Spider(name='spider') scheduler.open(spider) diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index bf90b4320..fc234a83d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -7,10 +7,10 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.core.scheduler import BaseScheduler -from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.spiders import Spider -from scrapy.utils.request import request_fingerprint +from scrapy.utils.request import fingerprint +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS] class MinimalScheduler: def __init__(self) -> None: - self.requests: Dict[str, Request] = {} + self.requests: Dict[bytes, Request] = {} def has_pending_requests(self) -> bool: return bool(self.requests) def enqueue_request(self, request: Request) -> bool: - fp = request_fingerprint(request) + fp = fingerprint(request) if fp not in self.requests: self.requests[fp] = request return True @@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase): @defer.inlineCallbacks def test_crawl(self): with MockServer() as mockserver: - settings = {"SCHEDULER": self.scheduler_cls} + settings = { + "SCHEDULER": self.scheduler_cls, + } with LogCapture() as log: - yield CrawlerRunner(settings).crawl(TestSpider, mockserver) + crawler = get_crawler(TestSpider, settings) + yield crawler.crawl(mockserver) for path in PATHS: self.assertIn(f"{{'path': '{path}'}}", str(log)) self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log)) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 8a35e9fd7..3719c7c9f 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase): def test_crawler_runner_loading(self): module = 'tests.test_spiderloader.test_spiders.spider1' - runner = CrawlerRunner({'SPIDER_MODULES': [module]}) + runner = CrawlerRunner({ + 'SPIDER_MODULES': [module], + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + }) self.assertRaisesRegex(KeyError, 'Spider not found', runner.create_crawler, 'spider2') diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 1ef4eeb14..46452415a 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -3,6 +3,7 @@ import os import tempfile import shutil import contextlib +import warnings from pytest import warns @@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { 'SCRAPY_SETTINGS_MODULE': value, } - with set_env(**envvars), warns(None) as warnings: - settings = get_project_settings() - assert not warnings + with warnings.catch_warnings(): + warnings.simplefilter("error") + with set_env(**envvars): + settings = get_project_settings() + assert settings.get('SETTINGS_MODULE') == value def test_invalid_envvar(self): envvars = { 'SCRAPY_FOO': 'bar', } - with set_env(**envvars), warns(None) as warnings: - get_project_settings() - assert len(warnings) == 1 - assert warnings[0].category == ScrapyDeprecationWarning - assert str(warnings[0].message).endswith(': FOO') + with warns(ScrapyDeprecationWarning, match=': FOO') as record: + with set_env(**envvars): + get_project_settings() + assert len(record) == 1 def test_valid_and_invalid_envvars(self): value = 'tests.test_cmdline.settings' @@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase): 'SCRAPY_FOO': 'bar', 'SCRAPY_SETTINGS_MODULE': value, } - with set_env(**envvars), warns(None) as warnings: - settings = get_project_settings() - assert len(warnings) == 1 - assert warnings[0].category == ScrapyDeprecationWarning - assert str(warnings[0].message).endswith(': FOO') + with warns(ScrapyDeprecationWarning, match=': FOO') as record: + with set_env(**envvars): + settings = get_project_settings() + assert len(record) == 1 assert settings.get('SETTINGS_MODULE') == value diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index e9edfee98..5ee772c0b 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -308,13 +308,22 @@ class RequestFingerprintTest(FingerprintTest): ), ) + def setUp(self) -> None: + warnings.simplefilter("ignore", ScrapyDeprecationWarning) + + def tearDown(self) -> None: + warnings.simplefilter("default", ScrapyDeprecationWarning) + @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True) def test_part_separation(self): super().test_part_separation() + +class RequestFingerprintDeprecationTest(unittest.TestCase): + def test_deprecation_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: - self.function(Request("http://www.example.com")) + request_fingerprint(Request("http://www.example.com")) messages = [str(warning.message) for warning in warnings] self.assertTrue( any( @@ -326,7 +335,7 @@ class RequestFingerprintTest(FingerprintTest): def test_deprecation_non_default_parameters(self): with pytest.warns(ScrapyDeprecationWarning) as warnings: - self.function(Request("http://www.example.com"), keep_fragments=True) + request_fingerprint(Request("http://www.example.com"), keep_fragments=True) messages = [str(warning.message) for warning in warnings] self.assertTrue( any( diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 0a09f6109..d20852e62 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,7 +1,9 @@ import os import unittest +import warnings from urllib.parse import urlparse +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import (response_httprepr, open_in_browser, @@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response') def test_response_httprepr(self): - r1 = Response("http://www.example.com") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ScrapyDeprecationWarning) - r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response("http://www.example.com") + self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n') - r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body") - self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') + r1 = Response("http://www.example.com", status=404, + headers={"Content-type": "text/html"}, body=b"Some body") + self.assertEqual(response_httprepr(r1), + b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body') + + r1 = Response("http://www.example.com", status=6666, + headers={"Content-type": "text/html"}, body=b"Some body") + self.assertEqual(response_httprepr(r1), + b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body') def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" From f60c7ae768fa2f238ea6d7646098fe99ef8ad461 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 20 Jul 2022 14:01:22 +0500 Subject: [PATCH 106/237] Fixed heading levels in downloader middleware docs (#5567) --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 29e350651..986da0476 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -955,7 +955,7 @@ default because HTTP specs say so. .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST ---------------------- +^^^^^^^^^^^^^^^^^^^^^ Default: ``-1`` @@ -1119,7 +1119,7 @@ In order to use this parser: .. _support-for-new-robots-parser: Implementing support for a new parser -------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can implement support for a new robots.txt_ parser by subclassing the abstract base class :class:`~scrapy.robotstxt.RobotParser` and From 42056090516bb0cc5d349e232298c711ec452bc5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 17 Jul 2022 15:50:40 +0500 Subject: [PATCH 107/237] Fixed intersphinx references --- docs/conf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9a0afe73e..3241295af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -291,9 +291,10 @@ intersphinx_mapping = { 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), - 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current', None), - 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), + 'tox': ('https://tox.wiki/en/latest/', None), + 'twisted': ('https://docs.twisted.org/en/stable/', None), + 'twistedapi': ('https://docs.twisted.org/en/stable/api/', None), + 'w3lib': ('https://w3lib.readthedocs.io/en/latest', None), } intersphinx_disabled_reftypes = [] From b21c16099ed0acb0589677afd98c0ae3b78cd17d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 22 Jul 2022 19:18:33 +0500 Subject: [PATCH 108/237] Fix flake8 issues. --- tests/test_spidermiddleware.py | 6 +++--- tests/test_utils_python.py | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ed0912b82..edde6f682 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -305,8 +305,8 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): with self.assertRaisesRegex( _InvalidOutput, ( - "\.process_spider_output must return an iterable, got " + r"\.process_spider_output must return an iterable, got " ), ): yield self._get_middleware_result( @@ -317,7 +317,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase): def test_coroutine(self): with self.assertRaisesRegex( _InvalidOutput, - "\.process_spider_output must be an asynchronous generator", + r"\.process_spider_output must be an asynchronous generator", ): yield self._get_middleware_result( ProcessSpiderOutputCoroutineMiddleware, diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 00b06b839..b1a8fdc04 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -2,8 +2,6 @@ import functools import gc import operator import platform -import unittest -from datetime import datetime from itertools import count from warnings import catch_warnings, filterwarnings From af7dd16d8ded3e6cb2946603688f4f4a5212e80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Jul 2022 13:15:17 +0200 Subject: [PATCH 109/237] Merge pull request from GHSA-9x8m-2xpf-crp3 * Enforce matching proxy request meta and Proxy-Authorization header * Cover proxy credential security fix in the release notes * Remove extra empty line * Reword the security issue description * Address scenario where Proxy-Authorization is unexpectedly removed by a prior middleware * Set the release date of Scrapy 2.6.2 and 1.8.3 --- docs/news.rst | 106 ++++++- scrapy/downloadermiddlewares/httpproxy.py | 56 ++-- tests/test_downloadermiddleware_httpproxy.py | 313 ++++++++++++++++++- 3 files changed, 439 insertions(+), 36 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7993b4b4f..5bd9ca059 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,10 +5,57 @@ Release notes .. _release-2.6.2: -Scrapy 2.6.2 (to be determined) -------------------------------- +Scrapy 2.6.2 (2022-07-25) +------------------------- -Fixes additional regressions introduced in 2.6.0: +**Security bug fix:** + +- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + processes a request with :reqmeta:`proxy` metadata, and that + :reqmeta:`proxy` metadata includes proxy credentials, + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets + the ``Proxy-Authentication`` header, but only if that header is not already + set. + + There are third-party proxy-rotation downloader middlewares that set + different :reqmeta:`proxy` metadata every time they process a request. + + Because of request retries and redirects, the same request can be processed + by downloader middlewares more than once, including both + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and + any third-party proxy-rotation downloader middleware. + + These third-party proxy-rotation downloader middlewares could change the + :reqmeta:`proxy` metadata of a request to a new value, but fail to remove + the ``Proxy-Authentication`` header from the previous value of the + :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent + to a different proxy. + + To prevent the unintended leaking of proxy credentials, the behavior of + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now + as follows when processing a request: + + - If the request being processed defines :reqmeta:`proxy` metadata that + includes credentials, the ``Proxy-Authorization`` header is always + updated to feature those credentials. + + - If the request being processed defines :reqmeta:`proxy` metadata + without credentials, the ``Proxy-Authorization`` header is removed + *unless* it was originally defined for the same proxy URL. + + To remove proxy credentials while keeping the same proxy URL, remove + the ``Proxy-Authorization`` header. + + - If the request has no :reqmeta:`proxy` metadata, or that metadata is a + falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is + removed. + + It is no longer possible to set a proxy URL through the + :reqmeta:`proxy` metadata but set the credentials through the + ``Proxy-Authorization`` header. Set proxy credentials through the + :reqmeta:`proxy` metadata instead. + +Also fixes the following regressions introduced in 2.6.0: - :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple spiders (:issue:`5435`, :issue:`5436`) @@ -1925,6 +1972,59 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.3: + +Scrapy 1.8.3 (2022-07-25) +------------------------- + +**Security bug fix:** + +- When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` + processes a request with :reqmeta:`proxy` metadata, and that + :reqmeta:`proxy` metadata includes proxy credentials, + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets + the ``Proxy-Authentication`` header, but only if that header is not already + set. + + There are third-party proxy-rotation downloader middlewares that set + different :reqmeta:`proxy` metadata every time they process a request. + + Because of request retries and redirects, the same request can be processed + by downloader middlewares more than once, including both + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and + any third-party proxy-rotation downloader middleware. + + These third-party proxy-rotation downloader middlewares could change the + :reqmeta:`proxy` metadata of a request to a new value, but fail to remove + the ``Proxy-Authentication`` header from the previous value of the + :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent + to a different proxy. + + To prevent the unintended leaking of proxy credentials, the behavior of + :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now + as follows when processing a request: + + - If the request being processed defines :reqmeta:`proxy` metadata that + includes credentials, the ``Proxy-Authorization`` header is always + updated to feature those credentials. + + - If the request being processed defines :reqmeta:`proxy` metadata + without credentials, the ``Proxy-Authorization`` header is removed + *unless* it was originally defined for the same proxy URL. + + To remove proxy credentials while keeping the same proxy URL, remove + the ``Proxy-Authorization`` header. + + - If the request has no :reqmeta:`proxy` metadata, or that metadata is a + falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is + removed. + + It is no longer possible to set a proxy URL through the + :reqmeta:`proxy` metadata but set the credentials through the + ``Proxy-Authorization`` header. Set proxy credentials through the + :reqmeta:`proxy` metadata instead. + + .. _release-1.8.2: Scrapy 1.8.2 (2022-03-01) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index d2665b655..1deda42bd 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -45,31 +45,37 @@ class HttpProxyMiddleware: return creds, proxy_url def process_request(self, request, spider): - # ignore if proxy is already set + creds, proxy_url = None, None if 'proxy' in request.meta: - if request.meta['proxy'] is None: - return - # extract credentials if present - creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + if request.meta['proxy'] is not None: + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + elif self.proxies: + parsed = urlparse_cached(request) + scheme = parsed.scheme + if ( + ( + # 'no_proxy' is only supported by http schemes + scheme not in ('http', 'https') + or not proxy_bypass(parsed.hostname) + ) + and scheme in self.proxies + ): + creds, proxy_url = self.proxies[scheme] + + self._set_proxy_and_creds(request, proxy_url, creds) + + def _set_proxy_and_creds(self, request, proxy_url, creds): + if proxy_url: request.meta['proxy'] = proxy_url - if creds and not request.headers.get('Proxy-Authorization'): - request.headers['Proxy-Authorization'] = b'Basic ' + creds - return - elif not self.proxies: - return - - parsed = urlparse_cached(request) - scheme = parsed.scheme - - # 'no_proxy' is only supported by http schemes - if scheme in ('http', 'https') and proxy_bypass(parsed.hostname): - return - - if scheme in self.proxies: - self._set_proxy(request, scheme) - - def _set_proxy(self, request, scheme): - creds, proxy = self.proxies[scheme] - request.meta['proxy'] = proxy + elif request.meta.get('proxy') is not None: + request.meta['proxy'] = None if creds: - request.headers['Proxy-Authorization'] = b'Basic ' + creds + request.headers[b'Proxy-Authorization'] = b'Basic ' + creds + request.meta['_auth_proxy'] = proxy_url + elif '_auth_proxy' in request.meta: + if proxy_url != request.meta['_auth_proxy']: + if b'Proxy-Authorization' in request.headers: + del request.headers[b'Proxy-Authorization'] + del request.meta['_auth_proxy'] + elif b'Proxy-Authorization' in request.headers: + del request.headers[b'Proxy-Authorization'] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7c97bf32a..67134cf93 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -65,12 +65,12 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): @@ -78,12 +78,12 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): @@ -92,26 +92,26 @@ class TestHttpProxyMiddleware(TestCase): mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.meta['proxy'], 'https://proxy:3128') self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') def test_proxy_already_seted(self): @@ -152,3 +152,300 @@ class TestHttpProxyMiddleware(TestCase): # '/var/run/docker.sock' may be used by the user for # no_proxy value but is not parseable and should be skipped assert 'no' not in mw.proxies + + def test_add_proxy_without_credentials(self): + middleware = HttpProxyMiddleware() + request = Request('https://example.com') + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_add_proxy_with_credentials(self): + middleware = HttpProxyMiddleware() + request = Request('https://example.com') + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://user1:password1@example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_remove_proxy_without_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = None + assert middleware.process_request(request, spider) is None + self.assertIsNone(request.meta['proxy']) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_remove_proxy_with_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = None + assert middleware.process_request(request, spider) is None + self.assertIsNone(request.meta['proxy']) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_add_credentials(self): + """If the proxy request meta switches to a proxy URL with the same + proxy and adds credentials (there were no credentials before), the new + credentials must be used.""" + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user1:password1@example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_change_credentials(self): + """If the proxy request meta switches to a proxy URL with different + credentials, those new credentials must be used.""" + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://user2:password2@example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user2', + 'password2', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_remove_credentials(self): + """If the proxy request meta switches to a proxy URL with the same + proxy but no credentials, the original credentials must be still + used. + + To remove credentials while keeping the same proxy URL, users must + delete the Proxy-Authorization header. + """ + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + request.meta['proxy'] = 'https://example.com' + del request.headers[b'Proxy-Authorization'] + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_change_proxy_add_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user1:password1@example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.org') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_change_proxy_keep_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user1:password1@example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.org') + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + # Make sure, indirectly, that _auth_proxy is updated. + request.meta['proxy'] = 'https://example.com' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_change_proxy_change_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + + request.meta['proxy'] = 'https://user2:password2@example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.org') + encoded_credentials = middleware._basic_auth_header( + 'user2', + 'password2', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_change_proxy_remove_credentials(self): + """If the proxy request meta switches to a proxy URL with a different + proxy and no credentials, no credentials must be used.""" + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://example.org' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta, {'proxy': 'https://example.org'}) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_change_proxy_remove_credentials_preremoved_header(self): + """Corner case of proxy switch with credentials removal where the + credentials have been removed beforehand. + + It ensures that our implementation does not assume that the credentials + header exists when trying to remove it. + """ + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + request.meta['proxy'] = 'https://example.org' + del request.headers[b'Proxy-Authorization'] + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta, {'proxy': 'https://example.org'}) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_undefined_proxy(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': 'Basic foo'}, + ) + assert middleware.process_request(request, spider) is None + self.assertNotIn('proxy', request.meta) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_disabled_proxy(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': 'Basic foo'}, + meta={'proxy': None}, + ) + assert middleware.process_request(request, spider) is None + self.assertIsNone(request.meta['proxy']) + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_proxy_without_credentials(self): + middleware = HttpProxyMiddleware() + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': 'Basic foo'}, + meta={'proxy': 'https://example.com'}, + ) + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertNotIn(b'Proxy-Authorization', request.headers) + + def test_proxy_authentication_header_proxy_with_same_credentials(self): + middleware = HttpProxyMiddleware() + encoded_credentials = middleware._basic_auth_header( + 'user1', + 'password1', + ) + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': b'Basic ' + encoded_credentials}, + meta={'proxy': 'https://user1:password1@example.com'}, + ) + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials, + ) + + def test_proxy_authentication_header_proxy_with_different_credentials(self): + middleware = HttpProxyMiddleware() + encoded_credentials1 = middleware._basic_auth_header( + 'user1', + 'password1', + ) + request = Request( + 'https://example.com', + headers={'Proxy-Authorization': b'Basic ' + encoded_credentials1}, + meta={'proxy': 'https://user2:password2@example.com'}, + ) + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + encoded_credentials2 = middleware._basic_auth_header( + 'user2', + 'password2', + ) + self.assertEqual( + request.headers['Proxy-Authorization'], + b'Basic ' + encoded_credentials2, + ) From aecbccbaa567b07694141a4503e9abf1bb2c919f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Jul 2022 13:33:23 +0200 Subject: [PATCH 110/237] =?UTF-8?q?Bump=20version:=202.6.1=20=E2=86=92=202?= =?UTF-8?q?.6.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1d9b9c02f..2e2f7949a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.1 +current_version = 2.6.2 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 6a6a3d8e3..097a15a2a 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.1 +2.6.2 From 5862f6b8e1cd9e240c4b3e79f2eddd0b9f33ef8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 25 Jul 2022 17:25:10 +0200 Subject: [PATCH 111/237] Update frozen CI packages (#5574) --- pylintrc | 6 +----- tox.ini | 13 ++++++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pylintrc b/pylintrc index 2cdd6321e..d8e47dc11 100644 --- a/pylintrc +++ b/pylintrc @@ -9,11 +9,9 @@ disable=abstract-method, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-continuation, bad-indentation, bad-mcs-classmethod-argument, bad-super-call, - bad-whitespace, bare-except, blacklisted-name, broad-except, @@ -52,7 +50,6 @@ disable=abstract-method, logging-not-lazy, lost-exception, method-hidden, - misplaced-comparison-constant, missing-docstring, missing-final-newline, multiple-imports, @@ -60,12 +57,10 @@ disable=abstract-method, no-else-continue, no-else-raise, no-else-return, - no-init, no-member, no-method-argument, no-name-in-module, no-self-argument, - no-self-use, no-value-for-parameter, not-an-iterable, not-callable, @@ -102,6 +97,7 @@ disable=abstract-method, ungrouped-imports, unidiomatic-typecheck, unnecessary-comprehension, + unnecessary-dunder-call, unnecessary-lambda, unnecessary-pass, unreachable, diff --git a/tox.ini b/tox.ini index ab8a715c2..4d1bb574d 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ commands = [testenv:security] basepython = python3 deps = - bandit==1.7.3 + bandit==1.7.4 commands = bandit -r -c .bandit.yml {posargs:scrapy} @@ -57,16 +57,17 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - pytest-flake8 - flake8==3.9.2 # https://github.com/tholo/pytest-flake8/issues/81 + pytest-flake8==1.1.1 + flake8==4.0.1 commands = pytest --flake8 {posargs:docs scrapy tests} [testenv:pylint] -basepython = python3 +# reppy does not support Python 3.9+ +basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.12.2 + pylint==2.14.5 commands = pylint conftest.py docs extras scrapy setup.py tests @@ -117,6 +118,8 @@ setenv = {[pinned]setenv} [testenv:extra-deps] +# reppy does not support Python 3.9+ +basepython = python3.8 deps = {[testenv]deps} boto From 56e2eeac1066cd2d3c710b76a3d6210b3ac257f5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Jul 2022 09:41:12 +0500 Subject: [PATCH 112/237] fix typing issues by upgrading mypy --- tests/test_utils_request.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 5ee772c0b..8bc7922b6 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -52,7 +52,7 @@ class UtilsRequestTest(unittest.TestCase): class FingerprintTest(unittest.TestCase): maxDiff = None - function = staticmethod(fingerprint) + function: staticmethod = staticmethod(fingerprint) cache: Union[ "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]", "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]", diff --git a/tox.ini b/tox.ini index 4d1bb574d..2110e1020 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.910 + mypy==0.971 types-pyOpenSSL==20.0.3 types-setuptools==57.0.0 commands = From 67011cd9576b02d696c7e0c0dc7f76074e30901b Mon Sep 17 00:00:00 2001 From: Aftab Alam <88653530+itsAftabAlam@users.noreply.github.com> Date: Wed, 27 Jul 2022 10:20:26 +0530 Subject: [PATCH 113/237] updated README.rst , added hyperlink to banner (#5284) Co-authored-by: Mikhail Korobov --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index b543a30f4..358302c76 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,5 @@ .. image:: https://scrapy.org/img/scrapylogo.png + :target: https://scrapy.org/ ====== Scrapy From 83ecdf1bcab310be5f2c59e5c005b8968d72a72c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 27 Jul 2022 23:12:31 +0500 Subject: [PATCH 114/237] Update docs/topics/components.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/components.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 1fff2d61a..c44f3def2 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -47,7 +47,7 @@ Enforcing component requirements ================================ Sometimes, your components may only be intended to work under certain -conditions. For example, the may require a minimum version of Scrapy to work as +conditions. For example, they may require a minimum version of Scrapy to work as intended, or they may require certain settings to have specific values. In addition to describing those conditions in the documentation of your From 0f1112f3e22d91e505f736fc78fae94eda07c72d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 27 Jul 2022 23:12:43 +0500 Subject: [PATCH 115/237] Update docs/index.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index ea4950e4c..5404969e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -258,7 +258,7 @@ Extending Scrapy components. :doc:`topics/api` - Use it on extensions and middlewares to extend Scrapy functionality + Use it on extensions and middlewares to extend Scrapy functionality. All the rest From c7b90c6e1e3de20256d8bf6fc5d18da44d562b0d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 28 Jul 2022 13:44:36 +0500 Subject: [PATCH 116/237] Extract more common code. --- scrapy/spidermiddlewares/depth.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 29634c3ad..4c923b1b3 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -28,25 +28,22 @@ class DepthMiddleware: return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) - + self._init_depth(response, spider) return (r for r in result or () if self._filter(r, response, spider)) async def process_spider_output_async(self, response, result, spider): + self._init_depth(response, spider) + async for r in result or (): + if self._filter(r, response, spider): + yield r + + def _init_depth(self, response, spider): # base case (depth=0) if 'depth' not in response.meta: response.meta['depth'] = 0 if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - async for r in result or (): - if self._filter(r, response, spider): - yield r - def _filter(self, request, response, spider): if not isinstance(request, Request): return True From 13c5ad7e688271ae47f2a489bf665cb15387b3a3 Mon Sep 17 00:00:00 2001 From: "Alexandr N. Zamaraev" Date: Tue, 16 Aug 2022 11:03:37 +0700 Subject: [PATCH 117/237] Partial in is_generator_with_return_value See path in #5592 --- scrapy/utils/misc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..2e25f6421 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -9,6 +9,7 @@ from collections import deque from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules +from functools import partial from w3lib.html import replace_entities @@ -226,7 +227,11 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + func = callable + while isinstance(func, partial): + func = func.func + + code = re.sub(r"^[\t ]+", "", inspect.getsource(func)) tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From c49b5aaf77a668e44bffb54c52310084e4068162 Mon Sep 17 00:00:00 2001 From: zaid-ismail031 Date: Thu, 25 Aug 2022 02:17:03 +0200 Subject: [PATCH 118/237] Changed incorrect information regarding the return type of parse/request callback method. --- docs/topics/spiders.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index ece02ae47..ffe41cf3e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -181,9 +181,10 @@ scrapy.Spider scraped data and/or more URLs to follow. Other Requests callbacks have the same requirements as the :class:`Spider` class. - This method, as well as any other Request callback, must return an - iterable of :class:`~scrapy.Request` and/or :ref:`item objects - `. + This method, as well as any other Request callback, must return a + :class:`~scrapy.Request` object, an :ref:`item object `, an + iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects + `, or ``None``. :param response: the response to parse :type response: :class:`~scrapy.http.Response` From e411ea94eb853eedeee111469c6442d275864a09 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sun, 28 Aug 2022 20:28:13 +0430 Subject: [PATCH 119/237] BOM should take precedence over Content-Type header when detecting the encoding closes #5601 --- scrapy/http/response/text.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 89516b9b6..bfcde878d 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,8 +11,13 @@ from typing import Generator, Tuple from urllib.parse import urljoin import parsel -from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, - http_content_type_encoding, resolve_encoding) +from w3lib.encoding import ( + html_body_declared_encoding, + html_to_unicode, + http_content_type_encoding, + resolve_encoding, + read_bom, +) from w3lib.html import strip_html5_whitespace from scrapy.http import Request @@ -60,6 +65,7 @@ class TextResponse(Response): def _declared_encoding(self): return ( self._encoding + or self._bom_encoding() or self._headers_encoding() or self._body_declared_encoding() ) @@ -117,6 +123,10 @@ class TextResponse(Response): def _body_declared_encoding(self): return html_body_declared_encoding(self.body) + @memoizemethod_noargs + def _bom_encoding(self): + return read_bom(self.body)[0] + @property def selector(self): from scrapy.selector import Selector From a988c4b78b9ca104b74cff60c46c1842e3f25652 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Mon, 29 Aug 2022 17:08:30 +0430 Subject: [PATCH 120/237] add test --- tests/test_http_response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2986f884f..5d67a5e74 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +import codecs import unittest from unittest import mock @@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=gb2312"]}) r7 = self.response_class("http://www.example.com", body=b"\xa8D", headers={"Content-type": ["text/html; charset=gbk"]}) + r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3", + headers={"Content-type": ["text/html; charset=cp1251"]}) self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3._declared_encoding(), "cp1252") self.assertEqual(r4._headers_encoding(), None) self.assertEqual(r5._headers_encoding(), None) + self.assertEqual(r8._headers_encoding(), "cp1251") + self.assertEqual(r8._declared_encoding(), "utf-8") self._assert_response_encoding(r5, "utf-8") + self._assert_response_encoding(r8, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', "\xa3") self._assert_response_values(r2, 'utf-8', "\xa3") From f4bcc3e67de0896999be48d1b0fcede0be0d69e4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 01:15:41 -0300 Subject: [PATCH 121/237] fix: failed tests --- tests/test_crawler.py | 13 +++++++++---- tests/test_spider.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4e9ef7740..4e599d69c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -23,6 +23,8 @@ from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer @@ -381,10 +383,13 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) + if parse_version(w3lib_version) < parse_version("2.0.0"): + self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) + else: + self.assertIn("ValueError: invalid hostname:", log) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") diff --git a/tests/test_spider.py b/tests/test_spider.py index 689349999..7b36304b1 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,6 +22,8 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version class SpiderTest(unittest.TestCase): @@ -360,10 +362,12 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML'] + if parse_version(w3lib_version) >= parse_version('2.0.0'): + urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls)) + self.assertEqual([r.url for r in output], urls) def test_process_request_instance_method_with_response(self): From 1289422284991c09cbee8cea17bef837b3efc31b Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 08:17:58 -0300 Subject: [PATCH 122/237] chore: Skip `test_ipv6_default_name_resolver` test if w3lib version >= 2.0.0 --- tests/test_crawler.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4e599d69c..e2a14be55 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -380,16 +380,15 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"), + reason='w3lib 2.0.0 and later do not allow invalid domains.') def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') self.assertIn('Spider closed (finished)', log) - if parse_version(w3lib_version) < parse_version("2.0.0"): - self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) - self.assertIn( - "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", - log) - else: - self.assertIn("ValueError: invalid hostname:", log) + self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log) + self.assertIn( + "twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", + log) def test_caching_hostname_resolver_ipv6(self): log = self.run_script("caching_hostname_resolver_ipv6.py") From 582a6bf6dbcb01da40cbec6b51269add2db39cf1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 6 Sep 2022 10:03:18 -0300 Subject: [PATCH 123/237] refactor: Use `safe_url_string` to standardize url output --- tests/test_spider.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 7b36304b1..e1527620f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -22,8 +22,7 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata -from pkg_resources import parse_version -from w3lib import __version__ as w3lib_version +from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): @@ -362,12 +361,10 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - urls = ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML'] - if parse_version(w3lib_version) >= parse_version('2.0.0'): - urls = list(map(lambda u: u.replace("EXAMPLE.ORG", "example.org"), urls)) - self.assertEqual([r.url for r in output], urls) + self.assertEqual([r.url for r in output], + [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'), + safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'), + safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')]) def test_process_request_instance_method_with_response(self): From ce0ca51485545aed8eb33a285bc3d2fbf8a8e407 Mon Sep 17 00:00:00 2001 From: "Magsen (CD)" Date: Tue, 13 Sep 2022 12:07:58 +0200 Subject: [PATCH 124/237] fix: typo in tutorial fix: typo in tutorial --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 75928077e..092123d1d 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -379,7 +379,7 @@ like this: Let's open up scrapy shell and play a bit to find out how to extract the data we want:: - $ scrapy shell 'https://quotes.toscrape.com' + scrapy shell 'https://quotes.toscrape.com' We get a list of selectors for the quote HTML elements with: From 77c055ee28a767b2c8222274cc7556ab9cc56edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 14 Sep 2022 14:47:14 +0200 Subject: [PATCH 125/237] Relax Proxy-Authorization restrictions --- scrapy/downloadermiddlewares/httpproxy.py | 5 ++++- tests/test_downloadermiddleware_httpproxy.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 1deda42bd..dd8a7e797 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -78,4 +78,7 @@ class HttpProxyMiddleware: del request.headers[b'Proxy-Authorization'] del request.meta['_auth_proxy'] elif b'Proxy-Authorization' in request.headers: - del request.headers[b'Proxy-Authorization'] + if proxy_url: + request.meta['_auth_proxy'] = proxy_url + else: + del request.headers[b'Proxy-Authorization'] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 70eb94d77..44434f90e 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase): self.assertNotIn(b'Proxy-Authorization', request.headers) def test_proxy_authentication_header_proxy_without_credentials(self): + """As long as the proxy URL in request metadata remains the same, the + Proxy-Authorization header is used and kept, and may even be + changed.""" middleware = HttpProxyMiddleware() request = Request( 'https://example.com', @@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase): ) assert middleware.process_request(request, spider) is None self.assertEqual(request.meta['proxy'], 'https://example.com') - self.assertNotIn(b'Proxy-Authorization', request.headers) + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo') + + request.headers['Proxy-Authorization'] = b'Basic bar' + assert middleware.process_request(request, spider) is None + self.assertEqual(request.meta['proxy'], 'https://example.com') + self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar') def test_proxy_authentication_header_proxy_with_same_credentials(self): middleware = HttpProxyMiddleware() From 1429aa011ce2a7a43691fc359c88bf40e512176c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Tue, 20 Sep 2022 12:47:20 -0400 Subject: [PATCH 126/237] Update test-standard link in contributing docs (#5631) --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 946bdc23e..9cfe10012 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -214,7 +214,7 @@ Tests ===== Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires +`. Running tests requires :doc:`tox `. .. _running-tests: From 5f194202114fd38530c78299d51b6966b4802f59 Mon Sep 17 00:00:00 2001 From: Tim B Date: Wed, 21 Sep 2022 07:27:27 +0100 Subject: [PATCH 127/237] Documented how settings must be picklable (#5629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- docs/topics/settings.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6722ce9ed..90f13f3ef 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -98,6 +98,10 @@ class. The global defaults are located in the ``scrapy.settings.default_settings`` module and documented in the :ref:`topics-settings-ref` section. +Compatibility with pickle +========================= + +Setting values must be :ref:`picklable `. Import paths and classes ======================== From 385acd5598fb06a126ac485087b58a4581096876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sat, 24 Sep 2022 14:58:14 -0400 Subject: [PATCH 128/237] Match pyOpenSSL and service_identity to Twisted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- setup.py | 6 +++--- tests/test_crawler.py | 2 +- tox.ini | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index ed197273f..fcc902c62 100644 --- a/setup.py +++ b/setup.py @@ -20,13 +20,13 @@ def has_environment_marker_platform_impl_support(): install_requires = [ 'Twisted>=18.9.0', - 'cryptography>=2.8', + 'cryptography>=3.3', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', 'parsel>=1.5.0', - 'pyOpenSSL>=19.1.0', + 'pyOpenSSL>=21.0.0', 'queuelib>=1.4.2', - 'service_identity>=16.0.0', + 'service_identity>=18.1.0', 'w3lib>=1.17.0', 'zope.interface>=5.1.0', 'protego>=0.1.15', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index e2a14be55..cf15ba9b9 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -327,7 +327,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_default_twisted_reactor_select(self): log = self.run_script('reactor_default_twisted_reactor_select.py') - if platform.system() == 'Windows': + if platform.system() in ['Windows', 'Darwin']: # The goal of this test function is to test that, when a reactor is # installed (the default one here) and a different reactor is # configured (select here), an error raises. diff --git a/tox.ini b/tox.ini index 2110e1020..2bf9454d0 100644 --- a/tox.ini +++ b/tox.ini @@ -73,15 +73,15 @@ commands = [pinned] deps = - cryptography==2.8 + cryptography==3.3 cssselect==0.9.1 h2==3.0 itemadapter==0.1.0 parsel==1.5.0 Protego==0.1.15 - pyOpenSSL==19.1.0 + pyOpenSSL==21.0.0 queuelib==1.4.2 - service_identity==16.0.0 + service_identity==18.1.0 Twisted[http2]==18.9.0 w3lib==1.17.0 zope.interface==5.1.0 From 79a4bc3da02bfe2b07c596f6b677760b2c04f96f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 14:17:57 -0400 Subject: [PATCH 129/237] Cleanup METHOD_SSLv3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- docs/topics/settings.rst | 1 - scrapy/core/downloader/contextfactory.py | 2 +- scrapy/core/downloader/tls.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90f13f3ef..a711fd197 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -564,7 +564,6 @@ This setting must be one of these string values: set this if you want the behavior of Scrapy<1.1 - ``'TLSv1.1'``: forces TLS version 1.1 - ``'TLSv1.2'``: forces TLS version 1.2 -- ``'SSLv3'``: forces SSL version 3 (**not recommended**) .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index b5318c7bb..4abde2238 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): which allows TLS protocol negotiation 'A TLS/SSL connection established with [this method] may - understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' + understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 19a56d9b6..698a1c85c 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) -METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' METHOD_TLSv11 = 'TLSv1.1' @@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only From 1d79994dccf396caca93f53cd50646281fee1def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 27 Sep 2022 17:01:33 +0200 Subject: [PATCH 130/237] Copy 2.6.3 release notes from the 2.6 branch --- docs/news.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index d27c105a5..9469d0fe5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,32 @@ Release notes ============= +.. _release-2.6.3: + +Scrapy 2.6.3 (2022-09-27) +------------------------- + +- Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3 + (:issue:`5634`, :issue:`5635`, :issue:`5636`). + +- Upgraded the minimum versions of the following dependencies: + + - cryptography_: 2.0 → 3.3 + + - pyOpenSSL_: 16.2.0 → 21.0.0 + + - service_identity_: 16.0.0 → 18.1.0 + + - Twisted_: 17.9.0 → 18.9.0 + + - zope.interface_: 4.1.3 → 5.0.0 + + (:issue:`5621`, :issue:`5632`) + +- Fixes test and documentation issues (:issue:`5612`, :issue:`5617`, + :issue:`5631`). + + .. _release-2.6.2: Scrapy 2.6.2 (2022-07-25) From 3ca7877781fdae5415c57a9a0f348b1797795209 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Thu, 29 Sep 2022 11:51:11 -0300 Subject: [PATCH 131/237] chore: Skip `batch_path_differ` test on Windows --- tests/test_feedexport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ecd1b59d3..ad2383018 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2285,6 +2285,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) + @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output') @defer.inlineCallbacks def test_batch_path_differ(self): """ @@ -2305,7 +2306,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) - self.assertEqual(len(items) + 1, len(data['json'])) + self.assertEqual(len(items), len([_ for _ in data['json'] if _])) @defer.inlineCallbacks def test_stats_batch_file_success(self): From 9f006e3aa595acc6499f08a90696d7cd0dc0bca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 25 Sep 2022 13:48:51 -0400 Subject: [PATCH 132/237] Match pyOpenSSL and types-pyOpenSSL versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2bf9454d0..0a47a1c0b 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ basepython = python3 deps = lxml-stubs==0.2.0 mypy==0.971 - types-pyOpenSSL==20.0.3 + types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = mypy --show-error-codes {posargs: scrapy tests} From 116d9a97481f5ee4028a1f3f8e72ca34b35a0be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Fri, 30 Sep 2022 22:58:14 -0400 Subject: [PATCH 133/237] Correct distutils deprecation warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gábor Lipták --- scrapy/utils/display.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f4d17224b..d28df40c7 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support import ctypes import platform import sys -from distutils.version import LooseVersion as parse_version +from packaging.version import Version as parse_version from pprint import pformat as pformat_ From c3f35d2ad79d19e7e000b8ba0df7d1d9f10658b2 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 19:37:51 +0200 Subject: [PATCH 134/237] ci(tests-windows): upgrade actions/setup-python to v4 --- .github/workflows/tests-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 955b9b449..0d85a8e09 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 9fcbf3bcbc8b8d09bb7d8b246dae9156d6848a5b Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 19:38:16 +0200 Subject: [PATCH 135/237] ci(tests-windows): upgrade actions/checkout to v3 --- .github/workflows/tests-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 955b9b449..2d8c140af 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -25,7 +25,7 @@ jobs: TOXENV: asyncio steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From 1a2cb61e22506c47e78bc800739d9cb2a8bf0144 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 20:27:10 +0200 Subject: [PATCH 136/237] ci(test-macos): upgrade actions/checkout to v3 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7819a4e12..8c9ef5c2f 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -10,7 +10,7 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 From af95331296a91444947095af19f8244d8bc915e1 Mon Sep 17 00:00:00 2001 From: Oscar Dominguez Date: Sat, 1 Oct 2022 20:27:23 +0200 Subject: [PATCH 137/237] ci(test-macos): upgrade actions/setup-python to v4 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7819a4e12..d49272fbf 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 759ad5dee4120dc2197ea6fa1559c92f77472abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Lipt=C3=A1k?= Date: Sun, 2 Oct 2022 09:09:04 -0400 Subject: [PATCH 138/237] Require packaging --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fcc902c62..8b9c43738 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ install_requires = [ 'protego>=0.1.15', 'itemadapter>=0.1.0', 'setuptools', + 'packaging', 'tldextract', 'lxml>=4.3.0', ] From c7d800ab229df50c2767ad67460e90f6213fc293 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Sun, 2 Oct 2022 19:12:48 +0500 Subject: [PATCH 139/237] CI: add Twine check in check workflow --- .github/workflows/checks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b26f344ff..a708474ef 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -39,3 +39,8 @@ jobs: run: | pip install -U tox tox + - name: Twine check + run: | + pip install twine + python setup.py sdist + twine check dist/* From 69bf5c662555db64979a90c5c284b1faa4f24992 Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Sun, 2 Oct 2022 20:27:24 +0500 Subject: [PATCH 140/237] CI: move twinecheck to tox env --- .github/workflows/checks.yml | 8 +++----- tox.ini | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a708474ef..e515959ad 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,6 +25,9 @@ jobs: - python-version: "3.10" # Keep in sync with .readthedocs.yml env: TOXENV: docs + - python-version: "3.10" + env: + TOXENV: twinecheck steps: - uses: actions/checkout@v2 @@ -39,8 +42,3 @@ jobs: run: | pip install -U tox tox - - name: Twine check - run: | - pip install twine - python setup.py sdist - twine check dist/* diff --git a/tox.ini b/tox.ini index 2bf9454d0..2d94ba78f 100644 --- a/tox.ini +++ b/tox.ini @@ -71,6 +71,14 @@ deps = commands = pylint conftest.py docs extras scrapy setup.py tests +[testenv:twinecheck] +basepython = python3 +deps = + twine==4.0.1 +commands = + python setup.py sdist + twine check dist/* + [pinned] deps = cryptography==3.3 From 80194f1c0374b2aa429de4c7ea70a683c946db95 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 15:22:06 -0300 Subject: [PATCH 141/237] CrawlSpider: add support for async def callbacks --- scrapy/spiders/crawl.py | 6 ++++-- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 1dcf2e6ab..d860ae0b4 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,7 +6,7 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Sequence +from typing import Awaitable, Sequence from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor @@ -109,9 +109,11 @@ class CrawlSpider(Spider): rule = self._rules[failure.request.meta['rule']] return self._handle_failure(failure, rule.errback) - def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () + if isinstance(cb_res, Awaitable): + cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): yield request_or_item diff --git a/tests/spiders.py b/tests/spiders.py index 3b69aa7ae..2b78e1f7c 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -369,6 +369,18 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async def callback""" + name = 'crawl_spider_with_async_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async', follow=True), + ) + + async def parse_async(self, response, foo=None): + self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo) + return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c11871745..d14021319 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -37,6 +37,7 @@ from tests.spiders import ( BrokenStartRequestsSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, + CrawlSpiderWithAsyncCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -391,6 +392,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse] status 201 (foo: None)", str(log)) self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From da8f915091a769b3c3aca0f60d2370c2240213a1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 17:37:10 -0300 Subject: [PATCH 142/237] Adapt for asyng generator callbacks --- scrapy/spiders/crawl.py | 11 +++++++---- scrapy/utils/asyncgen.py | 2 +- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d860ae0b4..edac082d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Awaitable, Sequence +from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, HtmlResponse +from scrapy.http import Request, Response, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output @@ -78,7 +79,7 @@ class CrawlSpider(Spider): def parse_start_url(self, response, **kwargs): return [] - def process_results(self, response, results): + def process_results(self, response: Response, results: list): return results def _build_request(self, rule_index, link): @@ -112,7 +113,9 @@ class CrawlSpider(Spider): async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, Awaitable): + if isinstance(cb_res, AsyncIterable): + cb_res = await collect_asyncgen(cb_res) + elif isinstance(cb_res, Awaitable): cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 9f794de92..c84b51e8c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, AsyncIterable, Iterable, Union -async def collect_asyncgen(result: AsyncIterable): +async def collect_asyncgen(result: AsyncIterable) -> list: results = [] async for x in result: results.append(x) diff --git a/tests/spiders.py b/tests/spiders.py index 2b78e1f7c..5ea8a4a21 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -381,6 +381,18 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async generator callback""" + name = 'crawl_spider_with_async_generator_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async_gen', follow=True), + ) + + async def parse_async_gen(self, response, foo=None): + self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d14021319..8be4b6fe1 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -38,6 +38,7 @@ from tests.spiders import ( BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, + CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -402,6 +403,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse_async] status 201 (foo: None)", str(log)) self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_generator_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From 41041ae740260c4bddda06ab7f4d8e4351e6e0e4 Mon Sep 17 00:00:00 2001 From: Felipe A Date: Mon, 3 Oct 2022 00:48:12 -0300 Subject: [PATCH 143/237] refact: add Osx DS_Store file to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d77d24624..6c5c50e08 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ test-output.* # Windows Thumbs.db + +# OSX miscellaneous +.DS_Store \ No newline at end of file From 82d10f09145186fbd3a2c445287a7bf3e1e54263 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Oct 2022 20:27:06 +0600 Subject: [PATCH 144/237] Add Ubuntu tests for Python 3.11rc2. --- .github/workflows/tests-ubuntu.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index be40c7c71..9e62b8e23 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -20,6 +20,12 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio + - python-version: "3.11.0-rc.2" + env: + TOXENV: py + - python-version: "3.11.0-rc.2" + env: + TOXENV: asyncio - python-version: pypy3 env: TOXENV: pypy3 @@ -53,7 +59,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4' + if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version From fa58ab21e4670db1b11b6097d809cd18dd2ca667 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Oct 2022 20:59:06 +0600 Subject: [PATCH 145/237] Replace _getargspec_py23() with inspect.getfullargspec(). --- scrapy/utils/python.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 11c089ac2..8ce030d9d 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -180,23 +180,6 @@ def binary_is_text(data): return all(c not in _BINARYCHARS for c in data) -def _getargspec_py23(func): - """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, - defaults) - - Was identical to inspect.getargspec() in python2, but uses - inspect.getfullargspec() for python3 behind the scenes to avoid - DeprecationWarning. - - >>> def f(a, b=2, *ar, **kw): - ... pass - - >>> _getargspec_py23(f) - ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) - """ - return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) - - def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -248,9 +231,9 @@ def get_spec(func): """ if inspect.isfunction(func) or inspect.ismethod(func): - spec = _getargspec_py23(func) + spec = inspect.getfullargspec(func) elif hasattr(func, '__call__'): - spec = _getargspec_py23(func.__call__) + spec = inspect.getfullargspec(func.__call__) else: raise TypeError(f'{type(func)} is not callable') From e60e8224a23ddd4fd98c54318d128f6744f40a6a Mon Sep 17 00:00:00 2001 From: Abinash Satapathy Date: Thu, 6 Oct 2022 19:58:48 +0200 Subject: [PATCH 146/237] Update and rename INSTALL to INSTALL.md --- INSTALL | 4 ---- INSTALL.md | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 INSTALL create mode 100644 INSTALL.md diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 06e812936..000000000 --- a/INSTALL +++ /dev/null @@ -1,4 +0,0 @@ -For information about installing Scrapy see: - -* docs/intro/install.rst (local file) -* https://docs.scrapy.org/en/latest/intro/install.html (online version) diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 000000000..495413f97 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,4 @@ +For information about installing Scrapy see: + +* [Local docs](docs/intro/install.rst) +* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html) From 300c42bfdf0afe0809b03cf571ebe5da7352301e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 14:53:19 +0600 Subject: [PATCH 147/237] Install PyPy using actions/setup-python. --- .github/workflows/tests-ubuntu.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9e62b8e23..633e01c95 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -26,10 +26,9 @@ jobs: - python-version: "3.11.0-rc.2" env: TOXENV: asyncio - - python-version: pypy3 + - python-version: pypy3.9 env: TOXENV: pypy3 - PYPY_VERSION: 3.9-v7.3.9 # pinned deps - python-version: 3.7.13 @@ -38,10 +37,9 @@ jobs: - python-version: 3.7.13 env: TOXENV: asyncio-pinned - - python-version: pypy3 + - python-version: pypy3.7 env: TOXENV: pypy3-pinned - PYPY_VERSION: 3.7-v7.3.5 # extras # extra-deps includes reppy, which does not support Python 3.9 @@ -59,7 +57,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version @@ -68,13 +66,6 @@ jobs: - name: Run tests env: ${{ matrix.env }} run: | - if [[ ! -z "$PYPY_VERSION" ]]; then - export PYPY_VERSION="pypy$PYPY_VERSION-linux64" - wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2" - tar -jxf ${PYPY_VERSION}.tar.bz2 - $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION" - source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" - fi pip install -U tox tox From 4aea925714f7c9beb68d3d478ad528f00d488942 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:01:50 +0600 Subject: [PATCH 148/237] Update action versions. --- .github/workflows/tests-ubuntu.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 633e01c95..9cd0df468 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -49,10 +49,10 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 424849b27582d3a74fab0c43d842576722b48e25 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:17:55 +0600 Subject: [PATCH 149/237] Update mypy. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2804ebe95..12a4516ce 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,8 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.971 + mypy==0.982 + types-attrs==19.1.0 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 commands = From ccb6a8c098501c0f4322faeac17f9b97dad0c5e3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:27:14 +0600 Subject: [PATCH 150/237] Add a note about flake8. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 12a4516ce..7002c2f18 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,7 @@ deps = # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 pytest-flake8==1.1.1 + # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 flake8==4.0.1 commands = pytest --flake8 {posargs:docs scrapy tests} From d1515cc0755ae1d587ffbe4171544442fa10ef8d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Oct 2022 15:30:44 +0600 Subject: [PATCH 151/237] Update pylint. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7002c2f18..463828e7c 100644 --- a/tox.ini +++ b/tox.ini @@ -69,7 +69,7 @@ commands = basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.14.5 + pylint==2.15.3 commands = pylint conftest.py docs extras scrapy setup.py tests From ed9bc84d551c7302cd075aa52f30d38465749f05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Oct 2022 19:11:16 +0600 Subject: [PATCH 152/237] Remove a pin on libxml2-dev. --- .github/workflows/tests-ubuntu.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 9cd0df468..7915a9aab 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -60,8 +60,7 @@ jobs: if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' run: | sudo apt-get update - # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version - sudo apt-get install libxml2-dev/bionic-updates libxslt-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} From 92f2d75ed39e4d75f6cca23b8d3aaf934677f4f3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 8 Oct 2022 19:12:32 +0600 Subject: [PATCH 153/237] Add a classifier for Python 3.11. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8b9c43738..a43cf08c8 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,7 @@ setup( 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 5fa613b419f0d94b7dca8ae4ba2782ca268a9d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:02:57 +0600 Subject: [PATCH 154/237] Run flake8 directly. --- conftest.py | 10 ---------- tox.ini | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index 117087790..d7fe80321 100644 --- a/conftest.py +++ b/conftest.py @@ -42,16 +42,6 @@ def chdir(tmpdir): tmpdir.chdir() -def pytest_collection_modifyitems(session, config, items): - # Avoid executing tests when executing `--flake8` flag (pytest-flake8) - try: - from pytest_flake8 import Flake8Item - if config.getoption('--flake8'): - items[:] = [item for item in items if isinstance(item, Flake8Item)] - except ImportError: - pass - - def pytest_addoption(parser): parser.addoption( "--reactor", diff --git a/tox.ini b/tox.ini index 463828e7c..822e96fde 100644 --- a/tox.ini +++ b/tox.ini @@ -58,11 +58,10 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - pytest-flake8==1.1.1 # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 flake8==4.0.1 commands = - pytest --flake8 {posargs:docs scrapy tests} + flake8 {posargs:docs scrapy tests} [testenv:pylint] # reppy does not support Python 3.9+ From eeb199adda2d7b56c78879df79c7294148ca15f1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:10:59 +0600 Subject: [PATCH 155/237] Fix flake8 issues in previously ignored files. --- .flake8 | 2 ++ docs/_ext/scrapydocs.py | 24 +++++++++---------- scrapy/utils/testsite.py | 2 +- .../CrawlerProcess/asyncio_deferred_signal.py | 1 - .../CrawlerProcess/asyncio_enabled_reactor.py | 4 ++-- tests/CrawlerProcess/reactor_default.py | 3 +-- .../reactor_default_twisted_reactor_select.py | 4 +--- tests/CrawlerProcess/reactor_select.py | 2 -- ..._select_subclass_twisted_reactor_select.py | 4 ---- .../reactor_select_twisted_reactor_select.py | 3 --- 10 files changed, 19 insertions(+), 30 deletions(-) diff --git a/.flake8 b/.flake8 index 1c503fb0b..d7aebc24b 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,8 @@ max-line-length = 119 ignore = W503 exclude = + docs/conf.py + # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 640660943..d02a2e17b 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname): def setup(app): app.add_crossref_type( - directivename = "setting", - rolename = "setting", - indextemplate = "pair: %s; setting", + directivename="setting", + rolename="setting", + indextemplate="pair: %s; setting", ) app.add_crossref_type( - directivename = "signal", - rolename = "signal", - indextemplate = "pair: %s; signal", + directivename="signal", + rolename="signal", + indextemplate="pair: %s; signal", ) app.add_crossref_type( - directivename = "command", - rolename = "command", - indextemplate = "pair: %s; command", + directivename="command", + rolename="command", + indextemplate="pair: %s; command", ) app.add_crossref_type( - directivename = "reqmeta", - rolename = "reqmeta", - indextemplate = "pair: %s; reqmeta", + directivename="reqmeta", + rolename="reqmeta", + indextemplate="pair: %s; reqmeta", ) app.add_role('source', source_role) app.add_role('commit', commit_role) diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index fce77be32..5d3710391 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect): def render(self, request): content = util.Redirect.render(self, request) return content.replace(b'http-equiv=\"refresh\"', - b'http-no-equiv=\"do-not-refresh-me\"') + b'http-no-equiv=\"do-not-refresh-me\"') def test_site(): diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index bdd3c1fef..b83f6a585 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -5,7 +5,6 @@ from typing import Optional from scrapy import Spider from scrapy.crawler import CrawlerProcess from scrapy.utils.defer import deferred_from_coro -from twisted.internet.defer import Deferred class UppercasePipeline: diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f2a93074b..e561d63c7 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncioreactor.install(asyncio.get_event_loop()) -import scrapy -from scrapy.crawler import CrawlerProcess +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 5a21a3717..2c867df61 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor +from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): @@ -14,4 +14,3 @@ process = CrawlerProcess(settings={}) process.crawl(NoRequestsSpider) process.start() - diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index c476722ef..c2b30b044 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,6 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor +from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): @@ -16,5 +16,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index eac6e2f89..ca70c06a0 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -15,5 +15,3 @@ process = CrawlerProcess(settings={}) process.crawl(NoRequestsSpider) process.start() - - diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 47f480605..0035daf1e 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -25,7 +25,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - - - diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index e0d2dab26..4f8394edb 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -17,6 +17,3 @@ process = CrawlerProcess(settings={ process.crawl(NoRequestsSpider) process.start() - - - From 5bf42606792c69268cc34da287293e9406e25883 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Oct 2022 11:14:20 +0600 Subject: [PATCH 156/237] Update flake8. --- tests/test_http_response.py | 6 ++++-- tox.ini | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 5d67a5e74..b42c95045 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -705,7 +705,8 @@ class HtmlResponseTest(TextResponseTest): def test_html_encoding(self): - body = b"""Some page + body = b"""Some page + Price: \xa3100' """ r1 = self.response_class("http://www.example.com", body=body) @@ -719,7 +720,8 @@ class HtmlResponseTest(TextResponseTest): self._assert_response_values(r2, 'iso-8859-1', body) # for conflicting declarations headers must take precedence - body = b"""Some page + body = b"""Some page + Price: \xa3100' """ r3 = self.response_class("http://www.example.com", body=body, diff --git a/tox.ini b/tox.ini index 822e96fde..eee99cb2d 100644 --- a/tox.ini +++ b/tox.ini @@ -58,8 +58,7 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - # newer ones don't work: https://github.com/tholo/pytest-flake8/issues/87 - flake8==4.0.1 + flake8==5.0.4 commands = flake8 {posargs:docs scrapy tests} From b792632046093504b1df8bcb0b92d4cb52ebf436 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Mon, 10 Oct 2022 11:47:02 +0600 Subject: [PATCH 157/237] updated setup-python and checkout workflow --- .github/workflows/checks.yml | 4 ++-- .github/workflows/publish.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e515959ad..cc8f20f44 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -30,10 +30,10 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 44b682830..8e307189d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,10 +7,10 @@ jobs: if: startsWith(github.event.ref, 'refs/tags/') steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: "3.10" From d12fcc555b57dbd62c90212bd1d918d39e1ba45d Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 11 Oct 2022 10:32:45 -0700 Subject: [PATCH 158/237] Link to the Code of Conduct (#5659) --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 358302c76..d416ced3c 100644 --- a/README.rst +++ b/README.rst @@ -95,8 +95,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details. Code of Conduct --------------- -Please note that this project is released with a Contributor Code of Conduct -(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md). +Please note that this project is released with a Contributor `Code of Conduct `_. By participating in this project you agree to abide by its terms. Please report unacceptable behavior to opensource@zyte.com. From 96fb663ae1a95286b0634bc51bc17b42eb29bd0f Mon Sep 17 00:00:00 2001 From: Abdul Rauf Date: Tue, 11 Oct 2022 10:34:18 -0700 Subject: [PATCH 159/237] README: set Bash highlighting for pip install (#5648) --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d416ced3c..970bf2c35 100644 --- a/README.rst +++ b/README.rst @@ -64,7 +64,9 @@ Requirements Install ======= -The quick way:: +The quick way: + +.. code:: bash pip install scrapy From da9a2f8a946d6341be1cdd8cad0616a18bba6dbe Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 12 Oct 2022 11:10:39 -0300 Subject: [PATCH 160/237] Remove mention of minimum PyPy versions from the documentation (#5678) --- docs/intro/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 80a9c16d6..f28f5216a 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -10,7 +10,7 @@ Supported Python versions ========================= Scrapy requires Python 3.7+, either the CPython implementation (default) or -the PyPy 7.3.5+ implementation (see :ref:`python:implementations`). +the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -219,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is 5.9.0. +We recommend using the latest PyPy version. For PyPy3, only Linux installation was tested. Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy. From 715c05d504d22e87935ae42cee55ee35b12c2ebd Mon Sep 17 00:00:00 2001 From: gabrielztk <38334108+gabrielztk@users.noreply.github.com> Date: Thu, 13 Oct 2022 07:22:10 -0300 Subject: [PATCH 161/237] =?UTF-8?q?transport.producer.loseConnection()=20?= =?UTF-8?q?=E2=86=92=20transport.loseConnection()=20(#4995)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/downloader/handlers/http11.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 38935667d..6b8a18f1a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,8 +384,7 @@ class ScrapyAgent: logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": request, "handler": handler.__qualname__}) txresponse._transport.stopProducing() - with suppress(AttributeError): - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() return { "txresponse": txresponse, "body": b"", @@ -417,7 +416,7 @@ class ScrapyAgent: logger.warning(warning_msg, warning_args) - txresponse._transport._producer.loseConnection() + txresponse._transport.loseConnection() raise defer.CancelledError(warning_msg % warning_args) if warnsize and expected_size > warnsize: @@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) self.transport.stopProducing() - self.transport._producer.loseConnection() + self.transport.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) From 62cc26e209b66ce5941f15736f669c75dcac9a59 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Oct 2022 22:03:54 +0600 Subject: [PATCH 162/237] Change TWISTED_REACTOR in the default template. --- docs/topics/settings.rst | 5 +++++ scrapy/templates/project/module/settings.py.tmpl | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a711fd197..0b1ef71cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,6 +1642,11 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. +.. versionchanged:: VERSION + The :command:`startproject` command now sets this setting to + ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated + ``settings.py`` file. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 5e541e2c0..c0c34e986 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -89,3 +89,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' From 22a59d0005c03d866ce98d352024974a9e48e7d1 Mon Sep 17 00:00:00 2001 From: Nirjas Jakilim Date: Fri, 14 Oct 2022 23:41:50 +0600 Subject: [PATCH 163/237] CI: use the latest version of Ubuntu (#5675) --- .github/workflows/checks.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/tests-ubuntu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cc8f20f44..439dfee51 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: checks: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e307189d..f6b098b80 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,7 +3,7 @@ on: [push] jobs: publish: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags/') steps: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 7915a9aab..d2bfe4a5f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: From 043575123c57db9f055c3668a3d80485989df0b2 Mon Sep 17 00:00:00 2001 From: Mohammadtaher Abbasi Date: Sat, 15 Oct 2022 11:41:05 +0330 Subject: [PATCH 164/237] Add async callback support to the parse command (#5577) --- scrapy/commands/parse.py | 55 ++++++++++++++++++++++--------------- tests/test_command_parse.py | 15 ++++++++++ 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9b4fb0ed6..d93ab2ac5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -5,6 +5,8 @@ from typing import Dict from itemadapter import is_item, ItemAdapter from w3lib.url import is_url +from twisted.internet.defer import maybeDeferred + from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display @@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand): if not opts.nolinks: self.print_requests(colour=colour) - def run_callback(self, response, callback, cb_kwargs=None): - cb_kwargs = cb_kwargs or {} + def _get_items_and_requests(self, spider_output, opts, depth, spider, callback): items, requests = [], [] - - for x in iterate_spider_output(callback(response, **cb_kwargs)): + for x in spider_output: if is_item(x): items.append(x) elif isinstance(x, Request): requests.append(x) - return items, requests + return items, requests, opts, depth, spider, callback + + def run_callback(self, response, callback, cb_kwargs=None): + cb_kwargs = cb_kwargs or {} + d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs)) + return d def get_callback_from_rules(self, spider, response): if getattr(spider, 'rules', None): @@ -158,6 +163,25 @@ class Command(BaseRunSpiderCommand): logger.error('No response downloaded for: %(url)s', {'url': url}) + def scraped_data(self, args): + items, requests, opts, depth, spider, callback = args + if opts.pipelines: + itemproc = self.pcrawler.engine.scraper.itemproc + for item in items: + itemproc.process_item(item, spider) + self.add_items(depth, items) + self.add_requests(depth, requests) + + scraped_data = items if opts.output else [] + if depth < opts.depth: + for req in requests: + req.meta['_depth'] = depth + 1 + req.meta['_callback'] = req.callback + req.callback = callback + scraped_data += requests + + return scraped_data + def prepare_request(self, spider, request, opts): def callback(response, **cb_kwargs): # memorize first request @@ -191,23 +215,10 @@ class Command(BaseRunSpiderCommand): # parse items and requests depth = response.meta['_depth'] - items, requests = self.run_callback(response, cb, cb_kwargs) - if opts.pipelines: - itemproc = self.pcrawler.engine.scraper.itemproc - for item in items: - itemproc.process_item(item, spider) - self.add_items(depth, items) - self.add_requests(depth, requests) - - scraped_data = items if opts.output else [] - if depth < opts.depth: - for req in requests: - req.meta['_depth'] = depth + 1 - req.meta['_callback'] = req.callback - req.callback = callback - scraped_data += requests - - return scraped_data + d = self.run_callback(response, cb, cb_kwargs) + d.addCallback(self._get_items_and_requests, opts, depth, spider, callback) + d.addCallback(self.scraped_data) + return d # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0d992be56..8368356e2 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule +from scrapy.utils.test import get_from_asyncio_queue +class AsyncDefAsyncioSpider(scrapy.Spider): + + name = 'asyncdef{self.spider_name}' + + async def parse(self, response): + status = await get_from_asyncio_queue(response.status) + return [scrapy.Item(), dict(foo='bar')] class MySpider(scrapy.Spider): name = '{self.spider_name}' @@ -160,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url('/html')]) self.assertIn("INFO: It Works!", _textmode(stderr)) + @defer.inlineCallbacks + def test_asyncio_parse_items(self): + status, out, stderr = yield self.execute( + ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')] + ) + self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) + @defer.inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( From 75bb516edbc39f9a657e71e75f05f0fd6a33d60d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 10:26:38 +0200 Subject: [PATCH 165/237] Adapt tests to the new value of TWISTED_REACTOR for new projects --- tests/test_commands.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 76d5f3935..eaca41102 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -689,8 +689,15 @@ class MySpider(scrapy.Spider): ]) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) - def test_asyncio_enabled_false(self): + def test_asyncio_enabled_default(self): log = self.get_log(self.debug_log_spider, args=[]) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_asyncio_enabled_false(self): + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor' + ]) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') From 960a7f68f6939744b39d528e28f8f925a2e12ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 15 Oct 2022 11:27:00 +0200 Subject: [PATCH 166/237] Verify that the installed asyncio event loop matches ASYNCIO_EVENT_LOOP (#5529) Co-authored-by: Laerte Pereira <5853172+Laerte@users.noreply.github.com> --- scrapy/crawler.py | 14 ++++++++-- scrapy/utils/reactor.py | 18 ++++++++++++ .../asyncio_enabled_reactor_different_loop.py | 25 +++++++++++++++++ .../asyncio_enabled_reactor_same_loop.py | 28 +++++++++++++++++++ tests/test_crawler.py | 23 +++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py create mode 100644 tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e768bca12..65174d846 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,7 +31,12 @@ from scrapy.utils.log import ( ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -from scrapy.utils.reactor import install_reactor, verify_installed_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + verify_installed_asyncio_event_loop, + verify_installed_reactor, +) logger = logging.getLogger(__name__) @@ -78,17 +83,20 @@ class Crawler: crawler=self, ) - reactor_class = self.settings.get("TWISTED_REACTOR") + reactor_class = self.settings["TWISTED_REACTOR"] + event_loop = self.settings["ASYNCIO_EVENT_LOOP"] if init_reactor: # this needs to be done after the spider settings are merged, # but before something imports twisted.internet.reactor if reactor_class: - install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + install_reactor(reactor_class, event_loop) else: from twisted.internet import reactor # noqa: F401 log_reactor_info() if reactor_class: verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) self.extensions = ExtensionManager.from_crawler(self) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index bc543b230..652733ce8 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -90,6 +90,24 @@ def verify_installed_reactor(reactor_path): raise Exception(msg) +def verify_installed_asyncio_event_loop(loop_path): + from twisted.internet import reactor + loop_class = load_object(loop_path) + if isinstance(reactor._asyncioEventloop, loop_class): + return + installed = ( + f"{reactor._asyncioEventloop.__class__.__module__}" + f".{reactor._asyncioEventloop.__class__.__qualname__}" + ) + specified = f"{loop_class.__module__}.{loop_class.__qualname__}" + raise Exception( + "Scrapy found an asyncio Twisted reactor already " + f"installed, and its event loop class ({installed}) does " + "not match the one specified in the ASYNCIO_EVENT_LOOP " + f"setting ({specified})" + ) + + def is_asyncio_reactor_installed(): from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..ea8242f67 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,25 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..d24bf3031 --- /dev/null +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,28 @@ +import asyncio +import sys + +from uvloop import Loop + +from twisted.internet import asyncioreactor +if sys.version_info >= (3, 8) and sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + +import scrapy # noqa: E402 +from scrapy.crawler import CrawlerProcess # noqa: E402 + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", +}) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cf15ba9b9..c61d461f7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -454,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Using asyncio event loop: uvloop.Loop", log) self.assertIn("async pipeline opened!", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_enabled_reactor_same_loop.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertIn("Using asyncio event loop: uvloop.Loop", log) + + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') + @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') + @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106') + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_enabled_reactor_different_loop.py") + self.assertNotIn("Spider closed (finished)", log) + self.assertIn( + ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ), + log, + ) + def test_default_loop_asyncio_deferred_signal(self): log = self.run_script("asyncio_deferred_signal.py") self.assertIn("Spider closed (finished)", log) From c49764ffd7111d8a465a0d077d0df38bd40449fa Mon Sep 17 00:00:00 2001 From: mattkohl-flex Date: Mon, 17 Oct 2022 11:15:17 +0100 Subject: [PATCH 167/237] typo fixes --- docs/intro/install.rst | 2 +- docs/topics/contracts.rst | 2 +- docs/topics/extensions.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/request-response.rst | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index f28f5216a..9ab479edd 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -187,7 +187,7 @@ solutions: * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be - used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly + used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly if you're using `zsh`_ as default shell):: echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ef296dc9e..c29a3a410 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -102,7 +102,7 @@ override three methods: .. method:: Contract.post_process(output) This allows processing the output of the callback. Iterators are - converted listified before being passed to this hook. + converted to lists before being passed to this hook. Raise :class:`~scrapy.exceptions.ContractFail` from :class:`~scrapy.contracts.Contract.pre_process` or diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 297e1fdc5..130657b0b 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -17,7 +17,7 @@ settings, just like any other Scrapy code. It is customary for extensions to prefix their settings with their own name, to avoid collision with existing (and future) extensions. For example, a -hypothetic extension to handle `Google Sitemaps`_ would use settings like +hypothetical extension to handle `Google Sitemaps`_ would use settings like ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 477652704..33441838a 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -154,7 +154,7 @@ Too many spiders? If your project has too many spiders executed in parallel, the output of :func:`prefs()` can be difficult to read. For this reason, that function has a ``ignore`` argument which can be used to -ignore a particular class (and all its subclases). For +ignore a particular class (and all its subclasses). For example, this won't show any live references to spiders: >>> from scrapy.spiders import Spider diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..7eb6942ac 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -446,7 +446,7 @@ class). Scenarios where changing the request fingerprinting algorithm may cause undesired results include, for example, using the HTTP cache middleware (see :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`). -Changing the request fingerprinting algorithm would invalidade the current +Changing the request fingerprinting algorithm would invalidate the current cache, requiring you to redownload all requests again. Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in From 06c8f673afe9af08784e62d67930a8cbc9887ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 17 Oct 2022 15:04:29 +0200 Subject: [PATCH 168/237] 2.7 release notes (#5680) * Fix the display name of documented fingerprinter class methods * Initial draft for the Scrapy 2.7 release notes * Update VERSION and PREVIOUS_VERSION references * Clarify the restrictions lifted for item field output names * Fix the description of the BOM bug fix * Fix the note about changes in MIME sniffing * Fix typo * Extend highlights * Fyx typo --- docs/_ext/scrapydocs.py | 2 +- docs/news.rst | 193 +++++++++++++++++- docs/topics/components.rst | 4 +- docs/topics/coroutines.rst | 12 +- docs/topics/request-response.rst | 45 ++-- docs/topics/settings.rst | 10 +- docs/topics/spider-middleware.rst | 8 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/utils/request.py | 12 +- scrapy/utils/test.py | 2 +- tests/test_crawl.py | 2 +- tests/test_crawler.py | 8 +- tests/test_dupefilters.py | 12 +- tests/test_pipeline_crawl.py | 2 +- tests/test_scheduler.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_utils_request.py | 4 +- 18 files changed, 259 insertions(+), 65 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index d02a2e17b..f0f382da3 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -15,7 +15,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): - if node.tagname == 'index': + if node.tagname == 'index' and node['entries']: # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] diff --git a/docs/news.rst b/docs/news.rst index 9469d0fe5..d8b9fcd1e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,195 @@ Release notes ============= +.. _release-2.7.0: + +Scrapy 2.7.0 (to be released) +----------------------------- + +Highlights: + +- Added Python 3.11 support, dropped Python 3.6 support +- Improved support for :ref:`asynchronous callbacks ` +- :ref:`Asyncio support ` is enabled by default on new + projects +- Output names of item fields can now be arbitrary strings +- Centralized :ref:`request fingerprinting ` + configuration is now possible + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +Python 3.7 or greater is now required; support for Python 3.6 has been dropped. +Support for the upcoming Python 3.11 has been added. + +The minimum required version of some dependencies has changed as well: + +- lxml_: 3.5.0 → 4.3.0 + +- Pillow_ (:ref:`images pipeline `): 4.0.0 → 7.1.0 + +- zope.interface_: 5.0.0 → 5.1.0 + +(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`, +:issue:`5670`, :issue:`5678`) + + +Deprecations +~~~~~~~~~~~~ + +- :meth:`ImagesPipeline.thumb_path + ` must now accept an + ``item`` parameter (:issue:`5504`, :issue:`5508`). + +- The ``scrapy.downloadermiddlewares.decompression`` module is now + deprecated (:issue:`5546`, :issue:`5547`). + + +New features +~~~~~~~~~~~~ + +- The + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` + method of :ref:`spider middlewares ` can now be + defined as an :term:`asynchronous generator` (:issue:`4978`). + +- The output of :class:`~scrapy.Request` callbacks defined as + :ref:`coroutines ` is now processed asynchronously + (:issue:`4978`). + +- :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous + callbacks ` (:issue:`5657`). + +- New projects created with the :command:`startproject` command have + :ref:`asyncio support ` enabled by default (:issue:`5590`, + :issue:`5679`). + +- The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a + dictionary to customize the output name of item fields, lifting the + restriction that required output names to be valid Python identifiers, e.g. + preventing them to have whitespace (:issue:`1008`, :issue:`3266`, + :issue:`3696`). + +- You can now customize :ref:`request fingerprinting ` + through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of + having to change it on every Scrapy component that relies on request + fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`, + :issue:`4524`). + +- ``jsonl`` is now supported and encouraged as a file extension for `JSON + Lines`_ files (:issue:`4848`). + + .. _JSON Lines: https://jsonlines.org/ + +- :meth:`ImagesPipeline.thumb_path + ` now receives the + source :ref:`item ` (:issue:`5504`, :issue:`5508`). + + +Bug fixes +~~~~~~~~~ + +- When using Google Cloud Storage with a :ref:`media pipeline + `, :setting:`FILES_EXPIRES` now also works when + :setting:`FILES_STORE` does not point at the root of your Google Cloud + Storage bucket (:issue:`5317`, :issue:`5318`). + +- The :command:`parse` command now supports :ref:`asynchronous callbacks + ` (:issue:`5424`, :issue:`5577`). + +- When using the :command:`parse` command with a URL for which there is no + available spider, an exception is no longer raised (:issue:`3264`, + :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`). + +- :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte + order mark`_ when determining the text encoding of the response body, + following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`). + + .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark + .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding + +- MIME sniffing takes the response body into account in FTP and HTTP/1.0 + requests, as well as in cached requests (:issue:`4873`). + +- MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag + is missing (:issue:`4873`). + +- An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value + that does not match the asyncio event loop actually installed + (:issue:`5529`). + +- Fixed :meth:`Headers.getlist ` + returning only the last header (:issue:`5515`, :issue:`5526`). + +- Fixed :class:`LinkExtractor + ` not ignoring the + ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`, + :issue:`4066`) + + +Documentation +~~~~~~~~~~~~~ + +- Clarified the return type of :meth:`Spider.parse ` + (:issue:`5602`, :issue:`5608`). + +- To enable + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + to do `brotli compression`_, installing brotli_ is now recommended instead + of installing brotlipy_, as the former provides a more recent version of + brotli. + + .. _brotli: https://github.com/google/brotli + .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt + +- :ref:`Signal documentation ` now mentions :ref:`coroutine + support ` and uses it in code examples (:issue:`4852`, + :issue:`5358`). + +- :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_ + (:issue:`3582`, :issue:`5432`). + + .. _Common Crawl: https://commoncrawl.org/ + .. _Google cache: http://www.googleguide.com/cached_pages.html + +- The new :ref:`topics-components` topic covers enforcing requirements on + Scrapy components, like :ref:`downloader middlewares + `, :ref:`extensions `, + :ref:`item pipelines `, :ref:`spider middlewares + `, and more; :ref:`enforce-asyncio-requirement` + has also been added (:issue:`4978`). + +- :ref:`topics-settings` now indicates that setting values must be + :ref:`picklable ` (:issue:`5607`, :issue:`5629`). + +- Removed outdated documentation (:issue:`5446`, :issue:`5373`, + :issue:`5369`, :issue:`5370`, :issue:`5554`). + +- Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`, + :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`). + +- Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`, + :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`). + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added a continuous integration job to run `twine check`_ (:issue:`5655`, + :issue:`5656`). + + .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check + +- Addressed test issues and warnings (:issue:`5560`, :issue:`5561`, + :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`, + :issue:`5671`, :issue:`5675`). + +- Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`, + :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`). + +- Applied minor code improvements (:issue:`5661`). + + .. _release-2.6.3: Scrapy 2.6.3 (2022-09-27) @@ -3139,7 +3328,7 @@ New Features ~~~~~~~~~~~~ - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) -- Support `brotli`_-compressed content; requires optional `brotlipy`_ +- Support `brotli-compressed`_ content; requires optional `brotlipy`_ (:issue:`2535`) - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) @@ -3176,7 +3365,7 @@ New Features - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command (:issue:`2740`) -.. _brotli: https://github.com/google/brotli +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://github.com/python-hyper/brotlipy/ Bug fixes diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c44f3def2..ca301b827 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -75,9 +75,9 @@ If your requirement is a minimum Scrapy version, you may use class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('VERSION'): + if parse_version(scrapy.__version__) < parse_version('2.7'): raise RuntimeError( - f"{MyComponent.__qualname__} requires Scrapy VERSION or " + f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " f"method of spider middlewares as an asynchronous " f"generator." diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 750263385..a1ba4ba5c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -22,7 +22,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): If you are using any custom or third-party :ref:`spider middleware `, see :ref:`sync-async-spider-middleware`. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 Output of async callbacks is now processed asynchronously instead of collecting all of it first. @@ -49,7 +49,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): See also :ref:`sync-async-spider-middleware` and :ref:`universal-spider-middleware`. - .. versionadded:: VERSION + .. versionadded:: 2.7 General usage ============= @@ -129,7 +129,7 @@ Common use cases for asynchronous code include: Mixing synchronous and asynchronous spider middlewares ====================================================== -.. versionadded:: VERSION +.. versionadded:: 2.7 The output of a :class:`~scrapy.Request` callback is passed as the ``result`` parameter to the @@ -182,10 +182,10 @@ process_spider_output_async method `. Universal spider middlewares ============================ -.. versionadded:: VERSION +.. versionadded:: 2.7 To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy VERSION and later (avoiding +its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding :ref:`asynchronous-to-synchronous conversions `) while maintaining support for older Scrapy versions, you may define ``process_spider_output`` as a synchronous method and define an @@ -206,7 +206,7 @@ For example:: yield r .. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy VERSION and later without requiring + works in Scrapy 2.7 and later without requiring asynchronous-to-synchronous conversions, and works in earlier Scrapy versions as well. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 49cb69f67..4393e1c68 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -394,7 +394,7 @@ To change how request fingerprints are built for your requests, use the REQUEST_FINGERPRINTER_CLASS ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 Default: :class:`scrapy.utils.request.RequestFingerprinter` @@ -409,38 +409,38 @@ import path. REQUEST_FINGERPRINTER_IMPLEMENTATION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionadded:: VERSION +.. versionadded:: 2.7 -Default: ``'PREVIOUS_VERSION'`` +Default: ``'2.6'`` Determines which request fingerprinting algorithm is used by the default request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`). Possible values are: -- ``'PREVIOUS_VERSION'`` (default) +- ``'2.6'`` (default) This implementation uses the same request fingerprinting algorithm as - Scrapy PREVIOUS_VERSION and earlier versions. + Scrapy 2.6 and earlier versions. Even though this is the default value for backward compatibility reasons, it is a deprecated value. -- ``'VERSION'`` +- ``'2.7'`` - This implementation was introduced in Scrapy VERSION to fix an issue of the + This implementation was introduced in Scrapy 2.7 to fix an issue of the previous implementation. New projects should use this value. The :command:`startproject` command sets this value in the generated ``settings.py`` file. -If you are using the default value (``'PREVIOUS_VERSION'``) for this setting, and you are +If you are using the default value (``'2.6'``) for this setting, and you are using Scrapy components where changing the request fingerprinting algorithm would cause undesired results, you need to carefully decide when to change the value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS` -setting to a custom request fingerprinter class that implements the PREVIOUS_VERSION request +setting to a custom request fingerprinter class that implements the 2.6 request fingerprinting algorithm and does not log this warning ( -:ref:`PREVIOUS_VERSION-request-fingerprinter` includes an example implementation of such a +:ref:`2.6-request-fingerprinter` includes an example implementation of such a class). Scenarios where changing the request fingerprinting algorithm may cause @@ -449,14 +449,14 @@ undesired results include, for example, using the HTTP cache middleware (see Changing the request fingerprinting algorithm would invalidade the current cache, requiring you to redownload all requests again. -Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'VERSION'`` in +Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in your settings to switch already to the request fingerprinting implementation that will be the only request fingerprinting implementation available in a future version of Scrapy, and remove the deprecation warning triggered by using -the default value (``'PREVIOUS_VERSION'``). +the default value (``'2.6'``). -.. _PREVIOUS_VERSION-request-fingerprinter: +.. _2.6-request-fingerprinter: .. _custom-request-fingerprinter: Writing your own request fingerprinter @@ -464,6 +464,8 @@ Writing your own request fingerprinter A request fingerprinter is a class that must implement the following method: +.. currentmodule:: None + .. method:: fingerprint(self, request) Return a :class:`bytes` object that uniquely identifies *request*. @@ -476,6 +478,7 @@ A request fingerprinter is a class that must implement the following method: Additionally, it may also implement the following methods: .. classmethod:: from_crawler(cls, crawler) + :noindex: If present, this class method is called to create a request fingerprinter instance from a :class:`~scrapy.crawler.Crawler` object. It must return a @@ -495,11 +498,13 @@ Additionally, it may also implement the following methods: :class:`~scrapy.settings.Settings` object. It must return a new instance of the request fingerprinter. -The ``fingerprint`` method of the default request fingerprinter, +.. currentmodule:: scrapy.http + +The :meth:`fingerprint` method of the default request fingerprinter, :class:`scrapy.utils.request.RequestFingerprinter`, uses :func:`scrapy.utils.request.fingerprint` with its default parameters. For some -common use cases you can use :func:`~scrapy.utils.request.fingerprint` as well -in your ``fingerprint`` method implementation: +common use cases you can use :func:`scrapy.utils.request.fingerprint` as well +in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint @@ -519,7 +524,7 @@ account:: You can also write your own fingerprinting logic from scratch. -However, if you do not use :func:`~scrapy.utils.request.fingerprint`, make sure +However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints: - Caching saves CPU by ensuring that fingerprints are calculated only once @@ -553,7 +558,7 @@ If you need to be able to override the request fingerprinting for arbitrary requests from your spider callbacks, you may implement a request fingerprinter that reads fingerprints from :attr:`request.meta ` when available, and then falls back to -:func:`~scrapy.utils.request.fingerprint`. For example:: +:func:`scrapy.utils.request.fingerprint`. For example:: from scrapy.utils.request import fingerprint @@ -564,8 +569,8 @@ when available, and then falls back to return request.meta['fingerprint'] return fingerprint(request) -If you need to reproduce the same fingerprinting algorithm as Scrapy PREVIOUS_VERSION -without using the deprecated ``'PREVIOUS_VERSION'`` value of the +If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6 +without using the deprecated ``'2.6'`` value of the :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following request fingerprinter:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0b1ef71cf..40bcda288 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1642,7 +1642,7 @@ install the default reactor defined by Twisted for the current platform. This is to maintain backward compatibility and avoid possible problems caused by using a non-default reactor. -.. versionchanged:: VERSION +.. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. @@ -1661,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. -This setting can act as a stopping condition in case of URLs of ever-increasing -length, which may be caused for example by a programming error either in the -target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and +This setting can act as a stopping condition in case of URLs of ever-increasing +length, which may be caused for example by a programming error either in the +target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and :setting:`DEPTH_LIMIT`. Use ``0`` to allow URLs of any length. -The default value is copied from the `Microsoft Internet Explorer maximum URL +The default value is copied from the `Microsoft Internet Explorer maximum URL length`_, even though this setting exists for different reasons. .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246 diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 816cb5e03..303401a3c 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -105,17 +105,17 @@ object gives you access, for example, to the :ref:`settings `. :class:`~scrapy.Request` objects and :ref:`item objects `. - .. versionchanged:: VERSION + .. versionchanged:: 2.7 This method may be defined as an :term:`asynchronous generator`, in which case ``result`` is an :term:`asynchronous iterable`. Consider defining this method as an :term:`asynchronous generator`, which will be a requirement in a future version of Scrapy. However, if you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy VERSION ` + either :ref:`enforcing Scrapy 2.7 ` as a minimum requirement of your spider middleware, or :ref:`making your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy VERSION. + it works with Scrapy versions earlier than Scrapy 2.7. :param response: the response which generated this output from the spider @@ -130,7 +130,7 @@ object gives you access, for example, to the :ref:`settings `. .. method:: process_spider_output_async(response, result, spider) - .. versionadded:: VERSION + .. versionadded:: 2.7 If defined, this method must be an :term:`asynchronous generator`, which will be called instead of :meth:`process_spider_output` if diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ff86af125..29ff028be 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -248,7 +248,7 @@ REFERER_ENABLED = True REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter' -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'PREVIOUS_VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index c0c34e986..bbf60982c 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -88,5 +88,5 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # Set settings whose default value is deprecated to a future-proof value -REQUEST_FINGERPRINTER_IMPLEMENTATION = 'VERSION' +REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cf33317ce..fbddc41fb 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -236,10 +236,10 @@ class RequestFingerprinter: 'REQUEST_FINGERPRINTER_IMPLEMENTATION' ) else: - implementation = 'PREVIOUS_VERSION' - if implementation == 'PREVIOUS_VERSION': + implementation = '2.6' + if implementation == '2.6': message = ( - '\'PREVIOUS_VERSION\' is a deprecated value for the ' + '\'2.6\' is a deprecated value for the ' '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n' '\n' 'It is also the default value. In other words, it is normal ' @@ -254,14 +254,14 @@ class RequestFingerprinter: ) warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) self._fingerprint = _request_fingerprint_as_bytes - elif implementation == 'VERSION': + elif implementation == '2.7': self._fingerprint = fingerprint else: raise ValueError( f'Got an invalid value on setting ' f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': ' - f'{implementation!r}. Valid values are \'PREVIOUS_VERSION\' (deprecated) ' - f'and \'VERSION\'.' + f'{implementation!r}. Valid values are \'2.6\' (deprecated) ' + f'and \'2.7\'.' ) def fingerprint(self, request): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 0b828f7c0..445cd2e3a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -65,7 +65,7 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): # Set by default settings that prevent deprecation warnings. settings = {} if prevent_warnings: - settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = 'VERSION' + settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7' settings.update(settings_dict or {}) runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8be4b6fe1..5383ec652 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -350,7 +350,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c61d461f7..da6024c2b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -110,7 +110,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): 'LOG_LEVEL': 'INFO', 'LOG_FILE': log_file, # settings to avoid extra warnings - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, } @@ -235,7 +235,7 @@ class NoRequestsSpider(scrapy.Spider): class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'}) + return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}) @defer.inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -283,14 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "REQUEST_FINGERPRINTER_IMPLEMENTATION": "VERSION", + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", }) yield runner.crawl(NoRequestsSpider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 8a37a8ebe..6ebb716b0 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -51,7 +51,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -60,7 +60,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -68,7 +68,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_direct_scheduler(self): settings = {'DUPEFILTER_CLASS': DirectDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -172,7 +172,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -199,7 +199,7 @@ class RFPDupeFilterTest(unittest.TestCase): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) @@ -233,7 +233,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION'} + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'} crawler = get_crawler(SimpleSpider, settings_dict=settings) spider = SimpleSpider.from_crawler(crawler) dupefilter = _get_dupefilter(crawler=crawler) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e46532a1c..0e174cd34 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -64,7 +64,7 @@ class FileDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', 'ITEM_PIPELINES': {self.pipeline_class: 1}, self.store_setting_key: self.tmpmediastore, } diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ac66056ba..50a7755c1 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -52,7 +52,7 @@ class MockCrawler(Crawler): SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', - REQUEST_FINGERPRINTER_IMPLEMENTATION='VERSION', + REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7', ) super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3719c7c9f..7a590f96c 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -98,7 +98,7 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.spider1' runner = CrawlerRunner({ 'SPIDER_MODULES': [module], - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', }) self.assertRaisesRegex(KeyError, 'Spider not found', diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 8bc7922b6..a92d9a0ac 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -505,7 +505,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_deprecated_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'PREVIOUS_VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) @@ -518,7 +518,7 @@ class RequestFingerprinterTestCase(unittest.TestCase): def test_recommended_implementation(self): settings = { - 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', + 'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7', } with warnings.catch_warnings(record=True) as logged_warnings: crawler = get_crawler(settings_dict=settings) From 20b79a0f2e47800bf4648c7f890c8170fc8f5ede Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Oct 2022 19:09:22 +0600 Subject: [PATCH 169/237] =?UTF-8?q?Bump=20version:=202.6.2=20=E2=86=92=202?= =?UTF-8?q?.7.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 2 +- scrapy/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2e2f7949a..f88071685 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.6.2 +current_version = 2.7.0 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index d8b9fcd1e..1ec183a1d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,7 +5,7 @@ Release notes .. _release-2.7.0: -Scrapy 2.7.0 (to be released) +Scrapy 2.7.0 (2022-10-17) ----------------------------- Highlights: diff --git a/scrapy/VERSION b/scrapy/VERSION index 097a15a2a..24ba9a38d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.6.2 +2.7.0 From 40d9ca3bdd6fd50438295f91cff275fcac57e0fa Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Mon, 17 Oct 2022 17:40:10 -0400 Subject: [PATCH 170/237] use pathlib --- conftest.py | 2 +- docs/Makefile | 4 +- docs/conf.py | 9 +- docs/conftest.py | 9 +- docs/intro/tutorial.rst | 10 +- docs/topics/item-pipeline.rst | 4 +- docs/topics/media-pipeline.rst | 9 +- docs/utils/linkfix.py | 9 +- scrapy/commands/__init__.py | 4 +- scrapy/commands/genspider.py | 48 +++-- scrapy/commands/runspider.py | 21 +- scrapy/commands/startproject.py | 44 ++-- scrapy/core/downloader/handlers/file.py | 5 +- scrapy/core/scheduler.py | 19 +- scrapy/crawler.py | 8 +- scrapy/dupefilters.py | 4 +- scrapy/extensions/feedexport.py | 16 +- scrapy/extensions/httpcache.py | 48 +++-- scrapy/extensions/spiderstate.py | 12 +- scrapy/http/response/__init__.py | 4 +- scrapy/pipelines/files.py | 41 ++-- scrapy/settings/default_settings.py | 4 +- scrapy/spiders/__init__.py | 9 +- scrapy/squeues.py | 11 +- scrapy/utils/conf.py | 32 +-- scrapy/utils/job.py | 6 +- scrapy/utils/project.py | 29 +-- scrapy/utils/request.py | 2 +- scrapy/utils/template.py | 12 +- scrapy/utils/test.py | 5 +- setup.py | 5 +- tests/__init__.py | 17 +- tests/keys/__init__.py | 22 +- tests/mockserver.py | 7 +- tests/test_cmdline/__init__.py | 14 +- .../__init__.py | 4 +- tests/test_command_check.py | 9 +- tests/test_command_parse.py | 22 +- tests/test_command_shell.py | 6 +- tests/test_commands.py | 199 +++++++++--------- tests/test_crawler.py | 32 ++- tests/test_dependencies.py | 7 +- tests/test_downloader_handlers.py | 69 +++--- ...st_downloadermiddleware_httpcompression.py | 7 +- tests/test_dupefilters.py | 4 +- tests/test_engine.py | 6 +- tests/test_feedexport.py | 155 +++++++------- tests/test_http2_client_protocol.py | 15 +- tests/test_pipeline_crawl.py | 14 +- tests/test_pipeline_files.py | 3 +- tests/test_proxy_connect.py | 6 +- tests/test_spiderloader/__init__.py | 40 ++-- tests/test_spiderstate.py | 4 +- tests/test_utils_gz.py | 44 ++-- tests/test_utils_iterators.py | 7 - tests/test_utils_misc/__init__.py | 3 +- tests/test_utils_project.py | 17 +- tests/test_utils_response.py | 12 +- tests/test_utils_template.py | 22 +- tests/test_webclient.py | 19 +- 60 files changed, 595 insertions(+), 636 deletions(-) diff --git a/conftest.py b/conftest.py index d7fe80321..7c1da3556 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with open('tests/ignores.txt') as reader: +with Path('tests/ignores.txt').open() as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/Makefile b/docs/Makefile index 87d5d3047..596cb6cef 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -86,8 +86,8 @@ coverage: BUILDER = coverage coverage: build htmlview: html - $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ - os.path.realpath('build/html/index.html'))" + $(PYTHON) -c "import webbrowser; from pathlib import Path; \ + webbrowser.open('file://' + Path('build/html/index.html').resolve())" clean: -rm -rf build/* diff --git a/docs/conf.py b/docs/conf.py index 3241295af..d2a77003e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,13 +11,12 @@ import sys from datetime import datetime -from os import path +from pathlib import Path # If your extensions are in another directory, add it here. If the directory -# is relative to the documentation root, use os.path.abspath to make it -# absolute, like shown here. -sys.path.append(path.join(path.dirname(__file__), "_ext")) -sys.path.insert(0, path.dirname(path.dirname(__file__))) +# is relative to the documentation root, use Path.absolute to make it absolute. +sys.path.append(str(Path(__file__).parent / "_ext")) +sys.path.insert(0, str(Path(__file__).parent.parent)) # General configuration diff --git a/docs/conftest.py b/docs/conftest.py index a0636f8ac..24a72a4b6 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,5 +1,5 @@ -import os from doctest import ELLIPSIS, NORMALIZE_WHITESPACE +from pathlib import Path from scrapy.http.response.html import HtmlResponse from sybil import Sybil @@ -12,10 +12,9 @@ from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip -def load_response(url, filename): - input_path = os.path.join(os.path.dirname(__file__), '_tests', filename) - with open(input_path, 'rb') as input_file: - return HtmlResponse(url, body=input_file.read()) +def load_response(url: str, filename: str) -> HtmlResponse: + input_path = Path(__file__).parent / '_tests' / filename + return HtmlResponse(url, body=input_path.read_bytes()) def setup(namespace): diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 092123d1d..901a170b4 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -85,6 +85,8 @@ page content to extract data. This is the code for our first Spider. Save it in a file named ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project:: + from pathlib import Path + import scrapy @@ -102,8 +104,7 @@ This is the code for our first Spider. Save it in a file named def parse(self, response): page = response.url.split("/")[-2] filename = f'quotes-{page}.html' - with open(filename, 'wb') as f: - f.write(response.body) + Path(filename).write_bytes(response.body) self.log(f'Saved file {filename}') @@ -178,6 +179,8 @@ with a list of URLs. This list will then be used by the default implementation of :meth:`~scrapy.Spider.start_requests` to create the initial requests for your spider:: + from pathlib import Path + import scrapy @@ -191,8 +194,7 @@ for your spider:: def parse(self, response): page = response.url.split("/")[-2] filename = f'quotes-{page}.html' - with open(filename, 'wb') as f: - f.write(response.body) + Path(filename).write_bytes(response.body) The :meth:`~scrapy.Spider.parse` method will be called to handle each of the requests for those URLs, even though we haven't explicitly told Scrapy diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index af294f52c..1672ccbcc 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -186,6 +186,7 @@ item. :: import hashlib + from pathlib import Path from urllib.parse import quote import scrapy @@ -214,8 +215,7 @@ item. url = adapter["url"] url_hash = hashlib.md5(url.encode("utf8")).hexdigest() filename = f"{url_hash}.png" - with open(filename, "wb") as f: - f.write(response.body) + Path(filename).write_bytes(response.body) # Store filename in item. adapter["screenshot_filename"] = filename diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 0925e6bb5..a528746b0 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -156,7 +156,6 @@ By overriding ``file_path`` like this: .. code-block:: python import hashlib - from os.path import splitext def file_path(self, request, response=None, info=None, *, item=None): image_url_hash = hashlib.shake_256(request.url.encode()).hexdigest(5) @@ -498,7 +497,7 @@ See here the methods that you can override in your custom Files Pipeline: approach to download all files into the ``files`` folder with their original filenames (e.g. ``files/foo.png``):: - import os + from pathlib import PurePosixPath from urllib.parse import urlparse from scrapy.pipelines.files import FilesPipeline @@ -506,7 +505,7 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return 'files/' + os.path.basename(urlparse(request.url).path) + return 'files/' + PurePosixPath(urlparse(request.url).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. @@ -637,7 +636,7 @@ See here the methods that you can override in your custom Images Pipeline: approach to download all files into the ``files`` folder with their original filenames (e.g. ``files/foo.png``):: - import os + from pathlib import PurePosixPath from urllib.parse import urlparse from scrapy.pipelines.images import ImagesPipeline @@ -645,7 +644,7 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): def file_path(self, request, response=None, info=None, *, item=None): - return 'files/' + os.path.basename(urlparse(request.url).path) + return 'files/' + PurePosixPath(urlparse(request.url).path).name Similarly, you can use the ``item`` to determine the file path based on some item property. diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 95a3f17d5..7a0c5288c 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -13,6 +13,7 @@ Author: dufferzafar """ import re +from pathlib import Path def main(): @@ -27,7 +28,7 @@ def main(): # Read lines from the linkcheck output file try: - with open("build/linkcheck/output.txt") as out: + with Path("build/linkcheck/output.txt").open() as out: output_lines = out.readlines() except IOError: print("linkcheck output not found; please run linkcheck first.") @@ -51,14 +52,12 @@ def main(): # Update the previous file if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) + Path(_filename).write_text(_contents) _filename = newfilename # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() + _contents = Path(_filename).read_text() _contents = _contents.replace(match.group(3), match.group(4)) else: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index fb304b8c0..8570d90bd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -3,6 +3,7 @@ Base class for Scrapy commands """ import os import argparse +from pathlib import Path from typing import Any, Dict from twisted.python import failure @@ -93,8 +94,7 @@ class ScrapyCommand: self.settings.set('LOG_ENABLED', False, priority='cmdline') if opts.pidfile: - with open(opts.pidfile, "w") as f: - f.write(str(os.getpid()) + os.linesep) + Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep) if opts.pdb: failure.startDebugMode() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index ed5f588e9..01b4a0dbd 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -2,8 +2,9 @@ import os import shutil import string +from pathlib import Path from importlib import import_module -from os.path import join, dirname, abspath, exists, splitext +from typing import Optional from urllib.parse import urlparse import scrapy @@ -62,8 +63,7 @@ class Command(ScrapyCommand): if opts.dump: template_file = self._find_template(opts.dump) if template_file: - with open(template_file, "r") as f: - print(f.read()) + print(template_file.read_text()) return if len(args) != 2: raise UsageError() @@ -98,11 +98,11 @@ class Command(ScrapyCommand): } if self.settings.get('NEWSPIDER_MODULE'): spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = abspath(dirname(spiders_module.__file__)) + spiders_dir = Path(spiders_module.__file__).parent.resolve() else: spiders_module = None - spiders_dir = "." - spider_file = f"{join(spiders_dir, module)}.py" + spiders_dir = Path(".") + spider_file = f"{spiders_dir / module}.py" shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) print(f"Created spider {name!r} using template {template_name!r} ", @@ -110,24 +110,25 @@ class Command(ScrapyCommand): if spiders_module: print(f"in module:\n {spiders_module.__name__}.{module}") - def _find_template(self, template): - template_file = join(self.templates_dir, f'{template}.tmpl') - if exists(template_file): + def _find_template(self, template: str) -> Optional[Path]: + template_file = Path(self.templates_dir, f'{template}.tmpl') + if template_file.exists(): return template_file print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') def _list_templates(self): print("Available templates:") - for filename in sorted(os.listdir(self.templates_dir)): - if filename.endswith('.tmpl'): - print(f" {splitext(filename)[0]}") + for file in sorted(Path(self.templates_dir).iterdir()): + if file.suffix == '.tmpl': + print(f" {file.stem}") - def _spider_exists(self, name): + def _spider_exists(self, name: str) -> bool: if not self.settings.get('NEWSPIDER_MODULE'): # if run as a standalone command and file with same filename already exists - if exists(name + ".py"): - print(f"{abspath(name + '.py')} already exists") + path = Path(name + ".py") + if path.exists(): + print(f"{path.resolve()} already exists") return True return False @@ -143,17 +144,18 @@ class Command(ScrapyCommand): # a file with the same name exists in the target directory spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = dirname(spiders_module.__file__) - spiders_dir_abs = abspath(spiders_dir) - if exists(join(spiders_dir_abs, name + ".py")): - print(f"{join(spiders_dir_abs, (name + '.py'))} already exists") + spiders_dir = Path(spiders_module.__file__).parent + spiders_dir_abs = spiders_dir.resolve() + path = spiders_dir_abs / (name + ".py") + if path.exists(): + print(f"{path} already exists") return True return False @property - def templates_dir(self): - return join( - self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + def templates_dir(self) -> str: + return str(Path( + self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'), 'spiders' - ) + )) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index b957c29fb..c41135508 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,22 +1,23 @@ import sys -import os +from os import PathLike +from pathlib import Path from importlib import import_module +from types import ModuleType from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand -def _import_file(filepath): - abspath = os.path.abspath(filepath) - dirname, file = os.path.split(abspath) - fname, fext = os.path.splitext(file) - if fext not in ('.py', '.pyw'): +def _import_file(filepath: str | PathLike[str]) -> ModuleType: + abspath = Path(filepath).resolve() + dirname = str(abspath.parent) + if abspath.suffix not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") if dirname: sys.path = [dirname] + sys.path try: - module = import_module(fname) + module = import_module(abspath.stem) finally: if dirname: sys.path.pop(0) @@ -40,13 +41,13 @@ class Command(BaseRunSpiderCommand): def run(self, args, opts): if len(args) != 1: raise UsageError() - filename = args[0] - if not os.path.exists(filename): + filename = Path(args[0]) + if not filename.exists(): raise UsageError(f"File not found: {filename}\n") try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError(f"Unable to load {filename!r}: {e}\n") + raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 1b6374c39..4323cdb53 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -2,7 +2,7 @@ import re import os import string from importlib.util import find_spec -from os.path import join, exists, abspath +from pathlib import Path from shutil import ignore_patterns, move, copy2, copystat from stat import S_IWUSR as OWNER_WRITE_PERMISSION @@ -54,7 +54,7 @@ class Command(ScrapyCommand): return True return False - def _copytree(self, src, dst): + def _copytree(self, src: Path, dst: Path): """ Since the original function always creates the directory, to resolve the issue a new function had to be created. It's a simple copy and @@ -64,19 +64,19 @@ class Command(ScrapyCommand): https://github.com/scrapy/scrapy/pull/2005 """ ignore = IGNORE - names = os.listdir(src) + names = [x.name for x in src.iterdir()] ignored_names = ignore(src, names) - if not os.path.exists(dst): - os.makedirs(dst) + if not dst.exists(): + dst.mkdir(parents=True) for name in names: if name in ignored_names: continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - if os.path.isdir(srcname): + srcname = src / name + dstname = dst / name + if srcname.is_dir(): self._copytree(srcname, dstname) else: copy2(srcname, dstname) @@ -90,36 +90,36 @@ class Command(ScrapyCommand): raise UsageError() project_name = args[0] - project_dir = args[0] if len(args) == 2: - project_dir = args[1] + project_dir = Path(args[1]) + else: + project_dir = Path(args[0]) - if exists(join(project_dir, 'scrapy.cfg')): + if (project_dir / 'scrapy.cfg').exists(): self.exitcode = 1 - print(f'Error: scrapy.cfg already exists in {abspath(project_dir)}') + print(f'Error: scrapy.cfg already exists in {project_dir.resolve()}') return if not self._is_valid_name(project_name): self.exitcode = 1 return - self._copytree(self.templates_dir, abspath(project_dir)) - move(join(project_dir, 'module'), join(project_dir, project_name)) + self._copytree(Path(self.templates_dir), project_dir.resolve()) + move(project_dir / 'module', project_dir / project_name) for paths in TEMPLATES_TO_RENDER: - path = join(*paths) - tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) - render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) + tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths)) + render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name)) print(f"New Scrapy project '{project_name}', using template directory " f"'{self.templates_dir}', created in:") - print(f" {abspath(project_dir)}\n") + print(f" {project_dir.resolve()}\n") print("You can start your first spider with:") print(f" cd {project_dir}") print(" scrapy genspider example example.com") @property - def templates_dir(self): - return join( - self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + def templates_dir(self) -> str: + return str(Path( + self.settings['TEMPLATES_DIR'] or Path(scrapy.__path__[0], 'templates'), 'project' - ) + )) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 0d94e3df0..4824167da 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -1,3 +1,5 @@ +from pathlib import Path + from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes @@ -10,7 +12,6 @@ class FileDownloadHandler: @defers def download_request(self, request, spider): filepath = file_uri_to_path(request.url) - with open(filepath, 'rb') as fo: - body = fo.read() + body = Path(filepath).read_bytes() respcls = responsetypes.from_args(filename=filepath, body=body) return respcls(url=request.url, body=body) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 5ba0fb63b..366449f51 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,8 +1,7 @@ import json import logging -import os from abc import abstractmethod -from os.path import exists, join +from pathlib import Path from typing import Optional, Type, TypeVar from twisted.internet.defer import Deferred @@ -324,19 +323,19 @@ class Scheduler(BaseScheduler): def _dqdir(self, jobdir: Optional[str]) -> Optional[str]: """ Return a folder name to keep disk queue state at """ if jobdir is not None: - dqdir = join(jobdir, 'requests.queue') - if not exists(dqdir): - os.makedirs(dqdir) - return dqdir + dqdir = Path(jobdir, 'requests.queue') + if not dqdir.exists(): + dqdir.mkdir(parents=True) + return str(dqdir) return None def _read_dqs_state(self, dqdir: str) -> list: - path = join(dqdir, 'active.json') - if not exists(path): + path = Path(dqdir, 'active.json') + if not path.exists(): return [] - with open(path) as f: + with path.open() as f: return json.load(f) def _write_dqs_state(self, dqdir: str, state: list) -> None: - with open(join(dqdir, 'active.json'), 'w') as f: + with Path(dqdir, 'active.json').open('w') as f: json.dump(state, f) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e768bca12..b7108cdcc 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import logging import pprint import signal import warnings +from typing import TYPE_CHECKING from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -33,6 +36,9 @@ from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import install_reactor, verify_installed_reactor +if TYPE_CHECKING: + from scrapy.utils.request import RequestFingerprinter + logger = logging.getLogger(__name__) @@ -72,7 +78,7 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) - self.request_fingerprinter = create_instance( + self.request_fingerprinter: RequestFingerprinter = create_instance( load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']), settings=self.settings, crawler=self, diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index d1b0559ef..2b8b09614 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,5 +1,5 @@ import logging -import os +from pathlib import Path from typing import Optional, Set, Type, TypeVar from warnings import warn @@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = open(os.path.join(path, 'requests.seen'), 'a+') + self.file = Path(path, 'requests.seen').open('a+') self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e7097b7a1..0aa27e417 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -5,13 +5,13 @@ See documentation in docs/topics/feed-exports.rst """ import logging -import os import re import sys import warnings from datetime import datetime +from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Any, Callable, Optional, Tuple, Union +from typing import IO, Any, Callable, Optional, Tuple, Union from urllib.parse import unquote, urlparse from twisted.internet import defer, threads @@ -101,7 +101,7 @@ class BlockingFeedStorage: def open(self, spider): path = spider.crawler.settings['FEED_TEMPDIR'] - if path and not os.path.isdir(path): + if path and not Path(path).is_dir(): raise OSError('Not a Directory: ' + str(path)) return NamedTemporaryFile(prefix='feed-', dir=path) @@ -141,11 +141,11 @@ class FileFeedStorage: feed_options = feed_options or {} self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab' - def open(self, spider): - dirname = os.path.dirname(self.path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - return open(self.path, self.write_mode) + def open(self, spider) -> IO[Any]: + dirname = Path(self.path).parent + if dirname and not dirname.exists(): + dirname.mkdir(parents=True) + return Path(self.path).open(self.write_mode) def store(self, file): file.close() diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 843e14812..3057bf157 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,16 +1,18 @@ import gzip import logging -import os import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module +from pathlib import Path from time import time from weakref import WeakKeyDictionary from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers, Response +from scrapy.http.request import Request from scrapy.responsetypes import responsetypes +from scrapy.spiders import Spider from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode @@ -221,9 +223,9 @@ class DbmCacheStorage: self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None - def open_spider(self, spider): - dbpath = os.path.join(self.cachedir, f'{spider.name}.db') - self.db = self.dbmodule.open(dbpath, 'c') + def open_spider(self, spider: Spider): + dbpath = Path(self.cachedir, f'{spider.name}.db') + self.db = self.dbmodule.open(str(dbpath), 'c') logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider}) @@ -277,7 +279,7 @@ class FilesystemCacheStorage: self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open - def open_spider(self, spider): + def open_spider(self, spider: Spider): logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir}, extra={'spider': spider}) @@ -286,15 +288,15 @@ class FilesystemCacheStorage: def close_spider(self, spider): pass - def retrieve_response(self, spider, request): + def retrieve_response(self, spider: Spider, request: Request): """Return response if present in cache, or None otherwise.""" metadata = self._read_meta(spider, request) if metadata is None: return # not cached rpath = self._get_request_path(spider, request) - with self._open(os.path.join(rpath, 'response_body'), 'rb') as f: + with self._open(rpath / 'response_body', 'rb') as f: body = f.read() - with self._open(os.path.join(rpath, 'response_headers'), 'rb') as f: + with self._open(rpath / 'response_headers', 'rb') as f: rawheaders = f.read() url = metadata.get('response_url') status = metadata['status'] @@ -303,11 +305,11 @@ class FilesystemCacheStorage: response = respcls(url=url, headers=headers, status=status, body=body) return response - def store_response(self, spider, request, response): + def store_response(self, spider: Spider, request: Request, response): """Store the given response in the cache.""" rpath = self._get_request_path(spider, request) - if not os.path.exists(rpath): - os.makedirs(rpath) + if not rpath.exists(): + rpath.mkdir(parents=True) metadata = { 'url': request.url, 'method': request.method, @@ -315,29 +317,29 @@ class FilesystemCacheStorage: 'response_url': response.url, 'timestamp': time(), } - with self._open(os.path.join(rpath, 'meta'), 'wb') as f: + with self._open(rpath / 'meta', 'wb') as f: f.write(to_bytes(repr(metadata))) - with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: + with self._open(rpath / 'pickled_meta', 'wb') as f: pickle.dump(metadata, f, protocol=4) - with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: + with self._open(rpath / 'response_headers', 'wb') as f: f.write(headers_dict_to_raw(response.headers)) - with self._open(os.path.join(rpath, 'response_body'), 'wb') as f: + with self._open(rpath / 'response_body', 'wb') as f: f.write(response.body) - with self._open(os.path.join(rpath, 'request_headers'), 'wb') as f: + with self._open(rpath / 'request_headers', 'wb') as f: f.write(headers_dict_to_raw(request.headers)) - with self._open(os.path.join(rpath, 'request_body'), 'wb') as f: + with self._open(rpath / 'request_body', 'wb') as f: f.write(request.body) - def _get_request_path(self, spider, request): + def _get_request_path(self, spider: Spider, request: Request) -> Path: key = self._fingerprinter.fingerprint(request).hex() - return os.path.join(self.cachedir, spider.name, key[0:2], key) + return Path(self.cachedir, spider.name, key[0:2], key) - def _read_meta(self, spider, request): + def _read_meta(self, spider: Spider, request: Request): rpath = self._get_request_path(spider, request) - metapath = os.path.join(rpath, 'pickled_meta') - if not os.path.exists(metapath): + metapath = rpath / 'pickled_meta' + if not metapath.exists(): return # not found - mtime = os.stat(metapath).st_mtime + mtime = metapath.stat().st_mtime if 0 < self.expiration_secs < time() - mtime: return # expired with self._open(metapath, 'rb') as f: diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index bea00596e..e9c8b1d6a 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,5 +1,5 @@ -import os import pickle +from pathlib import Path from scrapy import signals from scrapy.exceptions import NotConfigured @@ -25,16 +25,16 @@ class SpiderState: def spider_closed(self, spider): if self.jobdir: - with open(self.statefn, 'wb') as f: + with Path(self.statefn).open('wb') as f: pickle.dump(spider.state, f, protocol=4) def spider_opened(self, spider): - if self.jobdir and os.path.exists(self.statefn): - with open(self.statefn, 'rb') as f: + if self.jobdir and Path(self.statefn).exists(): + with Path(self.statefn).open('rb') as f: spider.state = pickle.load(f) else: spider.state = {} @property - def statefn(self): - return os.path.join(self.jobdir, 'spider.state') + def statefn(self) -> str: + return str(Path(self.jobdir, 'spider.state')) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 4de6c9b5b..9359cc7c8 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -32,7 +32,7 @@ class Response(object_ref): def __init__( self, - url, + url: str, status=200, headers=None, body=b"", @@ -75,7 +75,7 @@ class Response(object_ref): def _get_url(self): return self._url - def _set_url(self, url): + def _set_url(self, url: str): if isinstance(url, str): self._url = url else: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 906e7eb24..ffb12d910 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -13,6 +13,8 @@ from collections import defaultdict from contextlib import suppress from ftplib import FTP from io import BytesIO +from pathlib import Path +from typing import Optional from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -39,41 +41,40 @@ class FileException(Exception): class FSFilesStore: - def __init__(self, basedir): + def __init__(self, basedir: str): if '://' in basedir: basedir = basedir.split('://', 1)[1] self.basedir = basedir - self._mkdir(self.basedir) - self.created_directories = defaultdict(set) + self._mkdir(Path(self.basedir)) + self.created_directories: defaultdict[str, set[str]] = defaultdict(set) - def persist_file(self, path, buf, info, meta=None, headers=None): + def persist_file(self, path: str, buf, info, meta=None, headers=None): absolute_path = self._get_filesystem_path(path) - self._mkdir(os.path.dirname(absolute_path), info) - with open(absolute_path, 'wb') as f: - f.write(buf.getvalue()) + self._mkdir(absolute_path.parent, info) + absolute_path.write_bytes(buf.getvalue()) - def stat_file(self, path, info): + def stat_file(self, path: str, info): absolute_path = self._get_filesystem_path(path) try: - last_modified = os.path.getmtime(absolute_path) + last_modified = absolute_path.stat().st_mtime except os.error: return {} - with open(absolute_path, 'rb') as f: + with absolute_path.open('rb') as f: checksum = md5sum(f) return {'last_modified': last_modified, 'checksum': checksum} - def _get_filesystem_path(self, path): + def _get_filesystem_path(self, path: str) -> Path: path_comps = path.split('/') - return os.path.join(self.basedir, *path_comps) + return Path(self.basedir, *path_comps) - def _mkdir(self, dirname, domain=None): + def _mkdir(self, dirname: Path, domain: Optional[str] = None): seen = self.created_directories[domain] if domain else set() - if dirname not in seen: - if not os.path.exists(dirname): - os.makedirs(dirname) - seen.add(dirname) + if str(dirname) not in seen: + if not dirname.exists(): + dirname.mkdir(parents=True) + seen.add(str(dirname)) class S3FilesStore: @@ -374,8 +375,8 @@ class FilesPipeline(MediaPipeline): store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) - def _get_store(self, uri): - if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir + def _get_store(self, uri: str): + if Path(uri).is_absolute(): # to support win32 paths like: C:\\some\dir scheme = 'file' else: scheme = urlparse(uri).scheme @@ -510,7 +511,7 @@ class FilesPipeline(MediaPipeline): def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() - media_ext = os.path.splitext(request.url)[1] + media_ext = Path(request.url).suffix # Handles empty and wild extensions by trying to guess the # mime type then extension or default to empty string otherwise if media_ext not in mimetypes.types_map: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ff86af125..84e0a94d2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -15,7 +15,7 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module -from os.path import join, abspath, dirname +from pathlib import Path AJAXCRAWL_ENABLED = False @@ -288,7 +288,7 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] -TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) +TEMPLATES_DIR = str((Path(__file__).parent / '..' / 'templates').resolve()) URLLENGTH_LIMIT = 2083 diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index d8248c606..9a97e7801 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -3,14 +3,19 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ +from __future__ import annotations + import logging -from typing import Optional +from typing import TYPE_CHECKING, Optional from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -49,7 +54,7 @@ class Spider(object_ref): spider._set_crawler(crawler) return spider - def _set_crawler(self, crawler): + def _set_crawler(self, crawler: Crawler): self.crawler = crawler self.settings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index dff9b1350..1f2dee55f 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,8 +3,9 @@ Scheduler queues """ import marshal -import os import pickle +from os import PathLike +from pathlib import Path from queuelib import queue @@ -16,10 +17,10 @@ def _with_mkdir(queue_class): class DirectoriesCreated(queue_class): - def __init__(self, path, *args, **kwargs): - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname, exist_ok=True) + def __init__(self, path: str | PathLike[str], *args, **kwargs): + dirname = Path(path).parent + if not dirname.exists(): + dirname.mkdir(parents=True, exist_ok=True) super().__init__(path, *args, **kwargs) return DirectoriesCreated diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 00cc53725..e247f5999 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -4,6 +4,8 @@ import sys import warnings from configparser import ConfigParser from operator import itemgetter +from pathlib import Path +from typing import Optional from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -65,17 +67,17 @@ def arglist_to_dict(arglist): return dict(x.split('=', 1) for x in arglist) -def closest_scrapy_cfg(path='.', prevpath=None): +def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents """ - if path == prevpath: + if prevpath is not None and str(path) == str(prevpath): return '' - path = os.path.abspath(path) - cfgfile = os.path.join(path, 'scrapy.cfg') - if os.path.exists(cfgfile): - return cfgfile - return closest_scrapy_cfg(os.path.dirname(path), path) + path = Path(path).resolve() + cfgfile = path / 'scrapy.cfg' + if cfgfile.exists(): + return str(cfgfile) + return closest_scrapy_cfg(path.parent, path) def init_env(project='default', set_syspath=True): @@ -88,7 +90,7 @@ def init_env(project='default', set_syspath=True): os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project) closest = closest_scrapy_cfg() if closest: - projdir = os.path.dirname(closest) + projdir = str(Path(closest).parent) if set_syspath and projdir not in sys.path: sys.path.append(projdir) @@ -101,13 +103,13 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True): - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config') +def get_sources(use_closest=True) -> list[str]: + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser() sources = [ '/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg', - xdg_config_home + '/scrapy.cfg', - os.path.expanduser('~/.scrapy.cfg'), + str(Path(xdg_config_home) / 'scrapy.cfg'), + str(Path('~/.scrapy.cfg').expanduser()), ] if use_closest: sources.append(closest_scrapy_cfg()) @@ -129,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output, output_format=None, - overwrite_output=None): +def feed_process_params_from_cli(settings, output: list[str], output_format=None, + overwrite_output: Optional[list[str]] = None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary @@ -180,7 +182,7 @@ def feed_process_params_from_cli(settings, output, output_format=None, feed_uri, feed_format = element.rsplit(':', 1) except ValueError: feed_uri = element - feed_format = os.path.splitext(element)[1].replace('.', '') + feed_format = Path(element).suffix.replace('.', '') else: if feed_uri == '-': feed_uri = 'stdout:' diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index c92ef36f5..a65f92e95 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from typing import Optional from scrapy.settings import BaseSettings @@ -6,6 +6,6 @@ from scrapy.settings import BaseSettings def job_dir(settings: BaseSettings) -> Optional[str]: path = settings['JOBDIR'] - if path and not os.path.exists(path): - os.makedirs(path) + if path and not Path(path).exists(): + Path(path).mkdir(parents=True) return path diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c66af497e..e54b71d45 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -2,7 +2,7 @@ import os import warnings from importlib import import_module -from os.path import join, dirname, abspath, isabs, exists +from pathlib import Path from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings @@ -25,36 +25,37 @@ def inside_project(): return bool(closest_scrapy_cfg()) -def project_data_dir(project='default'): +def project_data_dir(project='default') -> str: """Return the current project data dir, creating it if it doesn't exist""" if not inside_project(): raise NotConfigured("Not inside a project") cfg = get_config() if cfg.has_option(DATADIR_CFG_SECTION, project): - d = cfg.get(DATADIR_CFG_SECTION, project) + d = Path(cfg.get(DATADIR_CFG_SECTION, project)) else: scrapy_cfg = closest_scrapy_cfg() if not scrapy_cfg: raise NotConfigured("Unable to find scrapy.cfg file to infer project data dir") - d = abspath(join(dirname(scrapy_cfg), '.scrapy')) - if not exists(d): - os.makedirs(d) - return d + d = (Path(scrapy_cfg).parent / '.scrapy').resolve() + if not d.exists(): + d.mkdir(parents=True) + return str(d) -def data_path(path, createdir=False): +def data_path(path: str, createdir=False) -> str: """ Return the given path joined with the .scrapy data directory. If given an absolute path, return it unmodified. """ - if not isabs(path): + path_obj = Path(path) + if not path_obj.is_absolute(): if inside_project(): - path = join(project_data_dir(), path) + path_obj = Path(project_data_dir(), path) else: - path = join('.scrapy', path) - if createdir and not exists(path): - os.makedirs(path) - return path + path_obj = Path('.scrapy', path) + if createdir and not path_obj.exists(): + path_obj.mkdir(parents=True) + return str(path_obj) def get_project_settings(): diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index cf33317ce..545d489be 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -264,7 +264,7 @@ class RequestFingerprinter: f'and \'VERSION\'.' ) - def fingerprint(self, request): + def fingerprint(self, request: Request): return self._fingerprint(request) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index f068be737..8075902b3 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,23 +1,21 @@ """Helper functions for working with templates""" -import os import re import string +from pathlib import Path -def render_templatefile(path, **kwargs): - with open(path, 'rb') as fp: - raw = fp.read().decode('utf8') +def render_templatefile(path: str, **kwargs): + raw = Path(path).read_text('utf8') content = string.Template(raw).substitute(**kwargs) render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path if path.endswith('.tmpl'): - os.rename(path, render_path) + Path(path).rename(render_path) - with open(render_path, 'wb') as fp: - fp.write(content.encode('utf8')) + Path(render_path).write_text(content, 'utf8') CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 0b828f7c0..4d01f1ef1 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -4,6 +4,7 @@ This module contains some assorted functions used in tests import asyncio import os +from pathlib import Path from posixpath import split from unittest import mock @@ -71,11 +72,11 @@ def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True): return runner.create_crawler(spidercls or Spider) -def get_pythonpath(): +def get_pythonpath() -> str: """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" scrapy_path = import_module('scrapy').__path__[0] - return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') + return str(Path(scrapy_path).parent) + os.pathsep + os.environ.get('PYTHONPATH', '') def get_testenv(): diff --git a/setup.py b/setup.py index a43cf08c8..e413ea6e4 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ -from os.path import dirname, join +from pathlib import Path from pkg_resources import parse_version from setuptools import setup, find_packages, __version__ as setuptools_version -with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: - version = f.read().decode('ascii').strip() +version = (Path(__file__).parent / 'scrapy/VERSION').read_text('ascii').strip() def has_environment_marker_platform_impl_support(): diff --git a/tests/__init__.py b/tests/__init__.py index bb62851dc..be263fa16 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,6 +6,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests import os import socket +from pathlib import Path # ignore system-wide proxies for tests # which would send requests to a totally unsuspecting server @@ -16,14 +17,12 @@ os.environ['ftp_proxy'] = '' # Absolutize paths to coverage config and output file because tests that # spawn subprocesses also changes current working directory. -_sourceroot = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_sourceroot = Path(__file__).resolve().parent.parent if 'COV_CORE_CONFIG' in os.environ: - os.environ['COVERAGE_FILE'] = os.path.join(_sourceroot, '.coverage') - os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot, - os.environ['COV_CORE_CONFIG']) + os.environ['COVERAGE_FILE'] = str(_sourceroot / '.coverage') + os.environ['COV_CORE_CONFIG'] = str(_sourceroot / os.environ['COV_CORE_CONFIG']) -tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'sample_data') +tests_datadir = str(Path(__file__).parent.resolve() / 'sample_data') # In some environments accessing a non-existing host doesn't raise an @@ -35,8 +34,6 @@ except socket.gaierror: NON_EXISTING_RESOLVABLE = False -def get_testdata(*paths): +def get_testdata(*paths: str) -> bytes: """Return test data""" - path = os.path.join(tests_datadir, *paths) - with open(path, 'rb') as f: - return f.read() + return Path(tests_datadir, *paths).read_bytes() diff --git a/tests/keys/__init__.py b/tests/keys/__init__.py index bb4a8e5af..3a41b3a3e 100644 --- a/tests/keys/__init__.py +++ b/tests/keys/__init__.py @@ -1,5 +1,5 @@ -import os from datetime import datetime, timedelta +from pathlib import Path from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa @@ -22,21 +22,20 @@ from cryptography.x509.oid import NameOID # https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate def generate_keys(): - folder = os.path.dirname(__file__) + folder = Path(__file__).parent key = rsa.generate_private_key( public_exponent=65537, key_size=2048, backend=default_backend(), ) - with open(os.path.join(folder, 'localhost.key'), "wb") as f: - f.write( - key.private_bytes( - encoding=Encoding.PEM, - format=PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=NoEncryption(), - ) - ) + (folder / 'localhost.key').write_bytes( + key.private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=NoEncryption(), + ), + ) subject = issuer = Name( [ @@ -59,5 +58,4 @@ def generate_keys(): ) .sign(key, SHA256(), default_backend()) ) - with open(os.path.join(folder, 'localhost.crt'), "wb") as f: - f.write(cert.public_bytes(Encoding.PEM)) + (folder / 'localhost.crt').write_bytes(cert.public_bytes(Encoding.PEM)) diff --git a/tests/mockserver.py b/tests/mockserver.py index 72d7e0241..7916798f7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,6 +1,5 @@ import argparse import json -import os import random import sys from pathlib import Path @@ -253,7 +252,7 @@ class Root(resource.Resource): self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir - self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) + self.putChild(b"files", File(str(Path(tests_datadir, 'test_site/files/')))) except Exception: pass self.putChild(b"redirect-to", RedirectTo()) @@ -346,8 +345,8 @@ class MockFTPServer: def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), keyfile), - os.path.join(os.path.dirname(__file__), certfile), + str(Path(__file__).parent / keyfile), + str(Path(__file__).parent / certfile), ) if cipher_string: ctx = factory.getContext() diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 8233e0101..da73a4c45 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,11 +1,11 @@ import json -import os import pstats import shutil import sys import tempfile import unittest from io import StringIO +from pathlib import Path from subprocess import Popen, PIPE from scrapy.utils.test import get_testenv @@ -36,17 +36,17 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') def test_profiling(self): - path = tempfile.mkdtemp() - filename = os.path.join(path, 'res.prof') + path = Path(tempfile.mkdtemp()) + filename = path / 'res.prof' try: - self._execute('version', '--profile', filename) - self.assertTrue(os.path.exists(filename)) + self._execute('version', '--profile', str(filename)) + self.assertTrue(filename.exists()) out = StringIO() - stats = pstats.Stats(filename, stream=out) + stats = pstats.Stats(str(filename), stream=out) stats.print_stats() out.seek(0) stats = out.read() - self.assertIn(os.path.join('scrapy', 'commands', 'version.py'), + self.assertIn(str(Path('scrapy', 'commands', 'version.py')), stats) self.assertIn('tottime', stats) finally: diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py index d341888d3..fcafcef68 100644 --- a/tests/test_cmdline_crawl_with_pipeline/__init__.py +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -1,6 +1,6 @@ -import os import sys import unittest +from pathlib import Path from subprocess import Popen, PIPE @@ -8,7 +8,7 @@ class CmdlineCrawlPipelineTest(unittest.TestCase): def _execute(self, spname): args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname) - cwd = os.path.dirname(os.path.abspath(__file__)) + cwd = Path(__file__).resolve().parent proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) proc.communicate() return proc.returncode diff --git a/tests/test_command_check.py b/tests/test_command_check.py index c3d705194..4077a9bce 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -1,5 +1,3 @@ -from os.path import join, abspath - from tests.test_commands import CommandTest @@ -10,11 +8,10 @@ class CheckCommandTest(CommandTest): def setUp(self): super(CheckCommandTest, self).setUp() self.spider_name = 'check_spider' - self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve() def _write_contract(self, contracts, parse_def): - with open(self.spider, 'w') as file: - file.write(f""" + self.spider.write_text(f""" import scrapy class CheckSpider(scrapy.Spider): @@ -27,7 +24,7 @@ class CheckSpider(scrapy.Spider): {contracts} \"\"\" {parse_def} - """) + """) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 0d992be56..154287d74 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,6 +1,6 @@ import os import argparse -from os.path import join, abspath, isfile, exists +from pathlib import Path from twisted.internet import defer from scrapy.commands import parse @@ -23,9 +23,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): def setUp(self): super().setUp() self.spider_name = 'parse_spider' - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(f""" + (self.proj_mod_path / 'spiders' / 'myspider.py').write_text(f""" import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule @@ -88,9 +86,7 @@ class MyBadCrawlSpider(CrawlSpider): return [scrapy.Item(), dict(foo='bar')] """) - fname = abspath(join(self.proj_mod_path, 'pipelines.py')) - with open(fname, 'w') as f: - f.write(""" + (self.proj_mod_path / 'pipelines.py').write_text(""" import logging class MyPipeline: @@ -101,8 +97,7 @@ class MyPipeline: return item """) - fname = abspath(join(self.proj_mod_path, 'settings.py')) - with open(fname, 'a') as f: + with (self.proj_mod_path / 'settings.py').open("a") as f: f.write(f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """) @@ -234,7 +229,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} correct format containing correct data in it. """ file_name = 'data.json' - file_path = join(self.proj_path, file_name) + file_path = Path(self.proj_path, file_name) yield self.execute([ '--spider', self.spider_name, '-c', 'parse', @@ -242,12 +237,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.url('/html') ]) - self.assertTrue(exists(file_path)) - self.assertTrue(isfile(file_path)) + self.assertTrue(file_path.exists()) + self.assertTrue(file_path.is_file()) content = '[\n{},\n{"foo": "bar"}\n]' - with open(file_path, 'r') as f: - self.assertEqual(f.read(), content) + self.assertEqual(file_path.read_text(), content) def test_parse_add_options(self): command = parse.Command() diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33189e9be..33c98ad69 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -1,4 +1,4 @@ -from os.path import join +from pathlib import Path from twisted.trial import unittest from twisted.internet import defer @@ -96,8 +96,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_file(self): - filepath = join(tests_datadir, 'test_site', 'index.html') - _, out, _ = yield self.execute([filepath, '-c', 'item']) + filepath = Path(tests_datadir, 'test_site', 'index.html') + _, out, _ = yield self.execute([str(filepath), '-c', 'item']) assert b'{}' in out @defer.inlineCallbacks diff --git a/tests/test_commands.py b/tests/test_commands.py index 76d5f3935..39f718cce 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -9,12 +9,12 @@ import sys import tempfile from contextlib import contextmanager from itertools import chain -from os.path import exists, join, abspath, getmtime from pathlib import Path from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer +from typing import Generator, Optional from unittest import skipIf from pytest import mark @@ -66,8 +66,8 @@ class ProjectTest(unittest.TestCase): def setUp(self): self.temp_path = mkdtemp() self.cwd = self.temp_path - self.proj_path = join(self.temp_path, self.project_name) - self.proj_mod_path = join(self.proj_path, self.project_name) + self.proj_path = Path(self.temp_path, self.project_name) + self.proj_mod_path = self.proj_path / self.project_name self.env = get_testenv() def tearDown(self): @@ -104,10 +104,10 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename, regex): + def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) - with open(filename, "r") as f: + with Path(filename).open("r") as f: for line in f: match = pattern.search(line) if match is not None: @@ -122,13 +122,13 @@ class StartprojectTest(ProjectTest): print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert exists(join(self.proj_path, 'scrapy.cfg')) - assert exists(join(self.proj_path, 'testproject')) - assert exists(join(self.proj_mod_path, '__init__.py')) - assert exists(join(self.proj_mod_path, 'items.py')) - assert exists(join(self.proj_mod_path, 'pipelines.py')) - assert exists(join(self.proj_mod_path, 'settings.py')) - assert exists(join(self.proj_mod_path, 'spiders', '__init__.py')) + assert Path(self.proj_path, 'scrapy.cfg').exists() + assert Path(self.proj_path, 'testproject').exists() + assert Path(self.proj_mod_path, '__init__.py').exists() + assert Path(self.proj_mod_path, 'items.py').exists() + assert Path(self.proj_mod_path, 'pipelines.py').exists() + assert Path(self.proj_mod_path, 'settings.py').exists() + assert Path(self.proj_mod_path, 'spiders', '__init__.py').exists() self.assertEqual(1, self.call('startproject', self.project_name)) self.assertEqual(1, self.call('startproject', 'wrong---project---name')) @@ -138,13 +138,13 @@ class StartprojectTest(ProjectTest): project_dir = mkdtemp() self.assertEqual(0, self.call('startproject', self.project_name, project_dir)) - assert exists(join(abspath(project_dir), 'scrapy.cfg')) - assert exists(join(abspath(project_dir), 'testproject')) - assert exists(join(join(abspath(project_dir), self.project_name), '__init__.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'items.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'pipelines.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'settings.py')) - assert exists(join(join(abspath(project_dir), self.project_name), 'spiders', '__init__.py')) + assert Path(project_dir, 'scrapy.cfg').exists() + assert Path(project_dir, 'testproject').exists() + assert Path(project_dir, self.project_name, '__init__.py').exists() + assert Path(project_dir, self.project_name, 'items.py').exists() + assert Path(project_dir, self.project_name, 'pipelines.py').exists() + assert Path(project_dir, self.project_name, 'settings.py').exists() + assert Path(project_dir, self.project_name, 'spiders', '__init__.py').exists() self.assertEqual(0, self.call('startproject', self.project_name, project_dir + '2')) @@ -158,40 +158,42 @@ class StartprojectTest(ProjectTest): def test_existing_project_dir(self): project_dir = mkdtemp() project_name = self.project_name + '_existing' - project_path = os.path.join(project_dir, project_name) - os.mkdir(project_path) + project_path = Path(project_dir, project_name) + project_path.mkdir() p, out, err = self.proc('startproject', project_name, cwd=project_dir) print(out) print(err, file=sys.stderr) self.assertEqual(p.returncode, 0) - assert exists(join(abspath(project_path), 'scrapy.cfg')) - assert exists(join(abspath(project_path), project_name)) - assert exists(join(join(abspath(project_path), project_name), '__init__.py')) - assert exists(join(join(abspath(project_path), project_name), 'items.py')) - assert exists(join(join(abspath(project_path), project_name), 'pipelines.py')) - assert exists(join(join(abspath(project_path), project_name), 'settings.py')) - assert exists(join(join(abspath(project_path), project_name), 'spiders', '__init__.py')) + assert Path(project_path, 'scrapy.cfg').exists() + assert Path(project_path, project_name).exists() + assert Path(project_path, project_name, '__init__.py').exists() + assert Path(project_path, project_name, 'items.py').exists() + assert Path(project_path, project_name, 'pipelines.py').exists() + assert Path(project_path, project_name, 'settings.py').exists() + assert Path(project_path, project_name, 'spiders', '__init__.py').exists() -def get_permissions_dict(path, renamings=None, ignore=None): +def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]: - def get_permissions(path): - return oct(os.stat(path).st_mode) + def get_permissions(path: Path) -> str: + return oct(path.stat().st_mode) + + path_obj = Path(path) renamings = renamings or tuple() permissions_dict = { - '.': get_permissions(path), + '.': get_permissions(path_obj), } - for root, dirs, files in os.walk(path): + for root, dirs, files in os.walk(path_obj): nodes = list(chain(dirs, files)) if ignore: ignored_names = ignore(root, nodes) nodes = [node for node in nodes if node not in ignored_names] for node in nodes: - absolute_path = os.path.join(root, node) - relative_path = os.path.relpath(absolute_path, path) + absolute_path = Path(root, node) + relative_path = str(absolute_path.relative_to(path)) for search_string, replacement in renamings: relative_path = relative_path.replace( search_string, @@ -208,28 +210,27 @@ class StartprojectTemplatesTest(ProjectTest): def setUp(self): super().setUp() - self.tmpl = join(self.temp_path, 'templates') - self.tmpl_proj = join(self.tmpl, 'project') + self.tmpl = str(Path(self.temp_path, 'templates')) + self.tmpl_proj = str(Path(self.tmpl, 'project')) def test_startproject_template_override(self): - copytree(join(scrapy.__path__[0], 'templates'), self.tmpl) - with open(join(self.tmpl_proj, 'root_template'), 'w'): - pass - assert exists(join(self.tmpl_proj, 'root_template')) + copytree(Path(scrapy.__path__[0], 'templates'), self.tmpl) + Path(self.tmpl_proj, 'root_template').write_bytes(b"") + assert Path(self.tmpl_proj, 'root_template').exists() args = ['--set', f'TEMPLATES_DIR={self.tmpl}'] p, out, err = self.proc('startproject', self.project_name, *args) self.assertIn(f"New Scrapy project '{self.project_name}', " "using template directory", out) self.assertIn(self.tmpl_proj, out) - assert exists(join(self.proj_path, 'root_template')) + assert Path(self.proj_path, 'root_template').exists() def test_startproject_permissions_from_writable(self): """Check that generated files have the right permissions when the template folder has the same permissions as in the project, i.e. everything is writable.""" scrapy_path = scrapy.__path__[0] - project_template = os.path.join(scrapy_path, 'templates', 'project') + project_template = Path(scrapy_path, 'templates', 'project') project_name = 'startproject1' renamings = ( ('module', project_name), @@ -255,7 +256,7 @@ class StartprojectTemplatesTest(ProjectTest): ) process.wait() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) self.assertEqual(actual_permissions, expected_permissions) @@ -268,8 +269,8 @@ class StartprojectTemplatesTest(ProjectTest): See https://github.com/scrapy/scrapy/pull/4604 """ scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates') - project_template = os.path.join(templates_dir, 'project') + templates_dir = Path(scrapy_path, 'templates') + project_template = Path(templates_dir, 'project') project_name = 'startproject2' renamings = ( ('module', project_name), @@ -281,16 +282,16 @@ class StartprojectTemplatesTest(ProjectTest): IGNORE, ) - def _make_read_only(path): - current_permissions = os.stat(path).st_mode - os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION) + def _make_read_only(path: Path): + current_permissions = path.stat().st_mode + path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) read_only_templates_dir = str(Path(mkdtemp()) / 'templates') copytree(templates_dir, read_only_templates_dir) for root, dirs, files in os.walk(read_only_templates_dir): for node in chain(dirs, files): - _make_read_only(os.path.join(root, node)) + _make_read_only(Path(root, node)) destination = mkdtemp() process = subprocess.Popen( @@ -308,7 +309,7 @@ class StartprojectTemplatesTest(ProjectTest): ) process.wait() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) self.assertEqual(actual_permissions, expected_permissions) @@ -317,7 +318,7 @@ class StartprojectTemplatesTest(ProjectTest): """Check that pre-existing folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] - project_template = os.path.join(scrapy_path, 'templates', 'project') + project_template = Path(scrapy_path, 'templates', 'project') project_name = 'startproject3' renamings = ( ('module', project_name), @@ -330,7 +331,7 @@ class StartprojectTemplatesTest(ProjectTest): ) destination = mkdtemp() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) existing_nodes = { oct(permissions)[2:] + extension: permissions @@ -339,10 +340,9 @@ class StartprojectTemplatesTest(ProjectTest): 0o444, 0o555, 0o644, 0o666, 0o755, 0o777, ) } - os.mkdir(project_dir) - project_dir_path = Path(project_dir) + project_dir.mkdir() for node, permissions in existing_nodes.items(): - path = project_dir_path / node + path = project_dir / node if node.endswith('.d'): path.mkdir(mode=permissions) else: @@ -378,7 +378,7 @@ class StartprojectTemplatesTest(ProjectTest): os.umask(cur_mask) scrapy_path = scrapy.__path__[0] - project_template = os.path.join( + project_template = Path( scrapy_path, 'templates', 'project' @@ -409,7 +409,7 @@ class StartprojectTemplatesTest(ProjectTest): ) process.wait() - project_dir = os.path.join(destination, project_name) + project_dir = Path(destination, project_name) actual_permissions = get_permissions_dict(project_dir) self.assertEqual(actual_permissions, expected_permissions) @@ -420,7 +420,7 @@ class CommandTest(ProjectTest): def setUp(self): super().setUp() self.call('startproject', self.project_name) - self.cwd = join(self.temp_path, self.project_name) + self.cwd = Path(self.temp_path, self.project_name) self.env['SCRAPY_SETTINGS_MODULE'] = f'{self.project_name}.settings' @@ -429,10 +429,10 @@ class GenspiderCommandTest(CommandTest): def test_arguments(self): # only pass one argument. spider script shouldn't be created self.assertEqual(2, self.call('genspider', 'test_name')) - assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + assert not Path(self.proj_mod_path, 'spiders', 'test_name.py').exists() # pass two arguments . spider script should be created self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) - assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + assert Path(self.proj_mod_path, 'spiders', 'test_name.py').exists() def test_template(self, tplname='crawl'): args = [f'--template={tplname}'] if tplname else [] @@ -440,11 +440,11 @@ class GenspiderCommandTest(CommandTest): spmodule = f"{self.project_name}.spiders.{spname}" p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out) - self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))) - modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertTrue(Path(self.proj_mod_path, 'spiders', 'test_spider.py').exists()) + modify_time_before = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime p, out, err = self.proc('genspider', spname, 'test.com', *args) self.assertIn(f"Spider {spname!r} already exists in module", out) - modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + modify_time_after = Path(self.proj_mod_path, 'spiders', 'test_spider.py').stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) def test_template_basic(self): @@ -465,37 +465,37 @@ class GenspiderCommandTest(CommandTest): def test_same_name_as_project(self): self.assertEqual(2, self.call('genspider', self.project_name)) - assert not exists(join(self.proj_mod_path, 'spiders', f'{self.project_name}.py')) + assert not Path(self.proj_mod_path, 'spiders', f'{self.project_name}.py').exists() def test_same_filename_as_existing_spider(self, force=False): file_name = 'example' - file_path = join(self.proj_mod_path, 'spiders', f'{file_name}.py') + file_path = Path(self.proj_mod_path, 'spiders', f'{file_name}.py') self.assertEqual(0, self.call('genspider', file_name, 'example.com')) - assert exists(file_path) + assert file_path.exists() # change name of spider but not its file name - with open(file_path, 'r+') as spider_file: + with file_path.open('r+') as spider_file: file_data = spider_file.read() file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") spider_file.seek(0) spider_file.write(file_data) spider_file.truncate() - modify_time_before = getmtime(file_path) + modify_time_before = file_path.stat().st_mtime file_contents_before = file_data if force: p, out, err = self.proc('genspider', '--force', file_name, 'example.com') self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) - modify_time_after = getmtime(file_path) + modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{file_path} already exists", out) - modify_time_after = getmtime(file_path) + modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertEqual(file_contents_after, file_contents_before) def test_same_filename_as_existing_spider_force(self): @@ -504,11 +504,11 @@ class GenspiderCommandTest(CommandTest): def test_url(self, url='test.com', domain="test.com"): self.assertEqual(0, self.call('genspider', '--force', 'test_name', url)) self.assertEqual(domain, - self.find_in_file(join(self.proj_mod_path, + self.find_in_file(Path(self.proj_mod_path, 'spiders', 'test_name.py'), r'allowed_domains\s*=\s*\[\'(.+)\'\]').group(1)) self.assertEqual(f'http://{domain}/', - self.find_in_file(join(self.proj_mod_path, + self.find_in_file(Path(self.proj_mod_path, 'spiders', 'test_name.py'), r'start_urls\s*=\s*\[\'(.+)\'\]').group(1)) @@ -526,31 +526,31 @@ class GenspiderStandaloneCommandTest(ProjectTest): def test_generate_standalone_spider(self): self.call('genspider', 'example', 'example.com') - assert exists(join(self.temp_path, 'example.py')) + assert Path(self.temp_path, 'example.py').exists() def test_same_name_as_existing_file(self, force=False): file_name = 'example' - file_path = join(self.temp_path, file_name + '.py') + file_path = Path(self.temp_path, file_name + '.py') p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) - assert exists(file_path) - modify_time_before = getmtime(file_path) - file_contents_before = open(file_path, 'r').read() + assert file_path.exists() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_path.read_text() if force: # use different template to ensure contents were changed p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com') self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) - modify_time_after = getmtime(file_path) + modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{join(self.temp_path, file_name + '.py')} already exists", out) - modify_time_after = getmtime(file_path) + self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out) + modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = open(file_path, 'r').read() + file_contents_after = file_path.read_text() self.assertEqual(file_contents_after, file_contents_before) def test_same_name_as_existing_file_force(self): @@ -588,17 +588,16 @@ class BadSpider(scrapy.Spider): """ @contextmanager - def _create_file(self, content, name=None): - tmpdir = self.mktemp() - os.mkdir(tmpdir) + def _create_file(self, content, name=None) -> Generator[str, None, None]: + tmpdir = Path(self.mktemp()) + tmpdir.mkdir() if name: - fname = abspath(join(tmpdir, name)) + fname = (tmpdir / name).resolve() else: - fname = abspath(join(tmpdir, self.spider_filename)) - with open(fname, 'w') as f: - f.write(content) + fname = (tmpdir / self.spider_filename).resolve() + fname.write_text(content) try: - yield fname + yield str(fname) finally: rmtree(tmpdir) @@ -747,12 +746,11 @@ class MySpider(scrapy.Spider): ) return [] """ - with open(os.path.join(self.cwd, "example.json"), "w") as f1: - f1.write("not empty") + Path(self.cwd, "example.json").write_text("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with open(os.path.join(self.cwd, "example.json")) as f2: + with Path(self.cwd, "example.json").open() as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -854,9 +852,7 @@ class ViewCommandTest(CommandTest): class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): - fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) - with open(fname, 'w') as f: - f.write(code) + Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code) return self.proc('crawl', 'myspider', *args) def get_log(self, code, args=()): @@ -908,12 +904,11 @@ class MySpider(scrapy.Spider): ) return [] """ - with open(os.path.join(self.cwd, "example.json"), "w") as f1: - f1.write("not empty") + Path(self.cwd, "example.json").write_text("not empty") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with open(os.path.join(self.cwd, "example.json")) as f2: + with Path(self.cwd, "example.json").open() as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cf15ba9b9..19f4229a3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,9 +1,9 @@ import logging -import os import platform import subprocess import sys import warnings +from pathlib import Path from pytest import raises, mark from twisted import version as twisted_version @@ -100,15 +100,14 @@ class CrawlerLoggingTestCase(unittest.TestCase): assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): - log_file = self.mktemp() - with open(log_file, 'wb') as fo: - fo.write('previous message\n'.encode('utf-8')) + log_file = Path(self.mktemp()) + log_file.write_text('previous message\n', encoding='utf-8') class MySpider(scrapy.Spider): name = 'spider' custom_settings = { 'LOG_LEVEL': 'INFO', - 'LOG_FILE': log_file, + 'LOG_FILE': str(log_file), # settings to avoid extra warnings 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, @@ -124,8 +123,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): logging.warning('warning message') logging.error('error message') - with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf-8') + logged = log_file.read_text(encoding='utf-8') self.assertIn('previous message', logged) self.assertNotIn('debug message', logged) @@ -139,14 +137,13 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) def test_spider_custom_settings_log_append(self): - log_file = self.mktemp() - with open(log_file, 'wb') as fo: - fo.write('previous message\n'.encode('utf-8')) + log_file = Path(self.mktemp()) + log_file.write_text('previous message\n', encoding='utf-8') class MySpider(scrapy.Spider): name = 'spider' custom_settings = { - 'LOG_FILE': log_file, + 'LOG_FILE': str(log_file), 'LOG_FILE_APPEND': False, # disable telnet if not available to avoid an extra warning 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE, @@ -156,8 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) logging.debug('debug message') - with open(log_file, 'rb') as fo: - logged = fo.read().decode('utf-8') + logged = log_file.read_text(encoding='utf-8') self.assertNotIn('previous message', logged) self.assertIn('debug message', logged) @@ -296,9 +292,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: - def run_script(self, script_name, *script_args): - script_path = os.path.join(self.script_dir, script_name) - args = [sys.executable, script_path] + list(script_args) + def run_script(self, script_name: str, *script_args): + script_path = self.script_dir / script_name + args = [sys.executable, str(script_path)] + list(script_args) p = subprocess.Popen(args, env=get_testenv(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() @@ -306,7 +302,7 @@ class ScriptRunnerMixin: class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + script_dir = Path(__file__).parent.resolve() / 'CrawlerProcess' def test_simple(self): log = self.run_script('simple.py') @@ -463,7 +459,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') + script_dir = Path(__file__).parent.resolve() / 'CrawlerRunner' def test_response_ip_address(self): log = self.run_script("ip_address.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 5e63ebffb..2558e4f91 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -2,6 +2,7 @@ import os import re from configparser import ConfigParser from importlib import import_module +from pathlib import Path from twisted import version as twisted_version from twisted.trial import unittest @@ -29,11 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase): if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') - tox_config_file_path = os.path.join( - os.path.dirname(__file__), - '..', - 'tox.ini', - ) + tox_config_file_path = Path(__file__) / '..' / 'tox.ini' config_parser = ConfigParser() config_parser.read(tox_config_file_path) pattern = r'Twisted\[http2\]==([\d.]+)' diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 883960084..29ff8c2dc 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -3,6 +3,7 @@ import os import shutil import sys import tempfile +from pathlib import Path from typing import Optional, Type from unittest import mock, SkipTest @@ -10,7 +11,6 @@ from testfixtures import LogCapture from twisted.cred import checkers, credentials, portal from twisted.internet import defer, error, reactor from twisted.protocols.policies import WrappingFactory -from twisted.python.filepath import FilePath from twisted.trial import unittest from twisted.web import resource, server, static, util from twisted.web._newclient import ResponseFailed @@ -108,14 +108,13 @@ class LoadTestCase(unittest.TestCase): class FileTestCase(unittest.TestCase): def setUp(self): - self.tmpname = self.mktemp() - with open(self.tmpname + '^', 'w') as f: - f.write('0123456789') + self.tmpname = Path(self.mktemp() + '^') + Path(self.tmpname).write_text('0123456789') handler = create_instance(FileDownloadHandler, None, get_crawler()) self.download_request = handler.download_request def tearDown(self): - os.unlink(self.tmpname + '^') + self.tmpname.unlink() def test_download(self): def _test(response): @@ -124,7 +123,7 @@ class FileTestCase(unittest.TestCase): self.assertEqual(response.body, b'0123456789') self.assertEqual(response.protocol, None) - request = Request(path_to_file_uri(self.tmpname + '^')) + request = Request(path_to_file_uri(str(self.tmpname))) assert request.url.upper().endswith('%5E') return self.download_request(request, Spider('foo')).addCallback(_test) @@ -223,10 +222,10 @@ class HttpTestCase(unittest.TestCase): certfile = 'keys/localhost.crt' def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) @@ -626,10 +625,10 @@ class Https11CustomCiphers(unittest.TestCase): certfile = 'keys/localhost.crt' def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) self.site = server.Site(r, timeout=None) self.host = 'localhost' self.port = reactor.listenSSL( @@ -1002,16 +1001,15 @@ class BaseFTPTestCase(unittest.TestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file - self.directory = self.mktemp() - os.mkdir(self.directory) - userdir = os.path.join(self.directory, self.username) - os.mkdir(userdir) - fp = FilePath(userdir) + self.directory = Path(self.mktemp()) + self.directory.mkdir() + userdir = self.directory / self.username + userdir.mkdir() for filename, content in self.test_files: - fp.child(filename).setContent(content) + (userdir / filename).write_bytes(content) # setup server - realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) + realm = FTPRealm(anonymousRoot=str(self.directory), userHome=str(self.directory)) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) @@ -1076,28 +1074,28 @@ class BaseFTPTestCase(unittest.TestCase): def test_ftp_local_filename(self): f, local_fname = tempfile.mkstemp() - local_fname = to_bytes(local_fname) + fname_bytes = to_bytes(local_fname) + local_fname = Path(local_fname) os.close(f) - meta = {"ftp_local_filename": local_fname} + meta = {"ftp_local_filename": fname_bytes} meta.update(self.req_meta) request = Request(url=f"ftp://127.0.0.1:{self.portNum}/file.txt", meta=meta) d = self.download_handler.download_request(request, None) def _test(r): - self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {b'Local Filename': [local_fname], + self.assertEqual(r.body, fname_bytes) + self.assertEqual(r.headers, {b'Local Filename': [fname_bytes], b'Size': [b'17']}) - self.assertTrue(os.path.exists(local_fname)) - with open(local_fname, "rb") as f: - self.assertEqual(f.read(), b"I have the power!") - os.remove(local_fname) + self.assertTrue(local_fname.exists()) + self.assertEqual(local_fname.read_bytes(), b"I have the power!") + local_fname.unlink() return self._add_test_callbacks(d, _test) def _test_response_class(self, filename, response_class): f, local_fname = tempfile.mkstemp() - local_fname = to_bytes(local_fname) + local_fname = Path(local_fname) os.close(f) meta = {} meta.update(self.req_meta) @@ -1107,7 +1105,7 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(type(r), response_class) - os.remove(local_fname) + local_fname.unlink() return self._add_test_callbacks(d, _test) def test_response_class_from_url(self): @@ -1147,15 +1145,14 @@ class AnonymousFTPTestCase(BaseFTPTestCase): from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file - self.directory = self.mktemp() - os.mkdir(self.directory) + self.directory = Path(self.mktemp()) + self.directory.mkdir() - fp = FilePath(self.directory) for filename, content in self.test_files: - fp.child(filename).setContent(content) + (self.directory / filename).write_bytes(content) # setup server for anonymous access - realm = FTPRealm(anonymousRoot=self.directory) + realm = FTPRealm(anonymousRoot=str(self.directory)) p = portal.Portal(realm) p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 40e9f3a96..6f4e217e6 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,6 +1,6 @@ from gzip import GzipFile from io import BytesIO -from os.path import join +from pathlib import Path from unittest import TestCase, SkipTest from warnings import catch_warnings @@ -15,7 +15,7 @@ from tests import tests_datadir from w3lib.encoding import resolve_encoding -SAMPLEDIR = join(tests_datadir, 'compressed') +SAMPLEDIR = Path(tests_datadir, 'compressed') FORMAT = { 'gzip': ('html-gzip.bin', 'gzip'), @@ -46,8 +46,7 @@ class HttpCompressionTest(TestCase): samplefile, contentencoding = FORMAT[coding] - with open(join(SAMPLEDIR, samplefile), 'rb') as sample: - body = sample.read() + body = (SAMPLEDIR / samplefile).read_bytes() headers = { 'Server': 'Yaws/1.49 Yet Another Web Server', diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 8a37a8ebe..911d23069 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,8 +2,8 @@ import hashlib import tempfile import unittest import shutil -import os import sys +from pathlib import Path from testfixtures import LogCapture from scrapy.dupefilters import RFPDupeFilter @@ -157,7 +157,7 @@ class RFPDupeFilterTest(unittest.TestCase): df.request_seen(r1) df.close('finished') - with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file: + with Path(path, 'requests.seen').open('rb') as seen_file: line = next(seen_file).decode() assert not line.endswith('\r\r\n') if sys.platform == 'win32': diff --git a/tests/test_engine.py b/tests/test_engine.py index 5677052f6..aa3313659 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -10,11 +10,11 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ -import os import re import subprocess import sys from collections import defaultdict +from pathlib import Path from threading import Timer from urllib.parse import urlparse from dataclasses import dataclass @@ -127,8 +127,8 @@ class ChangeCloseReasonSpider(TestSpider): def start_test_site(debug=False): - root_dir = os.path.join(tests_datadir, "test_site") - r = static.File(root_dir) + root_dir = Path(tests_datadir, "test_site") + r = static.File(str(root_dir)) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) numbers = [str(x).encode("utf8") for x in range(2**18)] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ad2383018..98905d2c0 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -3,7 +3,6 @@ import csv import gzip import json import lzma -import os import random import shutil import string @@ -15,6 +14,7 @@ from collections import defaultdict from contextlib import ExitStack from io import BytesIO from logging import getLogger +from os import PathLike from pathlib import Path from string import ascii_letters, digits from unittest import mock @@ -63,41 +63,41 @@ def printf_escape(string): return string.replace('%', '%%') -def build_url(path): - if path[0] != '/': - path = '/' + path - return urljoin('file:', path) +def build_url(path: str | PathLike[str]) -> str: + path_str = str(path) + if path_str[0] != '/': + path_str = '/' + path_str + return urljoin('file:', path_str) class FileFeedStorageTest(unittest.TestCase): def test_store_file_uri(self): - path = os.path.abspath(self.mktemp()) - uri = path_to_file_uri(path) + path = Path(self.mktemp()).resolve() + uri = path_to_file_uri(str(path)) return self._assert_stores(FileFeedStorage(uri), path) def test_store_file_uri_makedirs(self): - path = os.path.abspath(self.mktemp()) - path = os.path.join(path, 'more', 'paths', 'file.txt') - uri = path_to_file_uri(path) + path = Path(self.mktemp()).resolve() / 'more' / 'paths' / 'file.txt' + uri = path_to_file_uri(str(path)) return self._assert_stores(FileFeedStorage(uri), path) def test_store_direct_path(self): - path = os.path.abspath(self.mktemp()) - return self._assert_stores(FileFeedStorage(path), path) + path = Path(self.mktemp()).resolve() + return self._assert_stores(FileFeedStorage(str(path)), path) def test_store_direct_path_relative(self): - path = self.mktemp() - return self._assert_stores(FileFeedStorage(path), path) + path = Path(self.mktemp()) + return self._assert_stores(FileFeedStorage(str(path)), path) def test_interface(self): path = self.mktemp() st = FileFeedStorage(path) verifyObject(IFeedStorage, st) - def _store(self, feed_options=None): - path = os.path.abspath(self.mktemp()) - storage = FileFeedStorage(path, feed_options=feed_options) + def _store(self, feed_options=None) -> Path: + path = Path(self.mktemp()).resolve() + storage = FileFeedStorage(str(path), feed_options=feed_options) spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") @@ -106,27 +106,26 @@ class FileFeedStorageTest(unittest.TestCase): def test_append(self): path = self._store() - return self._assert_stores(FileFeedStorage(path), path, b"contentcontent") + return self._assert_stores(FileFeedStorage(str(path)), path, b"contentcontent") def test_overwrite(self): path = self._store({"overwrite": True}) return self._assert_stores( - FileFeedStorage(path, feed_options={"overwrite": True}), + FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) @defer.inlineCallbacks - def _assert_stores(self, storage, path, expected_content=b"content"): + def _assert_stores(self, storage, path: Path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) file.write(b"content") yield storage.store(file) - self.assertTrue(os.path.exists(path)) + self.assertTrue(path.exists()) try: - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), expected_content) + self.assertEqual(path.read_bytes(), expected_content) finally: - os.unlink(path) + path.unlink() class FTPFeedStorageTest(unittest.TestCase): @@ -152,13 +151,12 @@ class FTPFeedStorageTest(unittest.TestCase): file.write(content) return storage.store(file) - def _assert_stored(self, path, content): + def _assert_stored(self, path: Path, content): self.assertTrue(path.exists()) try: - with path.open('rb') as fp: - self.assertEqual(fp.read(), content) + self.assertEqual(path.read_bytes(), content) finally: - os.unlink(str(path)) + path.unlink() @defer.inlineCallbacks def test_append(self): @@ -221,24 +219,24 @@ class BlockingFeedStorageTest(unittest.TestCase): b = BlockingFeedStorage() tmp = b.open(self.get_test_spider()) - tmp_path = os.path.dirname(tmp.name) - self.assertEqual(tmp_path, tempfile.gettempdir()) + tmp_path = Path(tmp.name).parent + self.assertEqual(str(tmp_path), tempfile.gettempdir()) def test_temp_file(self): b = BlockingFeedStorage() - tests_path = os.path.dirname(os.path.abspath(__file__)) - spider = self.get_test_spider({'FEED_TEMPDIR': tests_path}) + tests_path = Path(__file__).resolve().parent + spider = self.get_test_spider({'FEED_TEMPDIR': str(tests_path)}) tmp = b.open(spider) - tmp_path = os.path.dirname(tmp.name) + tmp_path = Path(tmp.name).parent self.assertEqual(tmp_path, tests_path) def test_invalid_folder(self): b = BlockingFeedStorage() - tests_path = os.path.dirname(os.path.abspath(__file__)) - invalid_path = os.path.join(tests_path, 'invalid_path') - spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path}) + tests_path = Path(__file__).resolve().parent + invalid_path = tests_path / 'invalid_path' + spider = self.get_test_spider({'FEED_TEMPDIR': str(invalid_path)}) self.assertRaises(OSError, b.open, spider=spider) @@ -564,13 +562,13 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): class DummyBlockingFeedStorage(BlockingFeedStorage): def __init__(self, uri, *args, feed_options=None): - self.path = file_uri_to_path(uri) + self.path = Path(file_uri_to_path(uri)) def _store_in_thread(self, file): - dirname = os.path.dirname(self.path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - with open(self.path, 'ab') as output_file: + dirname = self.path.parent + if dirname and not dirname.exists(): + dirname.mkdir(parents=True) + with self.path.open('ab') as output_file: output_file.write(file.read()) file.close() @@ -613,10 +611,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): foo = scrapy.Field() hello = scrapy.Field() - def _random_temp_filename(self, inter_dir=''): + def _random_temp_filename(self, inter_dir='') -> Path: chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) - return os.path.join(self.temp_dir, inter_dir, filename) + return Path(self.temp_dir, inter_dir, filename) def setUp(self): self.temp_dir = tempfile.mkdtemp() @@ -702,18 +700,17 @@ class FeedExportTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - with open(str(file_path), 'rb') as f: - content[feed_options['format']] = f.read() + content[feed_options['format']] = Path(file_path).read_bytes() finally: for file_path in FEEDS.keys(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - os.remove(str(file_path)) + Path(file_path).unlink() return content @@ -808,7 +805,7 @@ class FeedExportTest(FeedExportTestBase): def test_stats_file_success(self): settings = { "FEEDS": { - printf_escape(path_to_url(self._random_temp_filename())): { + printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", } }, @@ -823,7 +820,7 @@ class FeedExportTest(FeedExportTestBase): def test_stats_file_failed(self): settings = { "FEEDS": { - printf_escape(path_to_url(self._random_temp_filename())): { + printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", } }, @@ -846,7 +843,7 @@ class FeedExportTest(FeedExportTestBase): 'AWS_ACCESS_KEY_ID': 'access_key', 'AWS_SECRET_ACCESS_KEY': 'secret_key', "FEEDS": { - printf_escape(path_to_url(self._random_temp_filename())): { + printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", }, "s3://bucket/key/foo.csv": { @@ -1427,12 +1424,11 @@ class FeedExportTest(FeedExportTestBase): self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) @defer.inlineCallbacks - def test_pathlib_uri(self): - feed_path = Path(self._random_temp_filename()) + def test_str_uri(self): settings = { 'FEED_STORE_EMPTY': True, 'FEEDS': { - feed_path: {'format': 'csv'} + str(self._random_temp_filename()): {'format': 'csv'} }, } data = yield self.exported_no_data(settings) @@ -1538,8 +1534,8 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): def close(self): self.file.close() - def _named_tempfile(self, name): - return os.path.join(self.temp_dir, name) + def _named_tempfile(self, name) -> str: + return str(Path(self.temp_dir, name)) @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1559,18 +1555,17 @@ class FeedPostProcessedExportsTest(FeedExportTestBase): yield crawler.crawl() for file_path, feed_options in FEEDS.items(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - with open(str(file_path), 'rb') as f: - content[str(file_path)] = f.read() + content[str(file_path)] = Path(file_path).read_bytes() finally: for file_path in FEEDS.keys(): - if not os.path.exists(str(file_path)): + if not Path(file_path).exists(): continue - os.remove(str(file_path)) + Path(file_path).unlink() return content @@ -2031,11 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase): yield crawler.crawl() for path, feed in FEEDS.items(): - dir_name = os.path.dirname(path) - for file in sorted(os.listdir(dir_name)): - with open(os.path.join(dir_name, file), 'rb') as f: - data = f.read() - content[feed['format']].append(data) + dir_name = Path(path).parent + for file in sorted(dir_name.iterdir()): + content[feed['format']].append(file.read_bytes()) finally: self.tearDown() defer.returnValue(content) @@ -2045,7 +2038,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, + self._random_temp_filename() / 'jl' / self._file_mark: {'format': 'jl'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2061,7 +2054,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, + self._random_temp_filename() / 'csv' / self._file_mark: {'format': 'csv'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2077,7 +2070,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2094,8 +2087,8 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, - os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, + self._random_temp_filename() / 'xml' / self._file_mark: {'format': 'xml'}, + self._random_temp_filename() / 'json' / self._file_mark: {'format': 'json'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2120,7 +2113,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, + self._random_temp_filename() / 'pickle' / self._file_mark: {'format': 'pickle'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2137,7 +2130,7 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = settings or {} settings.update({ 'FEEDS': { - os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, + self._random_temp_filename() / 'marshal' / self._file_mark: {'format': 'marshal'}, }, }) batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT') @@ -2184,7 +2177,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt in ('json', 'jsonlines', 'xml', 'csv'): settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + self._random_temp_filename() / fmt / self._file_mark: {'format': fmt}, }, 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } @@ -2204,7 +2197,7 @@ class BatchDeliveriesTest(FeedExportTestBase): for fmt, expctd in formats: settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + self._random_temp_filename() / fmt / self._file_mark: {'format': fmt}, }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, @@ -2237,19 +2230,19 @@ class BatchDeliveriesTest(FeedExportTestBase): settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + self._random_temp_filename() / 'json' / self._file_mark: { 'format': 'json', 'indent': 0, 'fields': ['bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { + self._random_temp_filename() / 'xml' / self._file_mark: { 'format': 'xml', 'indent': 2, 'fields': ['foo'], 'encoding': 'latin-1', }, - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + self._random_temp_filename() / 'csv' / self._file_mark: { 'format': 'csv', 'indent': None, 'fields': ['foo', 'bar'], @@ -2272,7 +2265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): } settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + self._random_temp_filename() / 'json' / self._file_mark: { 'format': 'json', 'indent': None, 'encoding': 'utf-8', @@ -2299,7 +2292,7 @@ class BatchDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(batch_time)s'): { + self._random_temp_filename() / '%(batch_time)s': { 'format': 'json', }, }, @@ -2312,7 +2305,7 @@ class BatchDeliveriesTest(FeedExportTestBase): def test_stats_batch_file_success(self): settings = { "FEEDS": { - build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): { + build_url(str(self._random_temp_filename() / "json" / self._file_mark)): { "format": "json", } }, diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 49c83132f..402348cf9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -1,10 +1,10 @@ import json -import os import random import re import shutil import string from ipaddress import IPv4Address +from pathlib import Path from unittest import mock, skipIf from urllib.parse import urlencode @@ -163,9 +163,8 @@ class RequestHeaders(LeafResource): return bytes(json.dumps(headers), 'utf-8') -def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: - with open(key_file, 'r') as key, open(certificate_file, 'r') as certificate: - pem = ''.join(key.readlines()) + ''.join(certificate.readlines()) +def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: + pem = key_file.read_text() + certificate_file.read_text() return PrivateCertificate.loadPEM(pem) @@ -173,12 +172,12 @@ def get_client_certificate(key_file, certificate_file) -> PrivateCertificate: @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2ClientProtocolTestCase(TestCase): scheme = 'https' - key_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.key') - certificate_file = os.path.join(os.path.dirname(__file__), 'keys', 'localhost.crt') + key_file = Path(__file__).parent / 'keys' / 'localhost.key' + certificate_file = Path(__file__).parent / 'keys' / 'localhost.crt' def _init_resource(self): self.temp_directory = self.mktemp() - os.mkdir(self.temp_directory) + Path(self.temp_directory).mkdir() r = File(self.temp_directory) r.putChild(b'get-data-html-small', GetDataHtmlSmall()) r.putChild(b'get-data-html-large', GetDataHtmlLarge()) @@ -202,7 +201,7 @@ class Https2ClientProtocolTestCase(TestCase): # Start server for testing self.hostname = 'localhost' - context_factory = ssl_context_factory(self.key_file, self.certificate_file) + context_factory = ssl_context_factory(str(self.key_file), str(self.certificate_file)) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index e46532a1c..5d6547279 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,5 +1,5 @@ -import os import shutil +from pathlib import Path from testfixtures import LogCapture from twisted.internet import defer @@ -61,12 +61,12 @@ class FileDownloadCrawlTestCase(TestCase): self.mockserver.__enter__() # prepare a directory for storing files - self.tmpmediastore = self.mktemp() - os.mkdir(self.tmpmediastore) + self.tmpmediastore = Path(self.mktemp()) + self.tmpmediastore.mkdir() self.settings = { 'REQUEST_FINGERPRINTER_IMPLEMENTATION': 'VERSION', 'ITEM_PIPELINES': {self.pipeline_class: 1}, - self.store_setting_key: self.tmpmediastore, + self.store_setting_key: str(self.tmpmediastore), } self.runner = CrawlerRunner(self.settings) self.items = [] @@ -111,9 +111,7 @@ class FileDownloadCrawlTestCase(TestCase): # check that the image files where actually written to the media store for item in items: for i in item[self.media_key]: - self.assertTrue( - os.path.exists( - os.path.join(self.tmpmediastore, i['path']))) + self.assertTrue((self.tmpmediastore / i['path']).exists()) def _assert_files_download_failure(self, crawler, items, code, logs): @@ -133,7 +131,7 @@ class FileDownloadCrawlTestCase(TestCase): self.assertEqual(logs.count(file_dl_failure), 3) # check that no files were written to the media store - self.assertEqual(os.listdir(self.tmpmediastore), []) + self.assertEqual([x for x in self.tmpmediastore.iterdir()], []) @defer.inlineCallbacks def test_download_media(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index d641e7a43..4acd29bf7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -3,6 +3,7 @@ import random import time from datetime import datetime from io import BytesIO +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp from unittest import mock @@ -89,7 +90,7 @@ class FilesPipelineTestCase(unittest.TestCase): self.assertEqual(self.pipeline.store.basedir, self.tempdir) path = 'some/image/key.jpg' - fullpath = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') + fullpath = Path(self.tempdir, 'some', 'image', 'key.jpg') self.assertEqual(self.pipeline.store._get_filesystem_path(path), fullpath) @defer.inlineCallbacks diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index afdfb2578..ea7701b5d 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import re import sys +from pathlib import Path from subprocess import Popen, PIPE from urllib.parse import urlsplit, urlunsplit from testfixtures import LogCapture @@ -27,14 +28,13 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') + cert_path = Path(__file__).parent.resolve() / 'keys' / 'mitmproxy-ca.pem' self.proc = Popen([sys.executable, '-c', script, '--listen-host', '127.0.0.1', '--listen-port', '0', '--proxyauth', f'{self.auth_user}:{self.auth_pass}', - '--certs', cert_path, + '--certs', str(cert_path), '--ssl-insecure', ], stdout=PIPE, env=get_testenv()) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 3719c7c9f..b9025cc14 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,7 +1,7 @@ import sys -import os import shutil import warnings +from pathlib import Path from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -17,10 +17,10 @@ from scrapy.settings import Settings from scrapy.http import Request from scrapy.crawler import CrawlerRunner -module_dir = os.path.dirname(os.path.abspath(__file__)) +module_dir = Path(__file__).resolve().parent -def _copytree(source, target): +def _copytree(source: Path, target: Path): try: shutil.copytree(source, target) except shutil.Error: @@ -30,18 +30,18 @@ def _copytree(source, target): class SpiderLoaderTest(unittest.TestCase): def setUp(self): - orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = tempfile.mkdtemp() - self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + orig_spiders_dir = module_dir / 'test_spiders' + self.tmpdir = Path(tempfile.mkdtemp()) + self.spiders_dir = self.tmpdir / 'test_spiders_xxx' _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(self.tmpdir) + sys.path.append(str(self.tmpdir)) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.spider_loader = SpiderLoader.from_settings(settings) def tearDown(self): del self.spider_loader del sys.modules['test_spiders_xxx'] - sys.path.remove(self.tmpdir) + sys.path.remove(str(self.tmpdir)) def test_interface(self): verifyObject(ISpiderLoader, self.spider_loader) @@ -135,22 +135,22 @@ class SpiderLoaderTest(unittest.TestCase): class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): - orig_spiders_dir = os.path.join(module_dir, 'test_spiders') - self.tmpdir = self.mktemp() - os.mkdir(self.tmpdir) - self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + orig_spiders_dir = module_dir / 'test_spiders' + self.tmpdir = Path(self.mktemp()) + self.tmpdir.mkdir() + self.spiders_dir = self.tmpdir / 'test_spiders_xxx' _copytree(orig_spiders_dir, self.spiders_dir) - sys.path.append(self.tmpdir) + sys.path.append(str(self.tmpdir)) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) def tearDown(self): del sys.modules['test_spiders_xxx'] - sys.path.remove(self.tmpdir) + sys.path.remove(str(self.tmpdir)) def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) + shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider3.py', + self.tmpdir / 'test_spiders_xxx' / 'spider3dupe.py') with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -171,10 +171,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) + shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider1.py', + self.tmpdir / 'test_spiders_xxx' / 'spider1dupe.py') + shutil.copyfile(self.tmpdir / 'test_spiders_xxx' / 'spider2.py', + self.tmpdir / 'test_spiders_xxx' / 'spider2dupe.py') with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index 383fadfeb..ab215576e 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,5 @@ -import os from datetime import datetime +from pathlib import Path import shutil from twisted.trial import unittest @@ -13,7 +13,7 @@ class SpiderStateTest(unittest.TestCase): def test_store_load(self): jobdir = self.mktemp() - os.mkdir(jobdir) + Path(jobdir).mkdir() try: spider = Spider(name='default') dt = datetime.now() diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 4943731cb..ca98bff21 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -1,5 +1,5 @@ import unittest -from os.path import join +from pathlib import Path from w3lib.encoding import html_to_unicode @@ -8,46 +8,40 @@ from scrapy.http import Response from tests import tests_datadir -SAMPLEDIR = join(tests_datadir, 'compressed') +SAMPLEDIR = Path(tests_datadir, 'compressed') class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): - with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: - r1 = Response("http://www.example.com", body=f.read()) - self.assertTrue(gzip_magic_number(r1)) + r1 = Response("http://www.example.com", body=(SAMPLEDIR / 'feed-sample1.xml.gz').read_bytes()) + self.assertTrue(gzip_magic_number(r1)) - r2 = Response("http://www.example.com", body=gunzip(r1.body)) - self.assertFalse(gzip_magic_number(r2)) - self.assertEqual(len(r2.body), 9950) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + self.assertFalse(gzip_magic_number(r2)) + self.assertEqual(len(r2.body), 9950) def test_gunzip_truncated(self): - with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: - text = gunzip(f.read()) - assert text.endswith(b'') - self.assertFalse(gzip_magic_number(r2)) + r2 = Response("http://www.example.com", body=gunzip(r1.body)) + assert r2.body.endswith(b'') + self.assertFalse(gzip_magic_number(r2)) def test_is_gzipped_empty(self): r1 = Response("http://www.example.com") self.assertFalse(gzip_magic_number(r1)) def test_gunzip_illegal_eof(self): - with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f: - text = html_to_unicode('charset=cp1252', gunzip(f.read()))[1] - with open(join(SAMPLEDIR, 'unexpected-eof-output.txt'), 'rb') as o: - expected_text = o.read().decode("utf-8") - self.assertEqual(len(text), len(expected_text)) - self.assertEqual(text, expected_text) + text = html_to_unicode('charset=cp1252', gunzip((SAMPLEDIR / 'unexpected-eof.gz').read_bytes()))[1] + expected_text = (SAMPLEDIR / 'unexpected-eof-output.txt').read_text(encoding="utf-8") + self.assertEqual(len(text), len(expected_text)) + self.assertEqual(text, expected_text) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f84cb2956..ba3136b96 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,5 +1,3 @@ -import os - from pytest import mark from twisted.trial import unittest @@ -303,11 +301,6 @@ class LxmlXmliterTestCase(XmliterTestCase): class UtilsCsvTestCase(unittest.TestCase): - sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') - sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') - sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv') - sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv') - def test_csviter_defaults(self): body = get_testdata('feeds', 'feed-sample3.csv') response = TextResponse(url="http://example.com/", body=body) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index b83c1d6f0..dc5b9e123 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,6 +1,7 @@ import sys import os import unittest +from pathlib import Path from unittest import mock from scrapy.item import Item, Field @@ -55,7 +56,7 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(ImportError, walk_modules, 'nomodule999') def test_walk_modules_egg(self): - egg = os.path.join(os.path.dirname(__file__), 'test.egg') + egg = str(Path(__file__).parent / 'test.egg') sys.path.append(egg) try: mods = walk_modules('testegg') diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 46452415a..f35f039a9 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -4,6 +4,7 @@ import tempfile import shutil import contextlib import warnings +from pathlib import Path from pytest import warns @@ -18,9 +19,7 @@ def inside_a_project(): try: os.chdir(project_dir) - with open('scrapy.cfg', 'w') as f: - # create an empty scrapy.cfg - f.close() + Path('scrapy.cfg').touch() yield project_dir finally: @@ -31,20 +30,20 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): self.assertEqual( - os.path.join('.scrapy', 'somepath'), + str(Path('.scrapy', 'somepath')), data_path('somepath') ) - abspath = os.path.join(os.path.sep, 'absolute', 'path') + abspath = str(Path(os.path.sep, 'absolute', 'path')) self.assertEqual(abspath, data_path(abspath)) def test_data_path_inside_project(self): with inside_a_project() as proj_path: - expected = os.path.join(proj_path, '.scrapy', 'somepath') + expected = Path(proj_path, '.scrapy', 'somepath') self.assertEqual( - os.path.realpath(expected), - os.path.realpath(data_path('somepath')) + expected.resolve(), + Path(data_path('somepath')).resolve() ) - abspath = os.path.join(os.path.sep, 'absolute', 'path') + abspath = str(Path(os.path.sep, 'absolute', 'path').resolve()) self.assertEqual(abspath, data_path(abspath)) diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index d20852e62..cdf972933 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,6 +1,6 @@ -import os import unittest import warnings +from pathlib import Path from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning @@ -39,10 +39,9 @@ class ResponseUtilsTest(unittest.TestCase): def browser_open(burl): path = urlparse(burl).path - if not os.path.exists(path): + if not path or not Path(path).exists(): path = burl.replace('file://', '') - with open(path, "rb") as f: - bbody = f.read() + bbody = Path(path).read_bytes() self.assertIn(b'', bbody) return True response = HtmlResponse(url, body=body) @@ -98,10 +97,9 @@ class ResponseUtilsTest(unittest.TestCase): def check_base_url(burl): path = urlparse(burl).path - if not os.path.exists(path): + if not path or not Path(path).exists(): path = burl.replace('file://', '') - with open(path, "rb") as f: - bbody = f.read() + bbody = Path(path).read_bytes() self.assertEqual(bbody.count(b''), 1) return True diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 1d5e63363..b1aca5ed3 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp import unittest @@ -22,21 +22,19 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): template = 'from ${project_name}.spiders.${name} import ${classname}' rendered = 'from proj.spiders.spi import TheSpider' - template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') - render_path = os.path.join(self.tmp_path, 'templ.py') + template_path = Path(self.tmp_path, 'templ.py.tmpl') + render_path = Path(self.tmp_path, 'templ.py') - with open(template_path, 'wb') as tmpl_file: - tmpl_file.write(template.encode('utf8')) - assert os.path.isfile(template_path) # Failure of test itself + template_path.write_text(template, encoding='utf8') + assert template_path.is_file() # Failure of test itself - render_templatefile(template_path, **context) + render_templatefile(str(template_path), **context) - self.assertFalse(os.path.exists(template_path)) - with open(render_path, 'rb') as result: - self.assertEqual(result.read().decode('utf8'), rendered) + self.assertFalse(template_path.exists()) + self.assertEqual(render_path.read_text(encoding='utf8'), rendered) - os.remove(render_path) - assert not os.path.exists(render_path) # Failure of test itself + render_path.unlink() + assert not render_path.exists() # Failure of test itself if '__main__' == __name__: diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0d5827339..69d9a9e3a 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -2,8 +2,8 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ -import os import shutil +from pathlib import Path import OpenSSL.SSL from twisted.trial import unittest @@ -15,7 +15,6 @@ except ImportError: # deprecated in Twisted 19.7.0 # (remove once we bump our requirement past that version) from twisted.test.proto_helpers import StringTransport -from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks @@ -230,10 +229,10 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"error", ErrorResource()) @@ -379,10 +378,10 @@ class WebClientSSLTestCase(unittest.TestCase): return f"https://127.0.0.1:{self.portno}/{path}" def setUp(self): - self.tmpname = self.mktemp() - os.mkdir(self.tmpname) - FilePath(self.tmpname).child("file").setContent(b"0123456789") - r = static.File(self.tmpname) + self.tmpname = Path(self.mktemp()) + self.tmpname.mkdir() + (self.tmpname / "file").write_bytes(b"0123456789") + r = static.File(str(self.tmpname)) r.putChild(b"payload", PayloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) From 7daf735f45ab9f3d8760d7a59e5f72c277d0687d Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Tue, 18 Oct 2022 20:07:29 -0400 Subject: [PATCH 171/237] downgrade type hint syntax --- scrapy/commands/runspider.py | 3 ++- scrapy/squeues.py | 3 ++- scrapy/utils/conf.py | 10 +++++----- tests/test_commands.py | 6 +++--- tests/test_feedexport.py | 3 ++- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index c41135508..a8db1cd86 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,13 +3,14 @@ from os import PathLike from pathlib import Path from importlib import import_module from types import ModuleType +from typing import Union from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommand -def _import_file(filepath: str | PathLike[str]) -> ModuleType: +def _import_file(filepath: Union[str, PathLike]) -> ModuleType: abspath = Path(filepath).resolve() dirname = str(abspath.parent) if abspath.suffix not in ('.py', '.pyw'): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 1f2dee55f..2fa84fc00 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -6,6 +6,7 @@ import marshal import pickle from os import PathLike from pathlib import Path +from typing import Union from queuelib import queue @@ -17,7 +18,7 @@ def _with_mkdir(queue_class): class DirectoriesCreated(queue_class): - def __init__(self, path: str | PathLike[str], *args, **kwargs): + def __init__(self, path: Union[str, PathLike], *args, **kwargs): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e247f5999..0dfa714e8 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,7 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import Optional +from typing import List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -67,7 +67,7 @@ def arglist_to_dict(arglist): return dict(x.split('=', 1) for x in arglist) -def closest_scrapy_cfg(path: str | os.PathLike[str] = '.', prevpath: Optional[str | os.PathLike] = None) -> str: +def closest_scrapy_cfg(path: Union[str, os.PathLike] = '.', prevpath: Optional[Union[str, os.PathLike]] = None) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents """ @@ -103,7 +103,7 @@ def get_config(use_closest=True): return cfg -def get_sources(use_closest=True) -> list[str]: +def get_sources(use_closest=True) -> List[str]: xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or Path('~/.config').expanduser() sources = [ '/etc/scrapy.cfg', @@ -131,8 +131,8 @@ def feed_complete_default_values_from_settings(feed, settings): return out -def feed_process_params_from_cli(settings, output: list[str], output_format=None, - overwrite_output: Optional[list[str]] = None): +def feed_process_params_from_cli(settings, output: List[str], output_format=None, + overwrite_output: Optional[List[str]] = None): """ Receives feed export params (from the 'crawl' or 'runspider' commands), checks for inconsistencies in their quantities and returns a dictionary diff --git a/tests/test_commands.py b/tests/test_commands.py index 39f718cce..71ae85094 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -14,7 +14,7 @@ from shutil import rmtree, copytree from stat import S_IWRITE as ANYONE_WRITE_PERMISSION from tempfile import mkdtemp from threading import Timer -from typing import Generator, Optional +from typing import Dict, Generator, Optional, Union from unittest import skipIf from pytest import mark @@ -104,7 +104,7 @@ class ProjectTest(unittest.TestCase): return p, to_unicode(stdout), to_unicode(stderr) - def find_in_file(self, filename: str | os.PathLike[str], regex) -> Optional[re.Match]: + def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) with Path(filename).open("r") as f: @@ -175,7 +175,7 @@ class StartprojectTest(ProjectTest): assert Path(project_path, project_name, 'spiders', '__init__.py').exists() -def get_permissions_dict(path: str | os.PathLike[str], renamings=None, ignore=None) -> dict[str, str]: +def get_permissions_dict(path: Union[str, os.PathLike], renamings=None, ignore=None) -> Dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 98905d2c0..97c3a74b3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -17,6 +17,7 @@ from logging import getLogger from os import PathLike from pathlib import Path from string import ascii_letters, digits +from typing import Union from unittest import mock from urllib.parse import urljoin, quote from urllib.request import pathname2url @@ -63,7 +64,7 @@ def printf_escape(string): return string.replace('%', '%%') -def build_url(path: str | PathLike[str]) -> str: +def build_url(path: Union[str, PathLike]) -> str: path_str = str(path) if path_str[0] != '/': path_str = '/' + path_str From fb26e6b650005e63c7017b55ff7b9755938a6c11 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Tue, 18 Oct 2022 20:21:11 -0400 Subject: [PATCH 172/237] resolve path to search in error --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 71ae85094..c7284b6c3 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -547,7 +547,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{Path(self.temp_path, file_name + '.py')} already exists", out) + self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text() From 93d82648e59a1a211c114d0b8be2d76e0d79b582 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Wed, 19 Oct 2022 08:08:21 -0400 Subject: [PATCH 173/237] resolve this path too --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index c7284b6c3..b61f314ec 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -492,7 +492,7 @@ class GenspiderCommandTest(CommandTest): self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') - self.assertIn(f"{file_path} already exists", out) + self.assertIn(f"{file_path.resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) file_contents_after = file_path.read_text() From 065db7b56688ce5cbdc1508cfda9603d33b17a87 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Wed, 19 Oct 2022 08:28:26 -0400 Subject: [PATCH 174/237] fix some mypy issues --- scrapy/commands/__init__.py | 4 ++-- scrapy/commands/genspider.py | 5 +++-- scrapy/spiders/__init__.py | 2 +- scrapy/utils/conf.py | 4 ++-- tests/test_commands.py | 1 + tests/test_crawler.py | 2 ++ 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 8570d90bd..2c205a712 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -15,7 +15,7 @@ from scrapy.exceptions import UsageError class ScrapyCommand: requires_project = False - crawler_process = None + crawler_process: Any = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} @@ -23,7 +23,7 @@ class ScrapyCommand: exitcode = 0 def __init__(self): - self.settings = None # set in scrapy.cmdline + self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): if hasattr(self, '_crawler'): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 01b4a0dbd..facb593da 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ import string from pathlib import Path from importlib import import_module -from typing import Optional +from typing import Optional, cast from urllib.parse import urlparse import scrapy @@ -116,6 +116,7 @@ class Command(ScrapyCommand): return template_file print(f"Unable to find template: {template}\n") print('Use "scrapy genspider --list" to see all available templates.') + return None def _list_templates(self): print("Available templates:") @@ -144,7 +145,7 @@ class Command(ScrapyCommand): # a file with the same name exists in the target directory spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) - spiders_dir = Path(spiders_module.__file__).parent + spiders_dir = Path(cast(str, spiders_module.__file__)).parent spiders_dir_abs = spiders_dir.resolve() path = spiders_dir_abs / (name + ".py") if path.exists(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9a97e7801..0a9b124b9 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -22,7 +22,7 @@ class Spider(object_ref): class. """ - name: Optional[str] = None + name: str custom_settings: Optional[dict] = None def __init__(self, name=None, **kwargs): diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0dfa714e8..82defa033 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -5,7 +5,7 @@ import warnings from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union from scrapy.exceptions import ScrapyDeprecationWarning, UsageError @@ -176,7 +176,7 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None 'URIs are specified' ) - result = {} + result: Dict[str, Dict[str, Any]] = {} for element in output: try: feed_uri, feed_format = element.rsplit(':', 1) diff --git a/tests/test_commands.py b/tests/test_commands.py index b61f314ec..f8081f450 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -112,6 +112,7 @@ class ProjectTest(unittest.TestCase): match = pattern.search(line) if match is not None: return match + return None class StartprojectTest(ProjectTest): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 19f4229a3..a0703ad47 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -292,6 +292,8 @@ class CrawlerRunnerHasSpider(unittest.TestCase): class ScriptRunnerMixin: + script_dir: Path + def run_script(self, script_name: str, *script_args): script_path = self.script_dir / script_name args = [sys.executable, str(script_path)] + list(script_args) From c85de908193a534c60140492d5f793ab32ef3b48 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 19:44:35 -0400 Subject: [PATCH 175/237] fix path to tox.ini --- tests/test_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 2558e4f91..74fdd966b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -30,7 +30,7 @@ class ScrapyUtilsTest(unittest.TestCase): if not os.environ.get('_SCRAPY_PINNED', None): self.skipTest('Not in a pinned environment') - tox_config_file_path = Path(__file__) / '..' / 'tox.ini' + tox_config_file_path = Path(__file__).parent / '..' / 'tox.ini' config_parser = ConfigParser() config_parser.read(tox_config_file_path) pattern = r'Twisted\[http2\]==([\d.]+)' From 24d6ac1f529e1e2ea5b04cff30cf9015311ef420 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 20:03:38 -0400 Subject: [PATCH 176/237] downgrade the last 3.9 type hints --- scrapy/pipelines/files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ffb12d910..4e0211d1b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -14,7 +14,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import Optional +from typing import DefaultDict, Optional, Set from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -46,7 +46,7 @@ class FSFilesStore: basedir = basedir.split('://', 1)[1] self.basedir = basedir self._mkdir(Path(self.basedir)) - self.created_directories: defaultdict[str, set[str]] = defaultdict(set) + self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set) def persist_file(self, path: str, buf, info, meta=None, headers=None): absolute_path = self._get_filesystem_path(path) From 12a26755ae601c9b3244a4a6b55d0e880fc21864 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 20:14:43 -0400 Subject: [PATCH 177/237] improve render_templatefile --- scrapy/commands/startproject.py | 2 +- scrapy/utils/template.py | 15 +++++++++------ tests/test_utils_template.py | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 4323cdb53..9e1e95ab1 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -109,7 +109,7 @@ class Command(ScrapyCommand): move(project_dir / 'module', project_dir / project_name) for paths in TEMPLATES_TO_RENDER: tplfile = Path(project_dir, *(string.Template(s).substitute(project_name=project_name) for s in paths)) - render_templatefile(str(tplfile), project_name=project_name, ProjectName=string_camelcase(project_name)) + render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) print(f"New Scrapy project '{project_name}', using template directory " f"'{self.templates_dir}', created in:") print(f" {project_dir.resolve()}\n") diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 8075902b3..2177817d9 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -1,21 +1,24 @@ """Helper functions for working with templates""" +from os import PathLike import re import string from pathlib import Path +from typing import Union -def render_templatefile(path: str, **kwargs): - raw = Path(path).read_text('utf8') +def render_templatefile(path: Union[str, PathLike], **kwargs): + path_obj = Path(path) + raw = path_obj.read_text('utf8') content = string.Template(raw).substitute(**kwargs) - render_path = path[:-len('.tmpl')] if path.endswith('.tmpl') else path + render_path = path_obj.with_suffix('') if path_obj.suffix == '.tmpl' else path_obj - if path.endswith('.tmpl'): - Path(path).rename(render_path) + if path_obj.suffix == '.tmpl': + path_obj.rename(render_path) - Path(render_path).write_text(content, 'utf8') + render_path.write_text(content, 'utf8') CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index b1aca5ed3..4f1e8772b 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -28,7 +28,7 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): template_path.write_text(template, encoding='utf8') assert template_path.is_file() # Failure of test itself - render_templatefile(str(template_path), **context) + render_templatefile(template_path, **context) self.assertFalse(template_path.exists()) self.assertEqual(render_path.read_text(encoding='utf8'), rendered) From 607eece72a91beb0e9a598a987e51a56bcffb434 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Thu, 20 Oct 2022 20:18:06 -0400 Subject: [PATCH 178/237] move to after check --- scrapy/commands/runspider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a8db1cd86..ed16c3fb6 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -12,9 +12,9 @@ from scrapy.commands import BaseRunSpiderCommand def _import_file(filepath: Union[str, PathLike]) -> ModuleType: abspath = Path(filepath).resolve() - dirname = str(abspath.parent) if abspath.suffix not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") + dirname = str(abspath.parent) if dirname: sys.path = [dirname] + sys.path try: From b33244e2f0d877b8911f949308222db0b076d665 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Fri, 21 Oct 2022 19:17:04 +0500 Subject: [PATCH 179/237] Fix the flake8 per-file ignore syntax (#5688) --- .flake8 | 17 +++++++++-------- scrapy/utils/url.py | 1 + tests/test_loader.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.flake8 b/.flake8 index d7aebc24b..0c64d009e 100644 --- a/.flake8 +++ b/.flake8 @@ -6,16 +6,17 @@ ignore = W503 exclude = docs/conf.py +per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused - scrapy/__init__.py E402 - scrapy/core/downloader/handlers/http.py F401 - scrapy/http/__init__.py F401 - scrapy/linkextractors/__init__.py E402 F401 - scrapy/selector/__init__.py F401 - scrapy/spiders/__init__.py E402 F401 + scrapy/__init__.py:E402 + scrapy/core/downloader/handlers/http.py:F401 + scrapy/http/__init__.py:F401 + scrapy/linkextractors/__init__.py:E402,F401 + scrapy/selector/__init__.py:F401 + scrapy/spiders/__init__.py:E402,F401 # Issues pending a review: - scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 + scrapy/utils/url.py:F403,F405 + tests/test_loader.py:E741 diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d5e9ae82..21201ace5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -34,6 +34,7 @@ def url_has_any_extension(url, extensions): lowercase_path = parse_url(url).path.lower() return any(lowercase_path.endswith(ext) for ext in extensions) + def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already parsed url) diff --git a/tests/test_loader.py b/tests/test_loader.py index c0937b349..b3e44d36b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -295,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), ['Marta']) - + def test_init_method_with_base_response(self): """Selector should be None after initialization""" response = Response("https://scrapy.org") From 69d1b8fc0832f1cf00727c73eff6fbde11824c20 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Fri, 21 Oct 2022 20:21:08 -0400 Subject: [PATCH 180/237] dirname cannot be falsey --- scrapy/commands/runspider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index ed16c3fb6..22fa6a53d 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -15,13 +15,11 @@ def _import_file(filepath: Union[str, PathLike]) -> ModuleType: if abspath.suffix not in ('.py', '.pyw'): raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) - if dirname: - sys.path = [dirname] + sys.path + sys.path = [dirname] + sys.path try: module = import_module(abspath.stem) finally: - if dirname: - sys.path.pop(0) + sys.path.pop(0) return module From 28396c34977535ca59ed5d8481b50dd1e8171e49 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Fri, 21 Oct 2022 20:23:07 -0400 Subject: [PATCH 181/237] convert _get_request_path back to str --- scrapy/extensions/httpcache.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 3057bf157..58c4e644e 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -293,7 +293,7 @@ class FilesystemCacheStorage: metadata = self._read_meta(spider, request) if metadata is None: return # not cached - rpath = self._get_request_path(spider, request) + rpath = Path(self._get_request_path(spider, request)) with self._open(rpath / 'response_body', 'rb') as f: body = f.read() with self._open(rpath / 'response_headers', 'rb') as f: @@ -307,7 +307,7 @@ class FilesystemCacheStorage: def store_response(self, spider: Spider, request: Request, response): """Store the given response in the cache.""" - rpath = self._get_request_path(spider, request) + rpath = Path(self._get_request_path(spider, request)) if not rpath.exists(): rpath.mkdir(parents=True) metadata = { @@ -330,12 +330,12 @@ class FilesystemCacheStorage: with self._open(rpath / 'request_body', 'wb') as f: f.write(request.body) - def _get_request_path(self, spider: Spider, request: Request) -> Path: + def _get_request_path(self, spider: Spider, request: Request) -> str: key = self._fingerprinter.fingerprint(request).hex() - return Path(self.cachedir, spider.name, key[0:2], key) + return str(Path(self.cachedir, spider.name, key[0:2], key)) def _read_meta(self, spider: Spider, request: Request): - rpath = self._get_request_path(spider, request) + rpath = Path(self._get_request_path(spider, request)) metapath = rpath / 'pickled_meta' if not metapath.exists(): return # not found From 726680c7125ab3a6622b12e25d45dbfedc5a39b3 Mon Sep 17 00:00:00 2001 From: Matthew Donoughe Date: Fri, 21 Oct 2022 20:32:13 -0400 Subject: [PATCH 182/237] change crawler_process to optional --- scrapy/commands/__init__.py | 5 +++-- scrapy/commands/genspider.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2c205a712..1f088041c 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -4,9 +4,10 @@ Base class for Scrapy commands import os import argparse from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, Optional from twisted.python import failure +from scrapy.crawler import CrawlerProcess from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError @@ -15,7 +16,7 @@ from scrapy.exceptions import UsageError class ScrapyCommand: requires_project = False - crawler_process: Any = None + crawler_process: Optional[CrawlerProcess] = None # default settings to be used for this command instead of global defaults default_settings: Dict[str, Any] = {} diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index facb593da..ed9660252 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -133,6 +133,10 @@ class Command(ScrapyCommand): return True return False + assert ( + self.crawler_process is not None + ), "crawler_process must be set before calling run" + try: spidercls = self.crawler_process.spider_loader.load(name) except KeyError: From b61b71c6f015d45f6e98a4280bf4993517180045 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Tue, 25 Oct 2022 08:44:43 +0530 Subject: [PATCH 183/237] Replace indentation of source before parsing with ast. closes #5323 --- scrapy/utils/misc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 1221b39b2..c0258c8d9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,7 +226,14 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - code = re.sub(r"^[\t ]+", "", inspect.getsource(callable)) + pattern = r"(^[\t ]+)" + src = inspect.getsource(callable) + match = re.match(pattern, src) # Find indentation + code = re.sub(pattern, "", src) + if match: + # Remove indentation + code = re.sub(f"\n{match.group(0)}", "\n", code) + tree = ast.parse(code) for node in walk_callable(tree): if isinstance(node, ast.Return) and not returns_none(node): From f4e2a10ed6a44300738bd3701ea62b432c9c1a06 Mon Sep 17 00:00:00 2001 From: Kaushal Sharma Date: Tue, 25 Oct 2022 02:45:46 -0700 Subject: [PATCH 184/237] =?UTF-8?q?Image.ANTIALIAS=20=E2=86=92=20Image.Res?= =?UTF-8?q?ampling.LANCZOS=20(#5692)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/pipelines/images.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6b97190ee..67b3224b3 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -160,7 +160,14 @@ class ImagesPipeline(FilesPipeline): if size: image = image.copy() - image.thumbnail(size, self._Image.ANTIALIAS) + try: + # Image.Resampling.LANCZOS was added in Pillow 9.1.0 + # remove this try except block, + # when updating the minimum requirements for Pillow. + resampling_filter = self._Image.Resampling.LANCZOS + except AttributeError: + resampling_filter = self._Image.ANTIALIAS + image.thumbnail(size, resampling_filter) buf = BytesIO() image.save(buf, 'JPEG') From 830e1c5dd85618a27749bbe41d35b11fb2bdd348 Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 01:26:54 +0530 Subject: [PATCH 185/237] Add test for parsing decorated methods --- ...t_return_with_argument_inside_generator.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 1c85ca353..72277d701 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -165,6 +165,89 @@ https://example.org warn_on_generator_with_return_value(None, l2) self.assertEqual(len(w), 0) + def test_generators_return_none_with_decorator(self): + def decorator(func): + def inner_func(): + func() + return inner_func + + @decorator + def f3(): + yield 1 + return None + + @decorator + def g3(): + yield 1 + return + + @decorator + def h3(): + yield 1 + + @decorator + def i3(): + yield 1 + yield from generator_that_returns_stuff() + + @decorator + def j3(): + yield 1 + + def helper(): + return 0 + + yield helper() + + @decorator + def k3(): + """ +docstring + """ + url = """ +https://example.org + """ + yield url + return + + @decorator + def l3(): + return + + assert not is_generator_with_return_value(top_level_return_none) + assert not is_generator_with_return_value(f3) + assert not is_generator_with_return_value(g3) + assert not is_generator_with_return_value(h3) + assert not is_generator_with_return_value(i3) + assert not is_generator_with_return_value(j3) # not recursive + assert not is_generator_with_return_value(k3) # not recursive + assert not is_generator_with_return_value(l3) + + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, top_level_return_none) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, f3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, g3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, h3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, i3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, j3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, k3) + self.assertEqual(len(w), 0) + with warnings.catch_warnings(record=True) as w: + warn_on_generator_with_return_value(None, l3) + self.assertEqual(len(w), 0) + @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error) def test_indentation_error(self): with warnings.catch_warnings(record=True) as w: From b0ddffc47b9cee5e6146497b42de3787da76d2ad Mon Sep 17 00:00:00 2001 From: Godson-Gnanaraj Date: Wed, 26 Oct 2022 06:53:43 +0530 Subject: [PATCH 186/237] Misc. changes: - compile regex - readability improvements --- scrapy/utils/misc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index c0258c8d9..4d4fb9600 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -226,13 +226,13 @@ def is_generator_with_return_value(callable): return value is None or isinstance(value, ast.NameConstant) and value.value is None if inspect.isgeneratorfunction(callable): - pattern = r"(^[\t ]+)" src = inspect.getsource(callable) - match = re.match(pattern, src) # Find indentation - code = re.sub(pattern, "", src) + pattern = re.compile(r"(^[\t ]+)") + code = pattern.sub("", src) + + match = pattern.match(src) # finds indentation if match: - # Remove indentation - code = re.sub(f"\n{match.group(0)}", "\n", code) + code = re.sub(f"\n{match.group(0)}", "\n", code) # remove indentation tree = ast.parse(code) for node in walk_callable(tree): From 2464939b7ee8381f904f7b0625aee6de0989b6c8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 26 Oct 2022 15:58:20 -0300 Subject: [PATCH 187/237] Fixed deprecation warning in scrapy.core.engine (#5589) * Change `download` function logic * Fix CI error in 3.7 checks * Make `spider` parameter optional in `_download` function, assign spider value from self if `None` --- scrapy/core/engine.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6602f661d..1228e78da 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -257,9 +257,7 @@ class ExecutionEngine: def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" - if spider is None: - spider = self.spider - else: + if spider is not None: warnings.warn( "Passing a 'spider' argument to ExecutionEngine.download is deprecated", category=ScrapyDeprecationWarning, @@ -267,7 +265,7 @@ class ExecutionEngine: ) if spider is not self.spider: logger.warning("The spider '%s' does not match the open spider", spider.name) - if spider is None: + if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") return self._download(request, spider).addBoth(self._downloaded, request, spider) @@ -278,11 +276,14 @@ class ExecutionEngine: self.slot.remove_request(request) return self.download(result, spider) if isinstance(result, Request) else result - def _download(self, request: Request, spider: Spider) -> Deferred: + def _download(self, request: Request, spider: Optional[Spider]) -> Deferred: assert self.slot is not None # typing self.slot.add_request(request) + if spider is None: + spider = self.spider + def _on_success(result: Union[Response, Request]) -> Union[Response, Request]: if not isinstance(result, (Response, Request)): raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}") From b394f2165acd662e55762d524b916f146291e422 Mon Sep 17 00:00:00 2001 From: Andrei Andrukhovich Date: Wed, 26 Oct 2022 23:11:28 +0300 Subject: [PATCH 188/237] =?UTF-8?q?Fix=20typo:=20[they]=20depends=20?= =?UTF-8?q?=E2=86=92=20depend=20(#5694)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 9ab479edd..2c2079f68 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -52,7 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among * `twisted`_, an asynchronous networking framework * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs -Some of these packages themselves depends on non-Python packages +Some of these packages themselves depend on non-Python packages that might require additional installation steps depending on your platform. Please check :ref:`platform-specific guides below `. From a214147359b8840abbb67bbe2a4a4066b271d2a5 Mon Sep 17 00:00:00 2001 From: Johanan Idicula Date: Wed, 26 Oct 2022 21:43:35 -0400 Subject: [PATCH 189/237] ci: Update macos runner The GitHub Actions macos-10.15 runner image is now deprecated, and GitHub Actions has begun to temporarily fail jobs referencing it during brownout periods. The image will be fully unsupported by 2022-12-01, which is just about a month away. This change updates the macOS runner image to the latest generally-available version, to help reduce spurious CI failures during the brownout periods, and to stay abreast of the sunsetting of the macos-10.15 image. See also: actions/runner-images#5583 --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index fdb9f4980..61f1857f8 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: tests: - runs-on: macos-10.15 + runs-on: macos-11 strategy: fail-fast: false matrix: From ca50af645390e38299082d9ef4682c4be482ae70 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 17:12:26 +0600 Subject: [PATCH 190/237] Remove an unused import. --- tests/test_pipeline_images.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index c4ce2736f..c189d08bf 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -4,7 +4,6 @@ import io import random from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf from warnings import catch_warnings import attr From b6541830849a6edd41da6b539d272790a4661d7d Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 27 Oct 2022 17:00:36 +0500 Subject: [PATCH 191/237] Use Python 3.11 as the default in CI (#5696) --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/publish.yml | 2 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 9 +++------ .github/workflows/tests-windows.yml | 7 +++++++ .readthedocs.yml | 2 +- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 439dfee51..8c1ae4bd3 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -8,10 +8,10 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: security - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: flake8 # Pylint requires installing reppy, which does not support Python 3.9 @@ -22,10 +22,10 @@ jobs: - python-version: 3.7 env: TOXENV: typing - - python-version: "3.10" # Keep in sync with .readthedocs.yml + - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs - - python-version: "3.10" + - python-version: "3.11" env: TOXENV: twinecheck diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f6b098b80..991b0b6e8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Check Tag id: check-release-tag diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 61f1857f8..174d245ca 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index d2bfe4a5f..9c3ce8115 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,13 +17,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.10" - env: - TOXENV: asyncio - - python-version: "3.11.0-rc.2" + - python-version: "3.11" env: TOXENV: py - - python-version: "3.11.0-rc.2" + - python-version: "3.11" env: TOXENV: asyncio - python-version: pypy3.9 @@ -57,7 +54,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2' + if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 14683fd53..f60c48841 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -23,6 +23,13 @@ jobs: - python-version: "3.10" env: TOXENV: asyncio +# no binary package for lxml for 3.11 yet +# - python-version: "3.11" +# env: +# TOXENV: py +# - python-version: "3.11" +# env: +# TOXENV: asyncio steps: - uses: actions/checkout@v3 diff --git a/.readthedocs.yml b/.readthedocs.yml index 390be3749..e71d34f3a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.10" # Keep in sync with .github/workflows/checks.yml + python: "3.11" # Keep in sync with .github/workflows/checks.yml python: install: From 3a34fa839938d7640e56f3ce12ba60aed31a6ecf Mon Sep 17 00:00:00 2001 From: Godson <30664729+Godson-Gnanaraj@users.noreply.github.com> Date: Thu, 27 Oct 2022 17:32:12 +0530 Subject: [PATCH 192/237] Get the event loop from event_loop_policy to avoid a deprecation warning (#5689) --- scrapy/utils/defer.py | 8 +++++--- scrapy/utils/reactor.py | 21 ++++++++++++++++----- tests/test_utils_asyncio.py | 6 ++++-- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8fcf31cab..38aefd6d0 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -26,7 +26,7 @@ from twisted.python import failure from twisted.python.failure import Failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.reactor import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed, get_asyncio_event_loop_policy def defer_fail(_failure: Failure) -> Deferred: @@ -269,7 +269,8 @@ def deferred_from_coro(o) -> Any: return ensureDeferred(o) else: # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - return Deferred.fromFuture(asyncio.ensure_future(o)) + event_loop = get_asyncio_event_loop_policy().get_event_loop() + return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -320,7 +321,8 @@ def deferred_to_future(d: Deferred) -> Future: d = treq.get('https://example.com/additional') additional_response = await deferred_to_future(d) """ - return d.asFuture(asyncio.get_event_loop()) + policy = get_asyncio_event_loop_policy() + return d.asFuture(policy.get_event_loop()) def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 652733ce8..ddf354d88 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -51,6 +51,19 @@ class CallLaterOnce: return self._func(*self._a, **self._kw) +def get_asyncio_event_loop_policy(): + policy = asyncio.get_event_loop_policy() + if ( + sys.version_info >= (3, 8) + and sys.platform == "win32" + and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy) + ): + policy = asyncio.WindowsSelectorEventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + return policy + + def install_reactor(reactor_path, event_loop_path=None): """Installs the :mod:`~twisted.internet.reactor` with the specified import path. Also installs the asyncio event loop with the specified import @@ -58,16 +71,14 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - if sys.version_info >= (3, 8) and sys.platform == "win32": - policy = asyncio.get_event_loop_policy() - if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + policy = get_asyncio_event_loop_policy() if event_loop_path is not None: event_loop_class = load_object(event_loop_path) event_loop = event_loop_class() asyncio.set_event_loop(event_loop) else: - event_loop = asyncio.get_event_loop() + event_loop = policy.get_event_loop() + asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4d..741c6a505 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,3 +1,4 @@ +import warnings from unittest import TestCase from pytest import mark @@ -13,5 +14,6 @@ class AsyncioTest(TestCase): self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') def test_install_asyncio_reactor(self): - # this should do nothing - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + with warnings.catch_warnings(record=True) as w: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + self.assertEqual(len(w), 0) From b71d0292d5ff85d652e2943ee1a727a128b55594 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Oct 2022 18:13:47 +0600 Subject: [PATCH 193/237] Add a test for processing partial callbacks. --- .../test_return_with_argument_inside_generator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 72277d701..562f72fee 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -1,5 +1,6 @@ import unittest import warnings +from functools import partial from unittest import mock from scrapy.utils.misc import is_generator_with_return_value, warn_on_generator_with_return_value @@ -254,3 +255,10 @@ https://example.org warn_on_generator_with_return_value(None, top_level_return_none) self.assertEqual(len(w), 1) self.assertIn('Unable to determine', str(w[0].message)) + + def test_partial(self): + def cb(arg1, arg2): + yield {} + + partial_cb = partial(cb, arg1=42) + assert not is_generator_with_return_value(partial_cb) From fd692f309105d917f5f46bd00a88c550d6cc7da3 Mon Sep 17 00:00:00 2001 From: Magnus Offermanns Date: Thu, 27 Oct 2022 14:43:31 +0200 Subject: [PATCH 194/237] Prevent running the -O and -t command-line options together (#5605) Co-authored-by: Andrey Rakhmatullin --- docs/topics/commands.rst | 20 ++++++++++++++++++++ scrapy/commands/__init__.py | 6 ++++-- scrapy/utils/conf.py | 19 ++++++++++++++++--- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8c0b8e55f..362190116 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -271,11 +271,31 @@ crawl Start crawling using a spider. +Supported options: + +* ``-h, --help``: show a help message and exit + +* ``-a NAME=VALUE``: set a spider argument (may be repeated) + +* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``) + +* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``) + +* ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O`` + Usage examples:: $ scrapy crawl myspider [ ... myspider starts crawling ... ] + $ scrapy -o myfile:csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] + + $ scrapy -O myfile:json myspider + [ ... myspider starts crawling and saves the result in myfile in json format overwriting the original content... ] + + $ scrapy -o myfile -t csv myspider + [ ... myspider starts crawling and appends the result to the file myfile in csv format ... ] .. command:: check diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index fb304b8c0..49c4e2f42 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand): parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_argument("-o", "--output", metavar="FILE", action="append", - help="append scraped items to the end of FILE (use - for stdout)") + help="append scraped items to the end of FILE (use - for stdout)," + " to define format set a colon at the end of the output URI (i.e. -o FILE:FORMAT)") parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append", - help="dump scraped items into FILE, overwriting any existing file") + help="dump scraped items into FILE, overwriting any existing file," + " to define format set a colon at the end of the output URI (i.e. -O FILE:FORMAT)") parser.add_argument("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items") diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 00cc53725..6404edda6 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -155,6 +155,15 @@ def feed_process_params_from_cli(settings, output, output_format=None, raise UsageError( "Please use only one of -o/--output and -O/--overwrite-output" ) + if output_format: + raise UsageError( + "-t/--output-format is a deprecated command line option" + " and does not work in combination with -O/--overwrite-output." + " To specify a format please specify it after a colon at the end of the" + " output URI (i.e. -O :)." + " Example working in the tutorial: " + "scrapy crawl quotes -O quotes.json:json" + ) output = overwrite_output overwrite = True @@ -162,9 +171,13 @@ def feed_process_params_from_cli(settings, output, output_format=None, if len(output) == 1: check_valid_format(output_format) message = ( - 'The -t command line option is deprecated in favor of ' - 'specifying the output format within the output URI. See the ' - 'documentation of the -o and -O options for more information.' + "The -t/--output-format command line option is deprecated in favor of " + "specifying the output format within the output URI using the -o/--output or the" + " -O/--overwrite-output option (i.e. -o/-O :). See the documentation" + " of the -o or -O option or the following examples for more information. " + "Examples working in the tutorial: " + "scrapy crawl quotes -o quotes.csv:csv or " + "scrapy crawl quotes -O quotes.json:json" ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} From bd9e482c2f0db92065708c8291be6e8bc1f05218 Mon Sep 17 00:00:00 2001 From: iamkaushal Date: Thu, 27 Oct 2022 23:21:55 +0530 Subject: [PATCH 195/237] added typing.io and typing.re in pytest warning filter to ignore --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index af0f2fb6e..f5fbf2529 100644 --- a/pytest.ini +++ b/pytest.ini @@ -24,3 +24,5 @@ markers = filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated + ignore:typing.re is deprecated + ignore:typing.io is deprecated From 9f45be439de8a3b9a6d201c33e98b408a73c02bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=B4=8E=E0=B4=A4=E0=B4=BF=E0=B4=B0=E0=B4=BE=E0=B4=B3?= =?UTF-8?q?=E0=B4=BF=E0=B4=95=E0=B5=8D=E0=B4=95=E0=B5=8A=E0=B4=B0=E0=B5=81?= =?UTF-8?q?=20=E0=B4=AA=E0=B5=8B=E0=B4=B0=E0=B4=BE=E0=B4=B3=E0=B4=BF?= <108031802+pankali@users.noreply.github.com> Date: Fri, 28 Oct 2022 02:13:37 +0200 Subject: [PATCH 196/237] Update Code of Conduct to Contributor Covenant v2.1 --- CODE_OF_CONDUCT.md | 152 +++++++++++++++++++++++++++++++-------------- 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 902cd523e..3c8e4d1b5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,77 +1,133 @@ + # Contributor Covenant Code of Conduct ## Our Pledge -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to creating a positive environment -include: +Examples of behavior that contributes to a positive environment for our +community include: -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community -Examples of unacceptable behavior by participants include: +Examples of unacceptable behavior include: -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting -## Our Responsibilities +## Enforcement Responsibilities -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. ## Scope -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@zyte.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. +reported to the community leaders responsible for enforcement at +opensource@zyte.com. +All complaints will be reviewed and investigated promptly and fairly. -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version]. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations From 3259a4252566f4130f14ce2acd03531a735dae90 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 30 Oct 2022 13:17:37 -0300 Subject: [PATCH 197/237] CrawlSpider: pass cb_kwargs from process_request --- scrapy/spiders/crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index edac082d0..2d9328633 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -102,9 +102,9 @@ class CrawlSpider(Spider): request = self._build_request(rule_index, link) yield rule.process_request(request, response) - def _callback(self, response): + def _callback(self, response, **cb_kwargs): rule = self._rules[response.meta['rule']] - return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow) + return self._parse_response(response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow) def _errback(self, failure): rule = self._rules[failure.request.meta['rule']] From b18560315bda057610ecda165694f9c0f7445c1f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 30 Oct 2022 18:28:16 -0300 Subject: [PATCH 198/237] Add tests --- tests/spiders.py | 11 +++++++++++ tests/test_crawl.py | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/tests/spiders.py b/tests/spiders.py index 5ea8a4a21..7952e3d47 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -419,6 +419,17 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): self.logger.info('[errback] status %i', failure.value.response.status) +class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod): + name = 'crawl_spider_with_process_request_cb_kwargs' + rules = ( + Rule(LinkExtractor(), callback='parse', follow=True, process_request="process_request"), + ) + + def process_request(self, request, response): + request.cb_kwargs["foo"] = "process_request" + return request + + class BytesReceivedCallbackSpider(MetaSpider): full_response_length = 2**18 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5383ec652..5ec96e4a7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -41,6 +41,7 @@ from tests.spiders import ( CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, + CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, DuplicateStartRequestsSpider, FollowAllSpider, @@ -426,6 +427,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[errback] status 500", str(log)) self.assertIn("[errback] status 501", str(log)) + @defer.inlineCallbacks + def test_crawlspider_process_request_cb_kwargs(self): + crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse] status 200 (foo: process_request)", str(log)) + self.assertIn("[parse] status 201 (foo: process_request)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) From 940a73863bf7dcb16b3f2d9f5efb83efe4599712 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Nov 2022 19:00:33 +0600 Subject: [PATCH 199/237] Release notes for 2.7.1. --- docs/news.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 1ec183a1d..b7c8c85b5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,57 @@ Release notes ============= +.. _release-2.7.1: + +Scrapy 2.7.1 (Not relased yet) +------------------------------ + +New features +~~~~~~~~~~~~ + +- Relaxed the restriction introduced in 2.6.2 so that the + ``Proxy-Authentication`` header can again be set explicitly, as long as the + proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and + for as long as that proxy URL remains the same (:issue:`5626`). + +Bug fixes +~~~~~~~~~ + +- Using ``-O``/``--overwrite-output`` and ``-t``/``--output-format`` options + together now produces an error instead of ignoring the former option + (:issue:`5516`, :issue:`5605`). + +- Replaced deprecated :mod:`asyncio` APIs that implicitly use the current + event loop with code that explicitly requests a loop from the event loop + policy (:issue:`5685`, :issue:`5689`). + +- Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`, + :issue:`5589`). + +- Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). + +- Improved code that checks if generators return values, so that it no longer + fails on decorated methods and partial methods (:issue:`5323`, + :issue:`5592`, :issue:`5599`, :issue:`5691`). + +Documentation +~~~~~~~~~~~~~ + +- Upgraded the Code of Conduct to Contributor Covenant v2.1 (:issue:`5698`). + +- Fixed typos (:issue:`5681`, :issue:`5694`). + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Re-enabled some erroneously disabled flake8 checks (:issue:`5688`). + +- Ignored harmless deprecation warnings from :mod:`typing` in tests + (:issue:`5686`, :issue:`5697`). + +- Modernized CI configuration (:issue:`5695`, :issue:`5696`). + + .. _release-2.7.0: Scrapy 2.7.0 (2022-10-17) From 5ec175b8bb08f93c431d7d64d2389b90ec7a1f37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Nov 2022 13:54:00 +0600 Subject: [PATCH 200/237] Small relnotes fixes. --- docs/news.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b7c8c85b5..257626526 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,7 +14,8 @@ New features - Relaxed the restriction introduced in 2.6.2 so that the ``Proxy-Authentication`` header can again be set explicitly, as long as the proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and - for as long as that proxy URL remains the same (:issue:`5626`). + for as long as that proxy URL remains the same; this restores compatibility + with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). Bug fixes ~~~~~~~~~ @@ -27,7 +28,7 @@ Bug fixes event loop with code that explicitly requests a loop from the event loop policy (:issue:`5685`, :issue:`5689`). -- Fixed uses of deprecated Scrapy APIs in the Scrapy itself (:issue:`5588`, +- Fixed uses of deprecated Scrapy APIs in Scrapy itself (:issue:`5588`, :issue:`5589`). - Fixed uses of a deprecated Pillow API (:issue:`5684`, :issue:`5692`). @@ -51,7 +52,7 @@ Quality assurance - Ignored harmless deprecation warnings from :mod:`typing` in tests (:issue:`5686`, :issue:`5697`). -- Modernized CI configuration (:issue:`5695`, :issue:`5696`). +- Modernized our CI configuration (:issue:`5695`, :issue:`5696`). .. _release-2.7.0: From 6ded3cf4cd134b615239babe28bb28c3ff524b05 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Nov 2022 17:00:47 +0600 Subject: [PATCH 201/237] =?UTF-8?q?Bump=20version:=202.7.0=20=E2=86=92=202?= =?UTF-8?q?.7.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/news.rst | 4 ++-- scrapy/VERSION | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f88071685..b949d81c4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.7.0 +current_version = 2.7.1 commit = True tag = True tag_name = {new_version} diff --git a/docs/news.rst b/docs/news.rst index 257626526..e5fc2971a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,8 +5,8 @@ Release notes .. _release-2.7.1: -Scrapy 2.7.1 (Not relased yet) ------------------------------- +Scrapy 2.7.1 (2022-11-02) +------------------------- New features ~~~~~~~~~~~~ diff --git a/scrapy/VERSION b/scrapy/VERSION index 24ba9a38d..860487ca1 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.7.0 +2.7.1 From 6c0890ff54a8d49237415e5b7d7dfbf216e88577 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 7 Nov 2022 16:36:54 +0500 Subject: [PATCH 202/237] Simplify the changes after the merge --- tests/test_pipeline_images.py | 37 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 81c9a027a..0c9a5733f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -7,7 +7,6 @@ from shutil import rmtree from tempfile import mkdtemp from unittest import skipIf from unittest.mock import patch -from warnings import catch_warnings import attr from itemadapter import ItemAdapter @@ -91,6 +90,22 @@ class ImagesPipelineTestCase(unittest.TestCase): info=object()), 'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg') + def test_thumbnail_name_from_item(self): + """ + Custom thumbnail name based on item data, overriding default implementation + """ + + class CustomImagesPipeline(ImagesPipeline): + def thumb_path(self, request, thumb_id, response=None, info=None, item=None): + return f"thumb/{thumb_id}/{item.get('path')}" + + thumb_path = CustomImagesPipeline.from_settings(Settings( + {'IMAGES_STORE': self.tempdir} + )).thumb_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') + def test_get_images_exception(self): self.pipeline.min_width = 100 self.pipeline.min_height = 100 @@ -231,22 +246,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(converted.mode, 'RGB') self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) - def test_thumbnail_name_from_item(self): - """ - Custom thumbnail name based on item data, overriding default implementation - """ - - class CustomImagesPipeline(ImagesPipeline): - def thumb_path(self, request, thumb_id, response=None, info=None, item=None): - return f"thumb/{thumb_id}/{item.get('path')}" - - thumb_path = CustomImagesPipeline.from_settings(Settings( - {'IMAGES_STORE': self.tempdir} - )).thumb_path - item = dict(path='path-to-store-file') - request = Request("http://example.com") - self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file') - class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): @@ -536,11 +535,11 @@ class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): arg = str() - with catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as warnings: NoimagesDrop(arg) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as warnings: class SubclassedNoimagesDrop(NoimagesDrop): pass SubclassedNoimagesDrop(arg) From bbe24d79a5ee6a2afc8cd50bff4ac0e6df26886c Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 7 Nov 2022 17:08:54 +0500 Subject: [PATCH 203/237] Fix test issues --- tests/test_pipeline_images.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0c9a5733f..f98d40fda 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -5,7 +5,6 @@ import random import warnings from shutil import rmtree from tempfile import mkdtemp -from unittest import skipIf from unittest.mock import patch import attr @@ -535,16 +534,16 @@ class NoimagesDropTestCase(unittest.TestCase): def test_deprecation_warning(self): arg = str() - with warnings.catch_warnings(record=True) as warnings: + with warnings.catch_warnings(record=True) as w: NoimagesDrop(arg) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with warnings.catch_warnings(record=True) as warnings: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, ScrapyDeprecationWarning) + with warnings.catch_warnings(record=True) as w: class SubclassedNoimagesDrop(NoimagesDrop): pass SubclassedNoimagesDrop(arg) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, ScrapyDeprecationWarning) def _create_image(format, *a, **kw): From ae3fd0172972f672c3cf3291bcf9f28073e8d4d0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 10 Nov 2022 11:38:46 -0300 Subject: [PATCH 204/237] =?UTF-8?q?Fix=20stray=20=E2=80=9Ccommands?= =?UTF-8?q?=E2=80=9D=20(#5712)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/cmdline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 5ee1f0f44..68267fb74 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -7,7 +7,7 @@ import pkg_resources import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter +from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings @@ -32,7 +32,7 @@ def _iter_command_classes(module_name): inspect.isclass(obj) and issubclass(obj, ScrapyCommand) and obj.__module__ == module.__name__ - and not obj == ScrapyCommand + and obj not in (ScrapyCommand, BaseRunSpiderCommand) ): yield obj From 29bf7f5a6c8460e030e465351d2e6d38acf22f3d Mon Sep 17 00:00:00 2001 From: Hassan Shoayb <79839316+Hassan-Shoayb@users.noreply.github.com> Date: Mon, 14 Nov 2022 14:15:00 +0530 Subject: [PATCH 205/237] broad-crawls.rst: fix a typo (#5714) --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 63b60312e..0927ac2d2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -68,7 +68,7 @@ IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much -to increase it will depend on how much CPU and memory you crawler will have +to increase it will depend on how much CPU and memory your crawler will have available. A good starting point is ``100``:: From 1200a545439677942085f392d7477ee37b62691e Mon Sep 17 00:00:00 2001 From: islem-esi Date: Tue, 15 Nov 2022 16:28:45 +0100 Subject: [PATCH 206/237] minor fix for readability --- scrapy/cmdline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 68267fb74..8218a51c8 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -78,7 +78,8 @@ def _pop_command_name(argv): def _print_header(settings, inproject): version = scrapy.__version__ if inproject: - print(f"Scrapy {version} - project: {settings['BOT_NAME']}\n") + print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n") + else: print(f"Scrapy {version} - no active project\n") From d5b6c236a90aac37b7942793e4943347ebaa65b8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 21 Nov 2022 05:32:26 -0300 Subject: [PATCH 207/237] Remove deprecated code (#5719) --- scrapy/utils/boto.py | 23 ----------------------- scrapy/utils/gz.py | 11 ----------- scrapy/utils/python.py | 38 -------------------------------------- tests/test_utils_python.py | 34 ++-------------------------------- 4 files changed, 2 insertions(+), 104 deletions(-) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 3374c57c7..39a681001 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,27 +1,4 @@ """Boto/botocore helpers""" -import warnings - -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning - - -def is_botocore(): - """ Returns True if botocore is available, otherwise raises NotConfigured. Never returns False. - - Previously, when boto was supported in addition to botocore, this returned False if boto was available - but botocore wasn't. - """ - message = ( - 'is_botocore() is deprecated and always returns True or raises an Exception, ' - 'so it cannot be used for checking if boto is available instead of botocore. ' - 'You can use scrapy.utils.boto.is_botocore_available() to check if botocore ' - 'is available.' - ) - warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) - try: - import botocore # noqa: F401 - return True - except ImportError: - raise NotConfigured('missing botocore library') def is_botocore_available(): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 76156a4b8..0810e1f1d 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -2,17 +2,6 @@ import struct from gzip import GzipFile from io import BytesIO -from scrapy.utils.decorators import deprecated - - -# - GzipFile's read() has issues returning leftover uncompressed data when -# input is corrupted -# - read1(), which fetches data before raising EOFError on next call -# works here -@deprecated('GzipFile.read1') -def read1(gzf, size=-1): - return gzf.read1(size) - def gunzip(data): """Gunzip the given data and return as much data as possible. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8ce030d9d..0d9fdbf23 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,20 +1,16 @@ """ This module contains essential stuff that should've come with Python itself ;) """ -import errno import gc import inspect import re import sys -import warnings import weakref from functools import partial, wraps from itertools import chain from typing import AsyncGenerator, AsyncIterable, Iterable, Union -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator -from scrapy.utils.decorators import deprecated def flatten(x): @@ -112,12 +108,6 @@ def to_bytes(text, encoding=None, errors='strict'): return text.encode(encoding, errors) -@deprecated('to_unicode') -def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of ``text``. """ - return to_unicode(text, encoding, errors) - - def re_rsearch(pattern, text, chunk_size=1024): """ This function does a reverse search in a text using a regular expression @@ -263,30 +253,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - -@deprecated -def retry_on_eintr(function, *args, **kw): - """Run a function and retry it while getting EINTR errors""" - while True: - try: - return function(*args, **kw) - except IOError as e: - if e.errno != errno.EINTR: - raise - - def without_none_values(iterable): """Return a copy of ``iterable`` with all ``None`` entries removed. @@ -337,10 +303,6 @@ class MutableChain(Iterable): def __next__(self): return next(self.data) - @deprecated("scrapy.utils.python.MutableChain.__next__") - def next(self): - return self.__next__() - async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator: for it in iterables: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b1a8fdc04..403e4f8fe 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,18 +1,14 @@ import functools -import gc import operator import platform -from itertools import count -from warnings import catch_warnings, filterwarnings from twisted.trial import unittest -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain, MutableAsyncChain) @@ -27,12 +23,7 @@ class MutableChainTest(unittest.TestCase): m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) self.assertEqual(m.__next__(), 1) - with catch_warnings(record=True) as warnings: - self.assertEqual(m.next(), 2) - self.assertEqual(len(warnings), 1) - self.assertIn('scrapy.utils.python.MutableChain.__next__', - str(warnings[0].message)) - self.assertEqual(list(m), list(range(3, 13))) + self.assertEqual(list(m), list(range(2, 13))) class MutableAsyncChainTest(unittest.TestCase): @@ -209,27 +200,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - - with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - wk = WeakKeyCache(lambda k: next(_values)) - - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) - def test_get_func_args(self): def f1(a, b, c): pass From 1a6408c3faadbd2b8b7622b8aee230b112620fad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 21 Nov 2022 07:30:20 -0300 Subject: [PATCH 208/237] Remove `FilteringLinkExtractor` --- scrapy/linkextractors/__init__.py | 89 ------------------------------- scrapy/linkextractors/lxmlhtml.py | 83 ++++++++++++++++++++++------ tests/test_linkextractors.py | 32 ----------- 3 files changed, 66 insertions(+), 138 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 08a6ca1e8..b3b1eea55 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,18 +6,6 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re -from urllib.parse import urlparse -from warnings import warn - -from parsel.csstranslator import HTMLTranslator -from w3lib.url import canonicalize_url - -from scrapy.utils.deprecate import ScrapyDeprecationWarning -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import ( - url_is_from_any_domain, url_has_any_extension, -) - # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -55,82 +43,5 @@ def _is_valid_url(url): return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} -class FilteringLinkExtractor: - - _csstranslator = HTMLTranslator() - - def __new__(cls, *args, **kwargs): - from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor - if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor): - warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' - 'please use scrapy.linkextractors.LinkExtractor instead', - ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls) - - def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, - restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): - - self.link_extractor = link_extractor - - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] - - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) - - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, - arg_to_iter(restrict_css))) - - self.canonicalize = canonicalize - if deny_extensions is None: - deny_extensions = IGNORED_EXTENSIONS - self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} - self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(restrict_text)] - - def _link_allowed(self, link): - if not _is_valid_url(link.url): - return False - if self.allow_res and not _matches(link.url, self.allow_res): - return False - if self.deny_res and _matches(link.url, self.deny_res): - return False - parsed_url = urlparse(link.url) - if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): - return False - if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): - return False - if self.restrict_text and not _matches(link.text, self.restrict_text): - return False - return True - - def matches(self, url): - - if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): - return False - - allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] - denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] - return any(allowed) and not any(denied) - - def _process_links(self, links): - links = [x for x in links if self._link_allowed(x)] - if self.canonicalize: - for link in links: - link.url = canonicalize_url(link.url) - links = self.link_extractor._process_links(links) - return links - - def _extract_links(self, *args, **kwargs): - return self.link_extractor._extract_links(*args, **kwargs) - - # Top-level imports from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index b5d2585a8..55639f504 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -3,18 +3,20 @@ Link extractor based on lxml.html """ import operator from functools import partial -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse import lxml.etree as etree +from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor +from scrapy.linkextractors import (IGNORED_EXTENSIONS, _is_valid_url, _matches, + _re_type, re) from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url - +from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain # from lxml/src/lxml/html/__init__.py XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" @@ -98,7 +100,8 @@ class LxmlParserLinkExtractor: return links -class LxmlLinkExtractor(FilteringLinkExtractor): +class LxmlLinkExtractor: + _csstranslator = HTMLTranslator() def __init__( self, @@ -118,7 +121,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): restrict_text=None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) - lx = LxmlParserLinkExtractor( + self.link_extractor = LxmlParserLinkExtractor( tag=partial(operator.contains, tags), attr=partial(operator.contains, attrs), unique=unique, @@ -126,18 +129,64 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super().__init__( - link_extractor=lx, - allow=allow, - deny=deny, - allow_domains=allow_domains, - deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, - restrict_css=restrict_css, - canonicalize=canonicalize, - deny_extensions=deny_extensions, - restrict_text=restrict_text, - ) + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + self.allow_domains = set(arg_to_iter(allow_domains)) + self.deny_domains = set(arg_to_iter(deny_domains)) + + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.restrict_xpaths += tuple(map(self._csstranslator.css_to_xpath, + arg_to_iter(restrict_css))) + + if deny_extensions is None: + deny_extensions = IGNORED_EXTENSIONS + self.canonicalize = canonicalize + self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} + self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(restrict_text)] + + def _link_allowed(self, link): + if not _is_valid_url(link.url): + return False + if self.allow_res and not _matches(link.url, self.allow_res): + return False + if self.deny_res and _matches(link.url, self.deny_res): + return False + parsed_url = urlparse(link.url) + if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): + return False + if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): + return False + if self.restrict_text and not _matches(link.text, self.restrict_text): + return False + return True + + def matches(self, url): + + if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): + return False + + allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] + return any(allowed) and not any(denied) + + def _process_links(self, links): + links = [x for x in links if self._link_allowed(x)] + if self.canonicalize: + for link in links: + link.url = canonicalize_url(link.url) + links = self.link_extractor._process_links(links) + return links + + def _extract_links(self, *args, **kwargs): + return self.link_extractor._extract_links(*args, **kwargs) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 6f133d77a..e28dc9bdb 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,12 +1,9 @@ import pickle import re import unittest -from warnings import catch_warnings -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata @@ -517,32 +514,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super().test_restrict_xpaths_with_html_entities() - - def test_filteringlinkextractor_deprecation_warning(self): - """Make sure the FilteringLinkExtractor deprecation warning is not - issued for LxmlLinkExtractor""" - with catch_warnings(record=True) as warnings: - LxmlLinkExtractor() - self.assertEqual(len(warnings), 0) - - class SubclassedLxmlLinkExtractor(LxmlLinkExtractor): - pass - - SubclassedLxmlLinkExtractor() - self.assertEqual(len(warnings), 0) - - -class FilteringLinkExtractorTest(unittest.TestCase): - - def test_deprecation_warning(self): - args = [None] * 10 - with catch_warnings(record=True) as warnings: - FilteringLinkExtractor(*args) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) - with catch_warnings(record=True) as warnings: - class SubclassedFilteringLinkExtractor(FilteringLinkExtractor): - pass - SubclassedFilteringLinkExtractor(*args) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) From c04ccbceb91b99976b674f6a57e15ed5ad5b7565 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 21 Nov 2022 15:49:33 +0100 Subject: [PATCH 209/237] doc: add instructions to debug Scrapy spiders in Visual Studio Code --- docs/topics/debug.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 4d452b4df..edbcaf432 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -150,3 +150,31 @@ available in all future runs should they be necessary again:: For more information, check the :ref:`topics-logging` section. .. _base tag: https://www.w3schools.com/tags/tag_base.asp + +Visual Studio Code +================== + +.. highlight:: json + +To debug spiders with Visual Studio Code you can use the following ``launch.json``:: + + { + "version": "0.1.0", + "configurations": [ + { + "name": "Python: Launch Scrapy Spider", + "type": "python", + "request": "launch", + "module": "scrapy", + "args": [ + "runspider", + "${file}" + ], + "console": "integratedTerminal" + } + ] + } + + +Also, make sure you enable "User Uncaught Exceptions", to catch exceptions in +your Scrapy spider. From 24a18e9af13a482c1dd6036046226cf57318477b Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Mon, 21 Nov 2022 17:41:06 -0500 Subject: [PATCH 210/237] Adds virtualsize property to _check_limit error log. --- scrapy/extensions/memusage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index f5081a7d7..aba0c8d7e 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem}, extra={'crawler': self.crawler}) + {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " From 8f2adad7a7f2d858acd082afd6e9a6835bfddcd7 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 00:48:24 -0500 Subject: [PATCH 211/237] Log self.get_virtual_size() on every call of _check_limit --- scrapy/extensions/memusage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index aba0c8d7e..c94899e0f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,7 +79,7 @@ class MemoryUsage: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", - {'memusage': mem, 'virtualsize': self.get_virtual_size()}, extra={'crawler': self.crawler}) + {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " @@ -92,6 +92,8 @@ class MemoryUsage: self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded') else: self.crawler.stop() + else: + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) def _check_warning(self): if self.warned: # warn only once From eb159c78f10f546e38e62a9230608269ac13acdd Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 03:36:00 -0500 Subject: [PATCH 212/237] Use variable and convert to megabytes. --- scrapy/extensions/memusage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index c94899e0f..bf2ee4e6d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,7 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - if self.get_virtual_size() > self.limit: + current_mem_usage = self.get_virtual_size() + if current_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -93,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': self.get_virtual_size()}) + logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once From 8e0025f53dc724d986855b572b6237d0a96fd821 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 22 Nov 2022 09:38:45 -0300 Subject: [PATCH 213/237] Remove support for override settings with `SCRAPY_` environment variables --- scrapy/utils/project.py | 17 +++++------------ tests/test_utils_project.py | 18 +++++++----------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index c66af497e..fce198db4 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -6,7 +6,7 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured ENVVAR = 'SCRAPY_SETTINGS_MODULE' @@ -67,23 +67,16 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if - k.startswith('SCRAPY_')} valid_envvars = { 'CHECK', 'PROJECT', 'PYTHON_SHELL', 'SETTINGS_MODULE', } - setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars} - if setting_envvars: - setting_envvar_list = ', '.join(sorted(setting_envvars)) - warnings.warn( - 'Use of environment variables prefixed with SCRAPY_ to override ' - 'settings is deprecated. The following environment variables are ' - f'currently defined: {setting_envvar_list}', - ScrapyDeprecationWarning - ) + + scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if + k.startswith('SCRAPY_') and k.replace('SCRAPY_', '') in valid_envvars} + settings.setdict(scrapy_envvars, priority='project') return settings diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 46452415a..e77ffa18b 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -5,9 +5,6 @@ import shutil import contextlib import warnings -from pytest import warns - -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.project import data_path, get_project_settings @@ -80,10 +77,10 @@ class GetProjectSettingsTestCase(unittest.TestCase): envvars = { 'SCRAPY_FOO': 'bar', } - with warns(ScrapyDeprecationWarning, match=': FOO') as record: - with set_env(**envvars): - get_project_settings() - assert len(record) == 1 + with set_env(**envvars): + settings = get_project_settings() + + assert settings.get("SCRAPY_FOO") is None def test_valid_and_invalid_envvars(self): value = 'tests.test_cmdline.settings' @@ -91,8 +88,7 @@ class GetProjectSettingsTestCase(unittest.TestCase): 'SCRAPY_FOO': 'bar', 'SCRAPY_SETTINGS_MODULE': value, } - with warns(ScrapyDeprecationWarning, match=': FOO') as record: - with set_env(**envvars): - settings = get_project_settings() - assert len(record) == 1 + with set_env(**envvars): + settings = get_project_settings() assert settings.get('SETTINGS_MODULE') == value + assert settings.get('SCRAPY_FOO') is None From 1506479672ee54adc2d7d1ecffc0b224f2fcf7aa Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Tue, 22 Nov 2022 10:07:32 -0300 Subject: [PATCH 214/237] Remove deprecated test --- tests/test_cmdline/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 8233e0101..802f5c198 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -31,10 +31,6 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), 'override') - def test_override_settings_using_envvar(self): - self.env['SCRAPY_TEST1'] = 'override' - self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') - def test_profiling(self): path = tempfile.mkdtemp() filename = os.path.join(path, 'res.prof') From fc8968672a5cc699f0103aeb2da1ebea7ed3c235 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 11:49:28 -0500 Subject: [PATCH 215/237] renamed variables to clarify that we are using peak memory and not current memory utilization. --- scrapy/extensions/memusage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index bf2ee4e6d..7bc6564e7 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -75,8 +75,8 @@ class MemoryUsage: self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): - current_mem_usage = self.get_virtual_size() - if current_mem_usage > self.limit: + peak_mem_usage = self.get_virtual_size() + if peak_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", @@ -94,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024}) + logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once From bdc0bca5b1aab15873b82f93a4bdf8fb3ce82824 Mon Sep 17 00:00:00 2001 From: Christopher Gambrell Date: Tue, 22 Nov 2022 12:10:49 -0500 Subject: [PATCH 216/237] Replace M occurrences with MiB for accuracy. --- scrapy/extensions/memusage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 7bc6564e7..4fdf86479 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -79,12 +79,12 @@ class MemoryUsage: if peak_mem_usage > self.limit: self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit / 1024 / 1024 - logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", + logger.error("Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} terminated: " - f"memory usage exceeded {mem}M at {socket.gethostname()}" + f"memory usage exceeded {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -94,7 +94,7 @@ class MemoryUsage: else: self.crawler.stop() else: - logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024}) + logger.info("Peak memory usage is %(virtualsize)dMiB", {'virtualsize': peak_mem_usage / 1024 / 1024}) def _check_warning(self): if self.warned: # warn only once @@ -102,12 +102,12 @@ class MemoryUsage: if self.get_virtual_size() > self.warning: self.crawler.stats.set_value('memusage/warning_reached', 1) mem = self.warning / 1024 / 1024 - logger.warning("Memory usage reached %(memusage)dM", + logger.warning("Memory usage reached %(memusage)dMiB", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: subj = ( f"{self.crawler.settings['BOT_NAME']} warning: " - f"memory usage reached {mem}M at {socket.gethostname()}" + f"memory usage reached {mem}MiB at {socket.gethostname()}" ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) From 1fdd0a70a0d56bc7829aa3028c191c062cd9f935 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Nov 2022 12:16:48 +0500 Subject: [PATCH 217/237] Restore lost typing --- scrapy/core/http2/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 780191505..14bf4c5fe 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -151,7 +151,7 @@ class Stream: self._deferred_response = Deferred(_cancel) - def __repr__(self): + def __repr__(self) -> str: return f'Stream(id={self.stream_id!r})' @property From c3b1700774bd16623622963e98fd3ec759b8a88f Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Wed, 23 Nov 2022 12:17:30 +0500 Subject: [PATCH 218/237] Restore lost typing --- scrapy/http/request/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 305eef918..1ececaf1d 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -121,7 +121,7 @@ class Request(object_ref): def encoding(self) -> str: return self._encoding - def __repr__(self): + def __repr__(self) -> str: return f"<{self.method} {self.url}>" def copy(self) -> "Request": From e769532644e1176c7984dd513249285c7f64c7d0 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 23 Nov 2022 08:30:11 -0300 Subject: [PATCH 219/237] Remove `noconnect` deprecate code --- scrapy/core/downloader/handlers/http11.py | 17 ++--------------- scrapy/core/downloader/handlers/http2.py | 14 +------------- tests/test_downloader_handlers.py | 14 +------------- tests/test_downloader_handlers_http2.py | 15 --------------- 4 files changed, 4 insertions(+), 56 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6b8a18f1a..f07f0780e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -3,7 +3,6 @@ import ipaddress import logging import re -import warnings from contextlib import suppress from io import BytesIO from time import time @@ -22,7 +21,7 @@ from zope.interface import implementer from scrapy import signals from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload +from scrapy.exceptions import StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode @@ -279,17 +278,7 @@ class ScrapyAgent: proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) - omitConnectTunnel = b'noconnect' in proxyParams - if omitConnectTunnel: - warnings.warn( - "Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Zyte Smart Proxy Manager, it doesn't require " - "this mode anymore, so you should update scrapy-crawlera " - "to scrapy-zyte-smartproxy and remove '?noconnect' " - "from the Zyte Smart Proxy Manager URL.", - ScrapyDeprecationWarning, - ) - if scheme == b'https' and not omitConnectTunnel: + if scheme == b'https': proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) return self._TunnelingAgent( @@ -302,8 +291,6 @@ class ScrapyAgent: ) else: proxyScheme = proxyScheme or b'http' - proxyHost = to_bytes(proxyHost, encoding='ascii') - proxyPort = to_bytes(str(proxyPort), encoding='ascii') proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 7bb88a193..3f1b36e92 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,4 +1,3 @@ -import warnings from time import time from typing import Optional, Type, TypeVar from urllib.parse import urldefrag @@ -69,19 +68,8 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = proxy_host.decode() - omit_connect_tunnel = b'noconnect' in proxy_params - if omit_connect_tunnel: - warnings.warn( - "Using HTTPS proxies in the noconnect mode is not " - "supported by the downloader handler. If you use Zyte " - "Smart Proxy Manager, it doesn't require this mode " - "anymore, so you should update scrapy-crawlera to " - "scrapy-zyte-smartproxy and remove '?noconnect' from the " - "Zyte Smart Proxy Manager URL." - ) - if scheme == b'https' and not omit_connect_tunnel: + if scheme == b'https': # ToDo raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') return self._ProxyAgent( diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 883960084..c69bd3da1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured from scrapy.http import Headers, HtmlResponse, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes @@ -757,18 +757,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') - - http_proxy = f'{self.getURL("")}?noconnect' - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex(ScrapyDeprecationWarning, - r'Using HTTPS proxies in the noconnect mode is deprecated'): - return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 3a9db3ee5..079267535 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -242,21 +242,6 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'/') - - http_proxy = f"{self.getURL('')}?noconnect" - request = Request('https://example.com', meta={'proxy': http_proxy}) - with self.assertWarnsRegex( - Warning, - r'Using HTTPS proxies in the noconnect mode is not supported by the ' - r'downloader handler.' - ): - return self.download_request(request, Spider('foo')).addCallback(_test) - @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): with self.assertRaises(NotImplementedError): From f6e9e6592a28a11517295847b177b14f41cb8a26 Mon Sep 17 00:00:00 2001 From: Hanzallah Burney Date: Wed, 23 Nov 2022 19:48:34 +0500 Subject: [PATCH 220/237] Cleanup settings._DictProxy and scrapy.telnet (#5730) --- scrapy/settings/__init__.py | 24 ------------------------ scrapy/utils/deprecate.py | 5 ++--- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b4e12ffdc..43ee433d1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -435,30 +435,6 @@ class BaseSettings(MutableMapping): p.text(pformat(self.copy_to_dict())) -class _DictProxy(MutableMapping): - - def __init__(self, settings, priority): - self.o = {} - self.settings = settings - self.priority = priority - - def __len__(self): - return len(self.o) - - def __getitem__(self, k): - return self.o[k] - - def __setitem__(self, k, v): - self.settings.set(k, v, priority=self.priority) - self.o[k] = v - - def __delitem__(self, k): - del self.o[k] - - def __iter__(self, k, v): - return iter(self.o) - - class Settings(BaseSettings): """ This object stores Scrapy settings for the configuration of internal diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index ae727464c..a0c83f9f1 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -2,6 +2,7 @@ import warnings import inspect +from typing import List, Tuple from scrapy.exceptions import ScrapyDeprecationWarning @@ -126,9 +127,7 @@ def _clspath(cls, forced=None): return f'{cls.__module__}.{cls.__name__}' -DEPRECATION_RULES = [ - ('scrapy.telnet.', 'scrapy.extensions.telnet.'), -] +DEPRECATION_RULES: List[Tuple[str, str]] = [] def update_classpath(path): From b6e98ce6b6c766ee735f45f25f78d957327f54e9 Mon Sep 17 00:00:00 2001 From: Hanzallah Burney Date: Thu, 24 Nov 2022 15:01:15 +0100 Subject: [PATCH 221/237] Remove unnecessary backwards compatibility comments (#5732) --- scrapy/utils/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 6404edda6..7a5f8f065 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -44,14 +44,12 @@ def build_component_list(compdict, custom=None, convert=update_classpath): raise ValueError(f'Invalid value {value} for component {name}, ' 'please provide a real number or None instead') - # BEGIN Backward compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) return type(custom)(convert(c) for c in custom) if custom is not None: compdict.update(custom) - # END Backward compatibility _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) From f85c3f3d68b03b12c0af3b3aa09ab5faad19bc37 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Nov 2022 18:46:14 +0600 Subject: [PATCH 222/237] Add a comment about `tmpname + '^'`. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c69bd3da1..8835267fe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -109,6 +109,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): self.tmpname = self.mktemp() + # add a special char to check that they are handled correctly with open(self.tmpname + '^', 'w') as f: f.write('0123456789') handler = create_instance(FileDownloadHandler, None, get_crawler()) From f9a29f03d9a0eb9173a91f225177b7bee7d382c9 Mon Sep 17 00:00:00 2001 From: Mark Mayo Date: Sun, 27 Nov 2022 23:00:13 +1300 Subject: [PATCH 223/237] Address some issues reported by Pylint (#5677) Co-authored-by: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Co-authored-by: Andrey Rahmatullin --- conftest.py | 2 +- docs/_ext/scrapydocs.py | 2 +- docs/conftest.py | 8 +++-- docs/utils/linkfix.py | 9 +++--- pylintrc | 27 +---------------- scrapy/commands/__init__.py | 2 +- scrapy/commands/genspider.py | 2 +- scrapy/commands/parse.py | 2 -- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/downloader/handlers/http11.py | 22 +++++++------- scrapy/core/downloader/tls.py | 1 - scrapy/core/downloader/webclient.py | 4 +-- scrapy/core/http2/agent.py | 2 +- scrapy/core/http2/protocol.py | 2 +- scrapy/core/scheduler.py | 4 +-- scrapy/core/scraper.py | 30 +++++++++---------- scrapy/core/spidermw.py | 21 ++++++------- scrapy/downloadermiddlewares/cookies.py | 3 +- .../downloadermiddlewares/httpcompression.py | 1 - scrapy/downloadermiddlewares/redirect.py | 8 ++--- scrapy/downloadermiddlewares/retry.py | 17 +++++------ scrapy/downloadermiddlewares/robotstxt.py | 3 +- scrapy/downloadermiddlewares/stats.py | 4 +-- scrapy/dupefilters.py | 2 +- scrapy/exporters.py | 4 +-- scrapy/extensions/httpcache.py | 14 ++++----- scrapy/extensions/telnet.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/headers.py | 7 ++--- scrapy/http/request/__init__.py | 2 +- scrapy/http/request/form.py | 4 +-- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/__init__.py | 2 +- scrapy/pipelines/files.py | 6 ++-- scrapy/responsetypes.py | 10 +++---- scrapy/robotstxt.py | 1 - scrapy/settings/__init__.py | 6 ++-- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 5 ++-- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/sitemap.py | 4 +-- scrapy/utils/conf.py | 14 ++++----- scrapy/utils/defer.py | 22 ++++++-------- scrapy/utils/display.py | 2 +- scrapy/utils/ftp.py | 1 - scrapy/utils/iterators.py | 13 ++++---- scrapy/utils/misc.py | 15 ++++------ scrapy/utils/python.py | 5 ++-- scrapy/utils/response.py | 6 ++-- scrapy/utils/serialize.py | 19 ++++++------ scrapy/utils/spider.py | 5 ++-- setup.py | 2 +- tests/CrawlerProcess/reactor_default.py | 3 +- .../reactor_default_twisted_reactor_select.py | 3 +- tests/CrawlerProcess/reactor_select.py | 4 ++- ..._select_subclass_twisted_reactor_select.py | 4 +-- .../reactor_select_twisted_reactor_select.py | 4 ++- tests/CrawlerRunner/ip_address.py | 8 ++--- tests/mockserver.py | 3 +- tests/test_command_check.py | 4 +-- tests/test_command_parse.py | 8 ++--- tests/test_commands.py | 28 ++++++++--------- tests/test_crawler.py | 5 ++-- tests/test_downloader_handlers.py | 5 ++-- tests/test_downloader_handlers_http2.py | 6 ++-- tests/test_downloadermiddleware_cookies.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- tests/test_downloadermiddleware_httpcache.py | 7 ++--- ...st_downloadermiddleware_httpcompression.py | 3 +- tests/test_exporters.py | 6 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_http_request.py | 2 +- tests/test_http_response.py | 2 +- tests/test_loader_deprecated.py | 5 ++-- tests/test_pipeline_media.py | 4 +-- tests/test_request_attribute_binding.py | 4 +-- tests/test_request_cb_kwargs.py | 2 +- tests/test_scheduler_base.py | 2 -- tests/test_spider.py | 2 +- tests/test_spiderloader/__init__.py | 2 +- tests/test_spidermiddleware_httperror.py | 2 +- tests/test_spidermiddleware_output_chain.py | 1 - tests/test_spidermiddleware_referer.py | 4 +-- tests/test_squeues_request.py | 1 - tests/test_utils_defer.py | 5 ++-- tests/test_utils_deprecate.py | 1 + tests/test_webclient.py | 2 +- 87 files changed, 219 insertions(+), 289 deletions(-) mode change 100755 => 100644 docs/utils/linkfix.py diff --git a/conftest.py b/conftest.py index 7c1da3556..2a5d55083 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path('tests/ignores.txt').open() as reader: +with Path('tests/ignores.txt').open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index f0f382da3..337604cf1 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,8 +1,8 @@ +from operator import itemgetter from docutils.parsers.rst.roles import set_classes from docutils import nodes from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode -from operator import itemgetter class settingslist_node(nodes.General, nodes.Element): diff --git a/docs/conftest.py b/docs/conftest.py index 24a72a4b6..a6dacd265 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,15 +1,17 @@ from doctest import ELLIPSIS, NORMALIZE_WHITESPACE from pathlib import Path -from scrapy.http.response.html import HtmlResponse from sybil import Sybil +from sybil.parsers.doctest import DocTestParser +from sybil.parsers.skip import skip + try: # >2.0.1 from sybil.parsers.codeblock import PythonCodeBlockParser except ImportError: from sybil.parsers.codeblock import CodeBlockParser as PythonCodeBlockParser -from sybil.parsers.doctest import DocTestParser -from sybil.parsers.skip import skip + +from scrapy.http.response.html import HtmlResponse def load_response(url: str, filename: str) -> HtmlResponse: diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py old mode 100755 new mode 100644 index 7a0c5288c..a0d9a1cbd --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -13,6 +13,7 @@ Author: dufferzafar """ import re +import sys from pathlib import Path @@ -28,11 +29,11 @@ def main(): # Read lines from the linkcheck output file try: - with Path("build/linkcheck/output.txt").open() as out: + with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() except IOError: print("linkcheck output not found; please run linkcheck first.") - exit(1) + sys.exit(1) # For every line, fix the respective file for line in output_lines: @@ -52,12 +53,12 @@ def main(): # Update the previous file if _filename: - Path(_filename).write_text(_contents) + Path(_filename).write_text(_contents, encoding="utf-8") _filename = newfilename # Read the new file to memory - _contents = Path(_filename).read_text() + _contents = Path(_filename).read_text(encoding="utf-8") _contents = _contents.replace(match.group(3), match.group(4)) else: diff --git a/pylintrc b/pylintrc index 18819feba..0a2276fb8 100644 --- a/pylintrc +++ b/pylintrc @@ -9,27 +9,19 @@ disable=abstract-method, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-indentation, bad-mcs-classmethod-argument, - bad-super-call, bare-except, - blacklisted-name, broad-except, c-extension-no-member, catching-non-exception, cell-var-from-loop, comparison-with-callable, - consider-iterating-dictionary, consider-using-dict-items, - consider-using-from-import, consider-using-in, - consider-using-set-comprehension, - consider-using-sys-exit, consider-using-with, cyclic-import, dangerous-default-value, - deprecated-method, - deprecated-module, + disallowed-name, duplicate-code, # https://github.com/PyCQA/pylint/issues/214 eval-used, expression-not-assigned, @@ -52,18 +44,12 @@ disable=abstract-method, lost-exception, method-hidden, missing-docstring, - missing-final-newline, - multiple-imports, - multiple-statements, - no-else-continue, no-else-raise, - no-else-return, no-member, no-method-argument, no-name-in-module, no-self-argument, no-value-for-parameter, - not-an-iterable, not-callable, pointless-statement, pointless-string-statement, @@ -74,10 +60,7 @@ disable=abstract-method, redefined-outer-name, reimported, signature-differs, - singleton-comparison, super-init-not-called, - super-with-arguments, - superfluous-parens, too-few-public-methods, too-many-ancestors, too-many-arguments, @@ -89,31 +72,23 @@ disable=abstract-method, too-many-locals, too-many-public-methods, too-many-return-statements, - trailing-newlines, - trailing-whitespace, unbalanced-tuple-unpacking, undefined-variable, undefined-loop-variable, unexpected-special-method-signature, - ungrouped-imports, - unidiomatic-typecheck, unnecessary-comprehension, unnecessary-dunder-call, - unnecessary-lambda, unnecessary-pass, unreachable, - unspecified-encoding, unsubscriptable-object, unused-argument, unused-import, unused-private-member, unused-variable, unused-wildcard-import, - use-implicit-booleaness-not-comparison, used-before-assignment, useless-object-inheritance, # Required for Python 2 support useless-return, useless-super-delegation, wildcard-import, - wrong-import-order, wrong-import-position diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index c4d4bebb2..d0fb4efd8 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -95,7 +95,7 @@ class ScrapyCommand: self.settings.set('LOG_ENABLED', False, priority='cmdline') if opts.pidfile: - Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep) + Path(opts.pidfile).write_text(str(os.getpid()) + os.linesep, encoding="utf-8") if opts.pdb: failure.startDebugMode() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index ed9660252..b7c2f85fb 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -63,7 +63,7 @@ class Command(ScrapyCommand): if opts.dump: template_file = self._find_template(opts.dump) if template_file: - print(template_file.read_text()) + print(template_file.read_text(encoding="utf-8")) return if len(args) != 2: raise UsageError() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index d93ab2ac5..c2df7f8a5 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,14 +6,12 @@ from itemadapter import is_item, ItemAdapter from w3lib.url import is_url from twisted.internet.defer import maybeDeferred - from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index c0146a0a6..a75532d2a 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -33,5 +33,4 @@ class HTTP10DownloadHandler: crawler=self._crawler, ) return reactor.connectSSL(host, port, factory, client_context_factory) - else: - return reactor.connectTCP(host, port, factory) + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f07f0780e..1c98e60e1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -26,7 +26,6 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.python import to_bytes, to_unicode - logger = logging.getLogger(__name__) @@ -289,16 +288,15 @@ class ScrapyAgent: bindAddress=bindaddress, pool=self._pool, ) - else: - proxyScheme = proxyScheme or b'http' - proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) - return self._ProxyAgent( - reactor=reactor, - proxyURI=to_bytes(proxyURI, encoding='ascii'), - connectTimeout=timeout, - bindAddress=bindaddress, - pool=self._pool, - ) + proxyScheme = proxyScheme or b'http' + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) + return self._ProxyAgent( + reactor=reactor, + proxyURI=to_bytes(proxyURI, encoding='ascii'), + connectTimeout=timeout, + bindAddress=bindaddress, + pool=self._pool, + ) return self._Agent( reactor=reactor, @@ -567,7 +565,7 @@ class _ResponseReader(protocol.Protocol): self._finish_response(flags=["dataloss"]) return - elif not self._fail_on_dataloss_warned: + if not self._fail_on_dataloss_warned: logger.warning("Got data loss in %s. If you want to process broken " "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" " -- This message won't be shown in further requests", diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 698a1c85c..7d67a426f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -7,7 +7,6 @@ from twisted.internet.ssl import AcceptableCiphers from scrapy.utils.ssl import x509name_to_string, get_temp_key_info - logger = logging.getLogger(__name__) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 7d048c1e4..255ca62e6 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,8 +1,8 @@ import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag - from twisted.web.http import HTTPClient + from twisted.internet import defer from twisted.internet.protocol import ClientFactory @@ -185,7 +185,7 @@ class ScrapyHTTPClientFactory(ClientFactory): @param version: The HTTP version. @type version: L{bytes} @param status: The HTTP status code, an integer represented as a - bytestring. + bytestring. @type status: L{bytes} @param message: The HTTP status message. @type message: L{bytes} diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f7b0c3f99..c6b357be3 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -140,7 +140,7 @@ class ScrapyProxyH2Agent(H2Agent): connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None, ) -> None: - super(ScrapyProxyH2Agent, self).__init__( + super().__init__( reactor=reactor, pool=pool, context_factory=context_factory, diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 1d150b7ce..6047f9ca8 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -42,7 +42,7 @@ class InvalidNegotiatedProtocol(H2Error): self.negotiated_protocol = negotiated_protocol def __str__(self) -> str: - return (f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}") + return f"Expected {PROTOCOL_NAME!r}, received {self.negotiated_protocol!r}" class RemoteTerminatedConnection(H2Error): diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 366449f51..6dd5d51a8 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -333,9 +333,9 @@ class Scheduler(BaseScheduler): path = Path(dqdir, 'active.json') if not path.exists(): return [] - with path.open() as f: + with path.open(encoding="utf-8") as f: return json.load(f) def _write_dqs_state(self, dqdir: str, state: list) -> None: - with Path(dqdir, 'active.json').open('w') as f: + with Path(dqdir, 'active.json').open('w', encoding="utf-8") as f: json.dump(state, f) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e1fdd8d13..7225e0743 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -152,9 +152,9 @@ class Scraper: """ if isinstance(result, Response): return self.spidermw.scrape_response(self.call_spider, result, request, spider) - else: # result is a Failure - dfd = self.call_spider(result, request, spider) - return dfd.addErrback(self._log_download_errors, result, request, spider) + # else result is a Failure + dfd = self.call_spider(result, request, spider) + return dfd.addErrback(self._log_download_errors, result, request, spider) def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider) -> Deferred: if isinstance(result, Response): @@ -276,17 +276,15 @@ class Scraper: return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) - else: - logkws = self.logformatter.item_error(item, ex, response, spider) - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, - exc_info=failure_to_exc_info(output)) - return self.signals.send_catch_log_deferred( - signal=signals.item_error, item=item, response=response, - spider=spider, failure=output) - else: - logkws = self.logformatter.scraped(output, response, spider) - if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + logkws = self.logformatter.item_error(item, ex, response, spider) + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, + exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, - spider=spider) + signal=signals.item_error, item=item, response=response, + spider=spider, failure=output) + logkws = self.logformatter.scraped(output, response, spider) + if logkws is not None: + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) + return self.signals.send_catch_log_deferred( + signal=signals.item_scraped, item=output, response=response, + spider=spider) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1aa02f29f..91e4b9cb4 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -121,11 +121,10 @@ class SpiderMiddlewareManager(MiddlewareManager): if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading return dfd.result - else: - # we forbid waiting here because otherwise we would need to return a deferred from - # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" - raise _InvalidOutput(msg) + # we forbid waiting here because otherwise we would need to return a deferred from + # _process_spider_exception too, which complicates the architecture + msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded" + raise _InvalidOutput(msg) elif result is None: continue else: @@ -213,8 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if last_result_is_async: return MutableAsyncChain(result, recovered) - else: - return MutableChain(result, recovered) # type: ignore[arg-type] + return MutableChain(result, recovered) # type: ignore[arg-type] async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable] ) -> Union[MutableChain, MutableAsyncChain]: @@ -227,11 +225,10 @@ class SpiderMiddlewareManager(MiddlewareManager): result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result)) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) - else: - if isinstance(recovered, AsyncIterable): - recovered_collected = await collect_asyncgen(recovered) - recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) # type: ignore[arg-type] + if isinstance(recovered, AsyncIterable): + recovered_collected = await collect_asyncgen(recovered) + recovered = MutableChain(recovered_collected) + return MutableChain(result, recovered) # type: ignore[arg-type] def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request, spider: Spider) -> Deferred: diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index c592acb57..3cba269f1 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -9,7 +9,6 @@ from scrapy.http.cookies import CookieJar from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode - logger = logging.getLogger(__name__) @@ -129,7 +128,7 @@ class CookiesMiddleware: """ if not request.cookies: return [] - elif isinstance(request.cookies, dict): + if isinstance(request.cookies, dict): cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) else: cookies = request.cookies diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 4e7feeeaf..e57f6fcf8 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -8,7 +8,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.gz import gunzip - ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] try: diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index c8c84ffb2..fba825947 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -8,7 +8,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured - logger = logging.getLogger(__name__) @@ -56,10 +55,9 @@ class BaseRedirectMiddleware: {'reason': reason, 'redirected': redirected, 'request': request}, extra={'spider': spider}) return redirected - else: - logger.debug("Discarding %(request)s: max redirections reached", - {'request': request}, extra={'spider': spider}) - raise IgnoreRequest("max redirections reached") + logger.debug("Discarding %(request)s: max redirections reached", + {'request': request}, extra={'spider': spider}) + raise IgnoreRequest("max redirections reached") def _redirect_request_using_get(self, request, redirect_url): redirect_request = _build_redirect_request( diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c6cc7c56d..2de610e86 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -113,15 +113,14 @@ def get_retry_request( stats.inc_value(f'{stats_base_key}/count') stats.inc_value(f'{stats_base_key}/reason_count/{reason}') return new_request - else: - stats.inc_value(f'{stats_base_key}/max_reached') - logger.error( - "Gave up retrying %(request)s (failed %(retry_times)d times): " - "%(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}, - ) - return None + stats.inc_value(f'{stats_base_key}/max_reached') + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) + return None class RetryMiddleware: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index e66bf177e..7bd39aa43 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -81,8 +81,7 @@ class RobotsTxtMiddleware: return result self._parsers[netloc].addCallback(cb) return d - else: - return self._parsers[netloc] + return self._parsers[netloc] def _logerror(self, failure, request, spider): if failure.type is not IgnoreRequest: diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 25fb1ed9d..2bb022c16 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,9 +1,9 @@ +from twisted.web import http + from scrapy.exceptions import NotConfigured from scrapy.utils.python import global_object_name, to_bytes from scrapy.utils.request import request_httprepr -from twisted.web import http - def get_header_size(headers): size = 0 diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 2b8b09614..6dca11d31 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -55,7 +55,7 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = Path(path, 'requests.seen').open('a+') + self.file = Path(path, 'requests.seen').open('a+', encoding="utf-8") self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 76cbe4d4b..2135c3bbc 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -334,9 +334,9 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value): if isinstance(value, Item): return self.export_item(value) - elif is_item(value): + if is_item(value): return dict(self._serialize_item(value)) - elif is_listlike(value): + if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode if isinstance(value, (str, bytes)): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 58c4e644e..4e76fe5e5 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -17,7 +17,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode - logger = logging.getLogger(__name__) @@ -81,25 +80,24 @@ class RFC2616Policy: if b'no-store' in cc: return False # Never cache 304 (Not Modified) responses - elif response.status == 304: + if response.status == 304: return False # Cache unconditionally if configured to do so - elif self.always_store: + if self.always_store: return True # Any hint on response expiration is good - elif b'max-age' in cc or b'Expires' in response.headers: + if b'max-age' in cc or b'Expires' in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set - elif response.status in (300, 301, 308): + if response.status in (300, 301, 308): return True # Other statuses without expiration requires at least one validator - elif response.status in (200, 203, 401): + if response.status in (200, 203, 401): return b'Last-Modified' in response.headers or b'ETag' in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration # info and can not be revalidated - else: - return False + return False def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 1663604e7..054350e4a 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -11,6 +11,7 @@ import binascii import os from twisted.internet import protocol + try: from twisted.conch import manhole, telnet from twisted.conch.insults import insults @@ -26,7 +27,6 @@ from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.decorators import defers - logger = logging.getLogger(__name__) # signal to update telnet variables diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 56e5ad2d2..eb21e426e 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -56,7 +56,7 @@ class AutoThrottle: { 'slot': key, 'concurrency': conc, 'delay': slot.delay * 1000, 'delaydiff': diff * 1000, - 'latency': latency * 1000, 'size': size + 'latency': latency * 1000, 'size': size, }, extra={'spider': spider} ) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 9c03fe54f..2e0020890 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -37,12 +37,11 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, str): + if isinstance(x, str): return x.encode(self.encoding) - elif isinstance(x, int): + if isinstance(x, int): return str(x).encode(self.encoding) - else: - raise TypeError(f'Unsupported value type: {type(x)}') + raise TypeError(f'Unsupported value type: {type(x)}') def __getitem__(self, key): try: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1ececaf1d..a1001fc4a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -185,7 +185,7 @@ class Request(object_ref): } for attr in self.attributes: d.setdefault(attr, getattr(self, attr)) - if type(self) is not Request: + if type(self) is not Request: # pylint: disable=unidiomatic-typecheck d["_class"] = self.__module__ + '.' + self.__class__.__name__ return d diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0c947565a..8e0a7fae2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -190,7 +190,7 @@ def _select_value(ele: SelectElement, n: str, v: str): # And for select tags without options o = ele.value_options return (n, o[0]) if o else (None, None) - elif v is not None and multiple: + if v is not None and multiple: # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') @@ -236,7 +236,7 @@ def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tup el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') - elif len(el) > 1: + if len(el) > 1: raise ValueError(f"Multiple elements found ({el!r}) matching the " f"criteria in clickdata: {clickdata!r}") else: diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 55639f504..1e6ab984a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,7 +5,7 @@ import operator from functools import partial from urllib.parse import urljoin, urlparse -import lxml.etree as etree +from lxml import etree from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index b5725a8ee..c42dd423e 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -18,7 +18,7 @@ class ItemPipelineManager(MiddlewareManager): return build_component_list(settings.getwithbase('ITEM_PIPELINES')) def _add_middleware(self, pipe): - super(ItemPipelineManager, self)._add_middleware(pipe) + super()._add_middleware(pipe) if hasattr(pipe, 'process_item'): self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 4e0211d1b..51aedafe8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -221,16 +221,14 @@ class GCSFilesStore: checksum = blob.md5_hash last_modified = time.mktime(blob.updated.timetuple()) return {'checksum': checksum, 'last_modified': last_modified} - else: - return {} + return {} blob_path = self._get_blob_path(path) return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess) def _get_content_type(self, headers): if headers and 'Content-Type' in headers: return headers['Content-Type'] - else: - return 'application/octet-stream' + return 'application/octet-stream' def _get_blob_path(self, path): return self.prefix + path diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 3efd4d2fd..a34d7d25f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -42,11 +42,10 @@ class ResponseTypes: """Return the most appropriate Response class for the given mimetype""" if mimetype is None: return Response - elif mimetype in self.classes: + if mimetype in self.classes: return self.classes[mimetype] - else: - basetype = f"{mimetype.split('/')[0]}/*" - return self.classes.get(basetype, Response) + basetype = f"{mimetype.split('/')[0]}/*" + return self.classes.get(basetype, Response) def from_content_type(self, content_type, content_encoding=None): """Return the most appropriate Response class from an HTTP Content-Type @@ -83,8 +82,7 @@ class ResponseTypes: mimetype, encoding = self.mimetypes.guess_type(filename) if mimetype and not encoding: return self.from_mimetype(mimetype) - else: - return Response + return Response def from_body(self, body): """Try to guess the appropriate response based on the body content. diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index f8649e56b..1d40f0484 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -4,7 +4,6 @@ from abc import ABCMeta, abstractmethod from scrapy.utils.python import to_unicode - logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 43ee433d1..b540e6182 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -24,8 +24,7 @@ def get_settings_priority(priority): """ if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] - else: - return priority + return priority class SettingsAttribute: @@ -260,8 +259,7 @@ class BaseSettings(MutableMapping): """ if len(self) > 0: return max(self.getpriority(name) for name in self) - else: - return get_settings_priority('default') + return get_settings_priority('default') def __setitem__(self, name, value): self.set(name, value) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 448bc1367..98a186510 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -65,7 +65,7 @@ class OffsiteMiddleware: for domain in allowed_domains: if domain is None: continue - elif url_pattern.match(domain): + if url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " f"Ignoring URL entry {domain} in allowed_domains.") warnings.warn(message, URLWarning) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8027beb92..4a6c4de5e 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -189,8 +189,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - else: - return origin + return origin class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -216,7 +215,7 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response_url) if origin == self.origin(request_url): return self.stripped_referrer(response_url) - elif ( + if ( self.tls_protected(response_url) and self.potentially_trustworthy(request_url) or not self.tls_protected(response_url) ): diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 2d9328633..b514ed9ce 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -26,7 +26,7 @@ def _identity_process_request(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, str): + if isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1f72e76b7..43d138753 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -69,7 +69,7 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif gzip_magic_number(response): + if gzip_magic_number(response): return gunzip(response.body) # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) @@ -80,7 +80,7 @@ class SitemapSpider(Spider): # without actually being a .xml.gz file in the first place, # merely XML gzip-compressed on the fly, # in other word, here, we have plain XML - elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): + if response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 126ead020..8ea42ce75 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -35,9 +35,8 @@ def build_component_list(compdict, custom=None, convert=update_classpath): else: compbs.set(convert(k), v, priority=prio) return compbs - else: - _check_components(compdict) - return {convert(k): v for k, v in compdict.items()} + _check_components(compdict) + return {convert(k): v for k, v in compdict.items()} def _validate_values(compdict): """Fail if a value in the components dict is not a real number or None.""" @@ -181,11 +180,10 @@ def feed_process_params_from_cli(settings, output: List[str], output_format=None ) warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2) return {output[0]: {'format': output_format}} - else: - raise UsageError( - 'The -t command-line option cannot be used if multiple output ' - 'URIs are specified' - ) + raise UsageError( + 'The -t command-line option cannot be used if multiple output ' + 'URIs are specified' + ) result: Dict[str, Dict[str, Any]] = {} for element in output: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 38aefd6d0..ddacfaa49 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -58,10 +58,9 @@ def defer_succeed(result) -> Deferred: def defer_result(result) -> Deferred: if isinstance(result, Deferred): return result - elif isinstance(result, failure.Failure): + if isinstance(result, failure.Failure): return defer_fail(result) - else: - return defer_succeed(result) + return defer_succeed(result) def mustbe_deferred(f: Callable, *args, **kw) -> Deferred: @@ -267,10 +266,9 @@ def deferred_from_coro(o) -> Any: # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return ensureDeferred(o) - else: - # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = get_asyncio_event_loop_policy().get_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor + event_loop = get_asyncio_event_loop_policy().get_event_loop() + return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) return o @@ -295,12 +293,11 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred: if isinstance(result, Deferred): return result - elif asyncio.isfuture(result) or inspect.isawaitable(result): + if asyncio.isfuture(result) or inspect.isawaitable(result): return deferred_from_coro(result) - elif isinstance(result, failure.Failure): + if isinstance(result, failure.Failure): return defer.fail(result) - else: - return defer.succeed(result) + return defer.succeed(result) def deferred_to_future(d: Deferred) -> Future: @@ -352,5 +349,4 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: """ if not is_asyncio_reactor_installed(): return d - else: - return deferred_to_future(d) + return deferred_to_future(d) diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index d28df40c7..64969cb13 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -5,8 +5,8 @@ pprint and pformat wrappers with colorization support import ctypes import platform import sys -from packaging.version import Version as parse_version from pprint import pformat as pformat_ +from packaging.version import Version as parse_version def _enable_windows_terminal_processing(): diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 6cace4f07..775b25ea8 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,5 +1,4 @@ import posixpath - from ftplib import error_perm, FTP from posixpath import dirname diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 3b504e56a..da77ca46f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -138,8 +138,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): {'csvlnum': csv_r.line_num, 'csvrow': len(row), 'csvheader': len(headers)}) continue - else: - yield dict(zip(headers, row)) + yield dict(zip(headers, row)) def _body_or_str(obj, unicode=True): @@ -152,11 +151,9 @@ def _body_or_str(obj, unicode=True): if isinstance(obj, Response): if not unicode: return obj.body - elif isinstance(obj, TextResponse): + if isinstance(obj, TextResponse): return obj.text - else: - return obj.body.decode('utf-8') - elif isinstance(obj, str): + return obj.body.decode('utf-8') + if isinstance(obj, str): return obj if unicode else obj.encode('utf-8') - else: - return obj.decode('utf-8') if unicode else obj + return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index e0f7ca9e5..1e0342ace 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -30,10 +30,9 @@ def arg_to_iter(arg): """ if arg is None: return [] - elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): + if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): return arg - else: - return [arg] + return [arg] def load_object(path): @@ -49,9 +48,8 @@ def load_object(path): if not isinstance(path, str): if callable(path): return path - else: - raise TypeError("Unexpected argument type, expected string " - f"or object, got: {type(path)}") + raise TypeError("Unexpected argument type, expected string " + f"or object, got: {type(path)}") try: dot = path.rindex('.') @@ -115,9 +113,8 @@ def extract_regex(regex, text, encoding='utf-8'): if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] - else: - return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) - for s in strings] + return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) + for s in strings] def md5sum(file): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0d9fdbf23..9df1c91de 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -187,10 +187,9 @@ def get_func_args(func, stripself=False): elif hasattr(func, '__call__'): if inspect.isroutine(func): return [] - elif getattr(func, '__name__', None) == '__call__': + if getattr(func, '__name__', None) == '__call__': return [] - else: - return get_func_args(func.__call__, True) + return get_func_args(func.__call__, True) else: raise TypeError(f'{type(func)} is not callable') if stripself: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 741dce350..23bd2da65 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -9,14 +9,14 @@ import webbrowser from typing import Any, Callable, Iterable, Optional, Tuple, Union from weakref import WeakKeyDictionary +from twisted.web import http +from w3lib import html import scrapy from scrapy.http.response import Response -from twisted.web import http + from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.decorators import deprecated -from w3lib import html - _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary() diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index a73cf03c5..3602043f3 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -16,24 +16,23 @@ class ScrapyJSONEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, set): return list(o) - elif isinstance(o, datetime.datetime): + if isinstance(o, datetime.datetime): return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}") - elif isinstance(o, datetime.date): + if isinstance(o, datetime.date): return o.strftime(self.DATE_FORMAT) - elif isinstance(o, datetime.time): + if isinstance(o, datetime.time): return o.strftime(self.TIME_FORMAT) - elif isinstance(o, decimal.Decimal): + if isinstance(o, decimal.Decimal): return str(o) - elif isinstance(o, defer.Deferred): + if isinstance(o, defer.Deferred): return str(o) - elif is_item(o): + if is_item(o): return ItemAdapter(o).asdict() - elif isinstance(o, Request): + if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" - elif isinstance(o, Response): + if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" - else: - return super().default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index d0fd1757d..b2da69404 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -12,12 +12,11 @@ logger = logging.getLogger(__name__) def iterate_spider_output(result): if inspect.isasyncgen(result): return result - elif inspect.iscoroutine(result): + if inspect.iscoroutine(result): d = deferred_from_coro(result) d.addCallback(iterate_spider_output) return d - else: - return arg_to_iter(deferred_from_coro(result)) + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/setup.py b/setup.py index e413ea6e4..82ac86cdd 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ setup( 'Tracker': 'https://github.com/scrapy/scrapy/issues', }, description='A high-level Web Crawling and Web Scraping framework', - long_description=open('README.rst').read(), + long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 2c867df61..06b849de3 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,6 +1,7 @@ +from twisted.internet import reactor # noqa: F401 + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py index c2b30b044..a6dff0e05 100644 --- a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py @@ -1,6 +1,7 @@ +from twisted.internet import reactor # noqa: F401 + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import reactor # noqa: F401 class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index ca70c06a0..907f50784 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,6 +1,8 @@ +from twisted.internet import selectreactor + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import selectreactor + selectreactor.install() diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py index 0035daf1e..6f48da691 100644 --- a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py @@ -1,7 +1,7 @@ -import scrapy -from scrapy.crawler import CrawlerProcess from twisted.internet.main import installReactor from twisted.internet.selectreactor import SelectReactor +import scrapy +from scrapy.crawler import CrawlerProcess class SelectReactorSubclass(SelectReactor): diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py index 4f8394edb..c422b13ff 100644 --- a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py +++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py @@ -1,6 +1,8 @@ +from twisted.internet import selectreactor + import scrapy from scrapy.crawler import CrawlerProcess -from twisted.internet import selectreactor + selectreactor.install() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index f545de39f..417294447 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -8,7 +8,6 @@ from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging - from tests.mockserver import MockServer, MockDNSServer @@ -30,9 +29,10 @@ class LocalhostSpider(Spider): def parse(self, response): netloc = urlparse(response.url).netloc - self.logger.info("Host: %s" % netloc.split(":")[0]) - self.logger.info("Type: %s" % type(response.ip_address)) - self.logger.info("IP address: %s" % response.ip_address) + host = netloc.split(":")[0] + self.logger.info(f"Host: {host}") + self.logger.info(f"Type: {type(response.ip_address)}") + self.logger.info(f"IP address: {response.ip_address}") if __name__ == "__main__": diff --git a/tests/mockserver.py b/tests/mockserver.py index 7916798f7..6d2d95692 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -29,8 +29,7 @@ def getarg(request, name, default=None, type=None): if type is not None: value = type(value) return value - else: - return default + return default # most of the following resources are copied from twisted.web.test.test_webclient diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 4077a9bce..049076e5c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -6,7 +6,7 @@ class CheckCommandTest(CommandTest): command = 'check' def setUp(self): - super(CheckCommandTest, self).setUp() + super().setUp() self.spider_name = 'check_spider' self.spider = (self.proj_mod_path / 'spiders' / 'checkspider.py').resolve() @@ -24,7 +24,7 @@ class CheckSpider(scrapy.Spider): {contracts} \"\"\" {parse_def} - """) + """, encoding="utf-8") def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 4053c95d5..368b80513 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -92,7 +92,7 @@ class MyBadCrawlSpider(CrawlSpider): def parse(self, response): return [scrapy.Item(), dict(foo='bar')] -""") +""", encoding="utf-8") (self.proj_mod_path / 'pipelines.py').write_text(""" import logging @@ -103,9 +103,9 @@ class MyPipeline: def process_item(self, item, spider): logging.info('It Works!') return item -""") +""", encoding="utf-8") - with (self.proj_mod_path / 'settings.py').open("a") as f: + with (self.proj_mod_path / 'settings.py').open("a", encoding="utf-8") as f: f.write(f""" ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """) @@ -256,7 +256,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} self.assertTrue(file_path.is_file()) content = '[\n{},\n{"foo": "bar"}\n]' - self.assertEqual(file_path.read_text(), content) + self.assertEqual(file_path.read_text(encoding="utf-8"), content) def test_parse_add_options(self): command = parse.Command() diff --git a/tests/test_commands.py b/tests/test_commands.py index ea4cde363..91476abf8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -107,7 +107,7 @@ class ProjectTest(unittest.TestCase): def find_in_file(self, filename: Union[str, os.PathLike], regex) -> Optional[re.Match]: """Find first pattern occurrence in file""" pattern = re.compile(regex) - with Path(filename).open("r") as f: + with Path(filename).open("r", encoding="utf-8") as f: for line in f: match = pattern.search(line) if match is not None: @@ -475,7 +475,7 @@ class GenspiderCommandTest(CommandTest): assert file_path.exists() # change name of spider but not its file name - with file_path.open('r+') as spider_file: + with file_path.open('r+', encoding="utf-8") as spider_file: file_data = spider_file.read() file_data = file_data.replace("name = \'example\'", "name = \'renamed\'") spider_file.seek(0) @@ -489,14 +489,14 @@ class GenspiderCommandTest(CommandTest): self.assertIn(f"Created spider {file_name!r} using template \'basic\' in module", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{file_path.resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertEqual(file_contents_after, file_contents_before) def test_same_filename_as_existing_spider_force(self): @@ -536,7 +536,7 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertIn(f"Created spider {file_name!r} using template \'basic\' ", out) assert file_path.exists() modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text() + file_contents_before = file_path.read_text(encoding="utf-8") if force: # use different template to ensure contents were changed @@ -544,14 +544,14 @@ class GenspiderStandaloneCommandTest(ProjectTest): self.assertIn(f"Created spider {file_name!r} using template \'crawl\' ", out) modify_time_after = file_path.stat().st_mtime self.assertNotEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertNotEqual(file_contents_after, file_contents_before) else: p, out, err = self.proc('genspider', file_name, 'example.com') self.assertIn(f"{Path(self.temp_path, file_name + '.py').resolve()} already exists", out) modify_time_after = file_path.stat().st_mtime self.assertEqual(modify_time_after, modify_time_before) - file_contents_after = file_path.read_text() + file_contents_after = file_path.read_text(encoding="utf-8") self.assertEqual(file_contents_after, file_contents_before) def test_same_name_as_existing_file_force(self): @@ -596,7 +596,7 @@ class BadSpider(scrapy.Spider): fname = (tmpdir / name).resolve() else: fname = (tmpdir / self.spider_filename).resolve() - fname.write_text(content) + fname.write_text(content, encoding="utf-8") try: yield str(fname) finally: @@ -754,11 +754,11 @@ class MySpider(scrapy.Spider): ) return [] """ - Path(self.cwd, "example.json").write_text("not empty") + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with Path(self.cwd, "example.json").open() as f2: + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") @@ -798,7 +798,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' def setUp(self): - super(WindowsRunSpiderCommandTest, self).setUp() + super().setUp() def test_start_requests_errors(self): log = self.get_log(self.badspider, name='badspider.pyw') @@ -860,7 +860,7 @@ class ViewCommandTest(CommandTest): class CrawlCommandTest(CommandTest): def crawl(self, code, args=()): - Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code) + Path(self.proj_mod_path, 'spiders', 'myspider.py').write_text(code, encoding="utf-8") return self.proc('crawl', 'myspider', *args) def get_log(self, code, args=()): @@ -912,11 +912,11 @@ class MySpider(scrapy.Spider): ) return [] """ - Path(self.cwd, "example.json").write_text("not empty") + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") args = ['-O', 'example.json'] log = self.get_log(spider_code, args=args) self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log) - with Path(self.cwd, "example.json").open() as f2: + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: first_line = f2.readline() self.assertNotEqual(first_line, "not empty") diff --git a/tests/test_crawler.py b/tests/test_crawler.py index c09f1a6f2..e0902fdbe 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -11,6 +11,9 @@ from twisted.internet import defer from twisted.python.versions import Version from twisted.trial import unittest +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.exceptions import ScrapyDeprecationWarning @@ -23,8 +26,6 @@ from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet from scrapy.utils.test import get_testenv -from pkg_resources import parse_version -from w3lib import __version__ as w3lib_version from tests.mockserver import MockServer diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0334be743..3dc2745a0 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -110,7 +110,7 @@ class FileTestCase(unittest.TestCase): def setUp(self): # add a special char to check that they are handled correctly self.tmpname = Path(self.mktemp() + '^') - Path(self.tmpname).write_text('0123456789') + Path(self.tmpname).write_text("0123456789", encoding="utf-8") handler = create_instance(FileDownloadHandler, None, get_crawler()) self.download_request = handler.download_request @@ -722,8 +722,7 @@ class UriResource(resource.Resource): # ToDo: implement proper HTTPS proxy tests, not faking them. if request.method != b'CONNECT': return request.uri - else: - return b'' + return b'' class HttpProxyTestCase(unittest.TestCase): diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index 079267535..c2fa3ec57 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -171,7 +171,7 @@ class Https2InvalidDNSId(Https2TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https2InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -190,7 +190,7 @@ class Https2InvalidDNSPattern(Https2TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https2InvalidDNSPattern, self).setUp() + super().setUp() @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") @@ -245,4 +245,4 @@ class Https2ProxyTestCase(Http11ProxyTestCase): @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): with self.assertRaises(NotImplementedError): - yield super(Https2ProxyTestCase, self).test_download_with_proxy_https_timeout() + yield super().test_download_with_proxy_https_timeout() diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index ba7453255..dd835b9c9 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,6 +1,6 @@ import logging -from testfixtures import LogCapture from unittest import TestCase +from testfixtures import LogCapture import pytest diff --git a/tests/test_downloadermiddleware_decompression.py b/tests/test_downloadermiddleware_decompression.py index b2b5ce77d..1c4cae6d1 100644 --- a/tests/test_downloadermiddleware_decompression.py +++ b/tests/test_downloadermiddleware_decompression.py @@ -2,8 +2,8 @@ from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware from scrapy.spiders import Spider -from tests import get_testdata from scrapy.utils.test import assert_samelines +from tests import get_testdata def _test_data(formats): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 928c007f5..b3d8264ba 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -272,10 +272,9 @@ class RFC2616PolicyTest(DefaultStorageTest): if result: assert isinstance(result, (Request, Response)) return result - else: - result = mw.process_response(request, response, self.spider) - assert isinstance(result, Response) - return result + result = mw.process_response(request, response, self.spider) + assert isinstance(result, Response) + return result except Exception: print('Request', request) print('Response', response) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 6f4e217e6..b0272143d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest import TestCase, SkipTest from warnings import catch_warnings +from w3lib.encoding import resolve_encoding from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS @@ -12,8 +13,6 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir -from w3lib.encoding import resolve_encoding - SAMPLEDIR = Path(tests_datadir, 'compressed') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 69ac928c3..86c5d8b5e 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -400,8 +400,7 @@ class XmlItemExporterTest(BaseItemExporterTest): children = list(elem.iterchildren()) if children: return [(child.tag, sorted(xmltuple(child))) for child in children] - else: - return [(elem.tag, [(elem.text, ())])] + return [(elem.tag, [(elem.text, ())])] def xmlsplit(xmlcontent): doc = lxml.etree.fromstring(xmlcontent) @@ -621,8 +620,7 @@ class CustomExporterItemTest(unittest.TestCase): def serialize_field(self, field, name, value): if name == 'age': return str(int(value) + 1) - else: - return super().serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 402348cf9..e461bcf2c 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -164,7 +164,7 @@ class RequestHeaders(LeafResource): def get_client_certificate(key_file: Path, certificate_file: Path) -> PrivateCertificate: - pem = key_file.read_text() + certificate_file.read_text() + pem = key_file.read_text(encoding="utf-8") + certificate_file.read_text(encoding="utf-8") return PrivateCertificate.loadPEM(pem) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 579ef9fa2..9f7f1854f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -223,7 +223,7 @@ class RequestTest(unittest.TestCase): r1 = CustomRequest('http://www.example.com') r2 = r1.copy() - assert type(r2) is CustomRequest + assert isinstance(r2, CustomRequest) def test_replace(self): """Test Request.replace() method""" diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b42c95045..f51f3d988 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase): r1 = CustomResponse('http://www.example.com') r2 = r1.copy() - assert type(r2) is CustomResponse + assert isinstance(r2, CustomResponse) def test_replace(self): """Test Response.replace() method""" diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index 0fd52da5f..14b3b5568 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -391,10 +391,9 @@ class BasicItemLoaderTest(unittest.TestCase): def join(values, sep=None, loader_context=None, ignored=None): if sep is not None: return sep.join(values) - elif loader_context and 'sep' in loader_context: + if loader_context and 'sep' in loader_context: return loader_context['sep'].join(values) - else: - return ''.join(values) + return ''.join(values) class TestItemLoader(NameItemLoader): name_out = Compose(partial(join, sep='+')) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 84e867660..0a94ae699 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -388,11 +388,11 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): def thumb_path(self, request, thumb_id, response=None, info=None): self._mockcalled.append('thumb_path') - return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info) + return super().thumb_path(request, thumb_id, response, info) def get_images(self, response, request, info): self._mockcalled.append('get_images') - return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info) + return super().get_images(response, request, info) def image_downloaded(self, response, request, info): self._mockcalled.append('image_downloaded') diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 0406d906f..15e400327 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,12 +1,12 @@ from twisted.internet import defer from twisted.trial.unittest import TestCase +from testfixtures import LogCapture + from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler -from testfixtures import LogCapture - from tests.mockserver import MockServer from tests.spiders import SingleRequestSpider diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 002a04358..63e3aac00 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -91,7 +91,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.checks.append(kwargs['callback'] == 'some_callback') self.crawler.stats.inc_value('boolean_checks', 3) elif response.url.endswith('/general_without'): - self.checks.append(kwargs == {}) + self.checks.append(kwargs == {}) # pylint: disable=use-implicit-booleaness-not-comparison self.crawler.stats.inc_value('boolean_checks') def parse_no_kwargs(self, response): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index fc234a83d..6602b626d 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -11,10 +11,8 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer - PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] diff --git a/tests/test_spider.py b/tests/test_spider.py index e1527620f..cb66066b0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -7,6 +7,7 @@ from unittest import mock from testfixtures import LogCapture from twisted.trial import unittest +from w3lib.url import safe_url_string from scrapy import signals from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse @@ -22,7 +23,6 @@ from scrapy.spiders import ( from scrapy.linkextractors import LinkExtractor from scrapy.utils.test import get_crawler from tests import get_testdata -from w3lib.url import safe_url_string class SpiderTest(unittest.TestCase): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 697105967..0b6b51a5b 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -3,6 +3,7 @@ import shutil import warnings from pathlib import Path +import tempfile from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -10,7 +11,6 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spiders when running this test # alone import scrapy -import tempfile from scrapy.interfaces import ISpiderLoader from scrapy.spiderloader import SpiderLoader from scrapy.settings import Settings diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 46f74ae52..ee11ee492 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -6,11 +6,11 @@ from twisted.trial.unittest import TestCase as TrialTestCase from twisted.internet import defer from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.mockserver import MockServer from tests.spiders import MockServerSpider diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dac246fb6..8dd1def17 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -4,7 +4,6 @@ from twisted.trial.unittest import TestCase from scrapy import Request, Spider from scrapy.utils.test import get_crawler - from tests.mockserver import MockServer diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 9456b01d4..63daf0b8a 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,8 +1,8 @@ from urllib.parse import urlparse from unittest import TestCase import warnings - from scrapy.http import Response, Request + from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware @@ -380,7 +380,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): scheme = urlparse(request).scheme if scheme == 'https': return b'https://python.org/' - elif scheme == 'http': + if scheme == 'http': return b'http://python.org/' diff --git a/tests/test_squeues_request.py b/tests/test_squeues_request.py index c5fcc1853..22dafb2d2 100644 --- a/tests/test_squeues_request.py +++ b/tests/test_squeues_request.py @@ -16,7 +16,6 @@ from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler - """ Queues that handle requests """ diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index d39de7430..97c1c60d1 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -193,9 +193,8 @@ class AsyncCooperatorTest(unittest.TestCase): delay = random.random() / 8 reactor.callLater(delay, dfd.callback, None) return dfd - else: - # simulate trivial sync processing - results.append(o) + # simulate trivial sync processing + results.append(o) @staticmethod def get_async_iterable(length): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index e47afa266..50c63dfab 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -2,6 +2,7 @@ import inspect import unittest from unittest import mock import warnings + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 69d9a9e3a..3dcaf71dd 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -47,7 +47,7 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): from twisted.web.client import _makeGetterFactory return _makeGetterFactory( - to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs, ).deferred From 5bd27191a2fd9ec50936c05a377f76bd14aa8266 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 28 Nov 2022 23:37:57 +0500 Subject: [PATCH 224/237] Bump mypy, flake8, and pylint (#5738) --- scrapy/commands/__init__.py | 2 +- scrapy/core/scraper.py | 24 ++++++++++++++++++++++-- scrapy/crawler.py | 4 ++-- scrapy/http/response/__init__.py | 7 +++---- scrapy/http/response/text.py | 6 ++---- scrapy/middleware.py | 4 ++-- tox.ini | 6 +++--- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d0fb4efd8..b9ba3335e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -23,7 +23,7 @@ class ScrapyCommand: exitcode = 0 - def __init__(self): + def __init__(self) -> None: self.settings: Any = None # set in scrapy.cmdline def set_crawler(self, crawler): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7225e0743..69ac1cdaf 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -1,8 +1,22 @@ """This module implements the Scraper component which parses responses and extracts information from them""" +from __future__ import annotations + import logging from collections import deque -from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union +from typing import ( + Any, + AsyncGenerator, + AsyncIterable, + Deque, + Generator, + Iterable, + Optional, + Set, + TYPE_CHECKING, + Tuple, + Union, +) from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks @@ -26,6 +40,10 @@ from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.spider import iterate_spider_output +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + QueueTuple = Tuple[Union[Response, Failure], Request, Deferred] @@ -75,7 +93,7 @@ class Slot: class Scraper: - def __init__(self, crawler): + def __init__(self, crawler: Crawler) -> None: self.slot: Optional[Slot] = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) @@ -174,6 +192,7 @@ class Scraper: def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider) -> None: exc = _failure.value if isinstance(exc, CloseSpider): + assert self.crawler.engine is not None # typing self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return logkws = self.logformatter.spider_error(_failure, request, response, spider) @@ -214,6 +233,7 @@ class Scraper: """ assert self.slot is not None # typing if isinstance(output, Request): + assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) elif is_item(output): self.slot.itemproc_size += 1 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 04c6891e3..4700a30ab 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -109,7 +109,7 @@ class Crawler: self.settings.freeze() self.crawling = False self.spider = None - self.engine = None + self.engine: Optional[ExecutionEngine] = None @defer.inlineCallbacks def crawl(self, *args, **kwargs): diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index fb2d0f165..7626946ec 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -140,8 +140,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Request + dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -179,8 +178,8 @@ class Response(object_ref): def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Generator[Request, None, None] + dont_filter=False, errback=None, cb_kwargs=None, + flags=None) -> Generator[Request, None, None]: """ .. versionadded:: 2.0 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index bfcde878d..da81d0a4a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -142,8 +142,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, flags=None): - # type: (...) -> Request + dont_filter=False, errback=None, cb_kwargs=None, flags=None) -> Request: """ Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, @@ -184,8 +183,7 @@ class TextResponse(Response): def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, dont_filter=False, errback=None, cb_kwargs=None, flags=None, - css=None, xpath=None): - # type: (...) -> Generator[Request, None, None] + css=None, xpath=None) -> Generator[Request, None, None]: """ A generator that produces :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 8d7e5a602..431bd76dc 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,7 +1,7 @@ import logging import pprint from collections import defaultdict, deque -from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast +from typing import Any, Callable, Deque, Dict, Iterable, Tuple, Union, cast from twisted.internet.defer import Deferred @@ -19,7 +19,7 @@ class MiddlewareManager: component_name = 'foo middleware' - def __init__(self, *middlewares): + def __init__(self, *middlewares: Any) -> None: self.middlewares = middlewares # Only process_spider_output and process_spider_exception can be None. # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. diff --git a/tox.ini b/tox.ini index eee99cb2d..4d0f0291b 100644 --- a/tox.ini +++ b/tox.ini @@ -38,7 +38,7 @@ install_command = basepython = python3 deps = lxml-stubs==0.2.0 - mypy==0.982 + mypy==0.991 types-attrs==19.1.0 types-pyOpenSSL==21.0.0 types-setuptools==57.0.0 @@ -58,7 +58,7 @@ deps = {[testenv]deps} # Twisted[http2] is required to import some files Twisted[http2]>=17.9.0 - flake8==5.0.4 + flake8==6.0.0 commands = flake8 {posargs:docs scrapy tests} @@ -67,7 +67,7 @@ commands = basepython = python3.8 deps = {[testenv:extra-deps]deps} - pylint==2.15.3 + pylint==2.15.6 commands = pylint conftest.py docs extras scrapy setup.py tests From fe60c1224e39aa3d85b20afd54566f135d9de085 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Mon, 5 Dec 2022 15:44:39 +0500 Subject: [PATCH 225/237] Fix the name of the Proxy-Authorization header in release notes (#5744) --- docs/news.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e5fc2971a..c97de0ed8 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,7 @@ New features ~~~~~~~~~~~~ - Relaxed the restriction introduced in 2.6.2 so that the - ``Proxy-Authentication`` header can again be set explicitly, as long as the + ``Proxy-Authorization`` header can again be set explicitly, as long as the proxy URL in the :reqmeta:`proxy` metadata has no other credentials, and for as long as that proxy URL remains the same; this restores compatibility with scrapy-zyte-smartproxy 2.1.0 and older (:issue:`5626`). @@ -281,7 +281,7 @@ Scrapy 2.6.2 (2022-07-25) processes a request with :reqmeta:`proxy` metadata, and that :reqmeta:`proxy` metadata includes proxy credentials, :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets - the ``Proxy-Authentication`` header, but only if that header is not already + the ``Proxy-Authorization`` header, but only if that header is not already set. There are third-party proxy-rotation downloader middlewares that set @@ -294,7 +294,7 @@ Scrapy 2.6.2 (2022-07-25) These third-party proxy-rotation downloader middlewares could change the :reqmeta:`proxy` metadata of a request to a new value, but fail to remove - the ``Proxy-Authentication`` header from the previous value of the + the ``Proxy-Authorization`` header from the previous value of the :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent to a different proxy. @@ -2250,7 +2250,7 @@ Scrapy 1.8.3 (2022-07-25) processes a request with :reqmeta:`proxy` metadata, and that :reqmeta:`proxy` metadata includes proxy credentials, :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets - the ``Proxy-Authentication`` header, but only if that header is not already + the ``Proxy-Authorization`` header, but only if that header is not already set. There are third-party proxy-rotation downloader middlewares that set @@ -2263,7 +2263,7 @@ Scrapy 1.8.3 (2022-07-25) These third-party proxy-rotation downloader middlewares could change the :reqmeta:`proxy` metadata of a request to a new value, but fail to remove - the ``Proxy-Authentication`` header from the previous value of the + the ``Proxy-Authorization`` header from the previous value of the :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent to a different proxy. From 6757973b61766cdb28225733f1c12702fe0886db Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 11 Dec 2022 09:11:55 -0300 Subject: [PATCH 226/237] fix test_follow_whitespace_* tests --- tests/test_http_response.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index f51f3d988..cb65a6a7e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,14 +2,16 @@ import codecs import unittest from unittest import mock +from pkg_resources import parse_version +from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding -from scrapy.http import (Request, Response, TextResponse, HtmlResponse, - XmlResponse, Headers) +from scrapy.exceptions import NotSupported +from scrapy.http import (Headers, HtmlResponse, Request, Response, + TextResponse, XmlResponse) +from scrapy.link import Link from scrapy.selector import Selector from scrapy.utils.python import to_unicode -from scrapy.exceptions import NotSupported -from scrapy.link import Link from tests import get_testdata @@ -180,12 +182,18 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(ValueError, r.follow, None) def test_follow_whitespace_url(self): + target_url = 'http://example.com/foo' + if parse_version(w3lib_version) < parse_version("2.1.1"): + target_url += '%20' self._assert_followed_url('foo ', - 'http://example.com/foo%20') + target_url) def test_follow_whitespace_link(self): + target_url = 'http://example.com/foo' + if parse_version(w3lib_version) < parse_version("2.1.1"): + target_url += '%20' self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + target_url) def test_follow_flags(self): res = self.response_class('http://example.com/') From e92e201b192609259c01b8686bb10f188e40be36 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Mon, 12 Dec 2022 07:55:15 -0300 Subject: [PATCH 227/237] Silence `filelock` logger (#5754) --- scrapy/utils/log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 78e302d19..c8b4936ac 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -46,6 +46,9 @@ DEFAULT_LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'loggers': { + 'filelock': { + 'level': 'ERROR', + }, 'hpack': { 'level': 'ERROR', }, From 0dbd1d9b81399b624d8f8081dbc4ba08e66993e7 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 12 Dec 2022 08:58:40 -0300 Subject: [PATCH 228/237] use xfail, switch to packaging to parse version --- tests/test_http_response.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cb65a6a7e..891b38f6e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -2,7 +2,8 @@ import codecs import unittest from unittest import mock -from pkg_resources import parse_version +from packaging.version import Version as parse_version +from pytest import mark from w3lib import __version__ as w3lib_version from w3lib.encoding import resolve_encoding @@ -181,19 +182,23 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com") self.assertRaises(ValueError, r.follow, None) + @mark.xfail( + parse_version(w3lib_version) >= parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_url(self): - target_url = 'http://example.com/foo' - if parse_version(w3lib_version) < parse_version("2.1.1"): - target_url += '%20' self._assert_followed_url('foo ', - target_url) + 'http://example.com/foo%20') + @mark.xfail( + parse_version(w3lib_version) >= parse_version("2.1.1"), + reason="https://github.com/scrapy/w3lib/pull/207", + strict=True, + ) def test_follow_whitespace_link(self): - target_url = 'http://example.com/foo' - if parse_version(w3lib_version) < parse_version("2.1.1"): - target_url += '%20' self._assert_followed_url(Link('http://example.com/foo '), - target_url) + 'http://example.com/foo%20') def test_follow_flags(self): res = self.response_class('http://example.com/') From e099572cec6bb9aa7e164b3bff9fd8723286fbad Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Mon, 12 Dec 2022 09:56:50 -0300 Subject: [PATCH 229/237] change xfail condition --- tests/test_http_response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 891b38f6e..74e170ec0 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -183,22 +183,22 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(ValueError, r.follow, None) @mark.xfail( - parse_version(w3lib_version) >= parse_version("2.1.1"), + parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) def test_follow_whitespace_url(self): self._assert_followed_url('foo ', - 'http://example.com/foo%20') + 'http://example.com/foo') @mark.xfail( - parse_version(w3lib_version) >= parse_version("2.1.1"), + parse_version(w3lib_version) < parse_version("2.1.1"), reason="https://github.com/scrapy/w3lib/pull/207", strict=True, ) def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') + 'http://example.com/foo') def test_follow_flags(self): res = self.response_class('http://example.com/') From 8b09b0e0d7eb133db648dfd02a8a531219db0a15 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Dec 2022 20:41:48 -0800 Subject: [PATCH 230/237] set asyncio event loop at start of new thread --- scrapy/shell.py | 5 +++++ scrapy/utils/reactor.py | 27 +++++++++++++++++++-------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index f2dff2ae3..515b71bb6 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -21,6 +21,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop class Shell: @@ -76,6 +77,10 @@ class Shell: banner=self.vars.pop('banner', '')) def _schedule(self, request, spider): + if is_asyncio_reactor_installed(): + # set the asyncio event loop for the current thread + event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP'] + set_asyncio_event_loop(event_loop_path) spider = self._open_spider(request, spider) d = _request_deferred(request) d.addCallback(lambda x: (x, spider)) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index ddf354d88..532d7b0d0 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -71,14 +71,7 @@ def install_reactor(reactor_path, event_loop_path=None): reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): - policy = get_asyncio_event_loop_policy() - if event_loop_path is not None: - event_loop_class = load_object(event_loop_path) - event_loop = event_loop_class() - asyncio.set_event_loop(event_loop) - else: - event_loop = policy.get_event_loop() - + event_loop = set_asyncio_event_loop(event_loop_path) asyncioreactor.install(eventloop=event_loop) else: *module, _ = reactor_path.split(".") @@ -87,6 +80,24 @@ def install_reactor(reactor_path, event_loop_path=None): with suppress(error.ReactorAlreadyInstalledError): installer() +def set_asyncio_event_loop(event_loop_path): + """Sets and returns the event loop with specified import path.""" + policy = get_asyncio_event_loop_policy() + if event_loop_path is not None: + event_loop_class = load_object(event_loop_path) + event_loop = event_loop_class() + asyncio.set_event_loop(event_loop) + else: + try: + event_loop = policy.get_event_loop() + except RuntimeError: + # `get_event_loop` is expected to fail when called from a new thread + # with no asyncio event loop yet installed. Such is the case when + # called from `scrapy shell` + event_loop = policy.new_event_loop() + asyncio.set_event_loop(event_loop) + return event_loop + def verify_installed_reactor(reactor_path): """Raises :exc:`Exception` if the installed From c2de9372a27120abf81f2aaf6609e5acae82549e Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 13 Dec 2022 21:37:17 -0800 Subject: [PATCH 231/237] added unittest for scrapy shell command and fixed flake8 error --- scrapy/utils/reactor.py | 1 + tests/test_command_shell.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 532d7b0d0..e6b8de292 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -80,6 +80,7 @@ def install_reactor(reactor_path, event_loop_path=None): with suppress(error.ReactorAlreadyInstalledError): installer() + def set_asyncio_event_loop(event_loop_path): """Sets and returns the event loop with specified import path.""" policy = get_asyncio_event_loop_policy() diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 33c98ad69..f06a02f5f 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -115,3 +115,14 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) + + @defer.inlineCallbacks + def test_shell_fetch_async(self): + reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + url = self.url('/html') + code = f"fetch('{url}')" + args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"] + _, _, err = yield self.execute(args, check_code=True) + self.assertNotIn( + b"RuntimeError: There is no current event loop in thread", err + ) From 96e526aad8ac93966e383a054d151e6561270bc7 Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Thu, 15 Dec 2022 10:01:55 -0300 Subject: [PATCH 232/237] docs: change DOWNLOAD_DELAY description format To be consistent with the other settings' descriptions --- docs/topics/settings.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 40bcda288..c935e74a7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,8 +652,10 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -You can also change this setting per spider by setting ``download_delay`` -spider attribute. +Note + +This delay can be set per spider using :attr:`download_delay` spider attribute. + .. setting:: DOWNLOAD_HANDLERS From b63ca6f834d06ba9b63108079129705f6326354c Mon Sep 17 00:00:00 2001 From: Yohanna Lisnichuk Date: Thu, 15 Dec 2022 13:20:45 -0300 Subject: [PATCH 233/237] docs: use the right notes directive --- docs/topics/settings.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c935e74a7..022265992 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -652,10 +652,9 @@ per ip address instead of per domain. .. _spider-download_delay-attribute: -Note - -This delay can be set per spider using :attr:`download_delay` spider attribute. +.. note:: + This delay can be set per spider using :attr:`download_delay` spider attribute. .. setting:: DOWNLOAD_HANDLERS From 2e54237649bc8301a1c44ab60c6d7908eb9f71cc Mon Sep 17 00:00:00 2001 From: Kyle Reynolds Date: Tue, 20 Dec 2022 17:54:06 -0700 Subject: [PATCH 234/237] Update setup.py adding author_email to match_maintainer email for thoroughness' sake --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 82ac86cdd..a00e86bab 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ setup( description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', + author_email='Pablo Hoffman', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', license='BSD', From 3b5ce4c182d6b243645b9ce27d5f5084dbc4cc88 Mon Sep 17 00:00:00 2001 From: Kyle Reynolds Date: Wed, 21 Dec 2022 10:51:50 -0700 Subject: [PATCH 235/237] Update setup.py Sorry about the goof up. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a00e86bab..27445cad0 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ setup( description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst', encoding="utf-8").read(), author='Scrapy developers', - author_email='Pablo Hoffman', + author_email='pablo@pablohoffman.com', maintainer='Pablo Hoffman', maintainer_email='pablo@pablohoffman.com', license='BSD', From 7ae32ea38d9b78402528ac3dffc8e1c5f1cf86b7 Mon Sep 17 00:00:00 2001 From: Steve Niafas Date: Thu, 22 Dec 2022 15:33:14 +0200 Subject: [PATCH 236/237] Fix transparency for processed webp images (#5767) --- scrapy/pipelines/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 6a28a3b87..d92f24d56 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -171,7 +171,7 @@ class ImagesPipeline(FilesPipeline): 'method called without response_body argument.', category=ScrapyDeprecationWarning, stacklevel=2) - if image.format == 'PNG' and image.mode == 'RGBA': + if image.format in ('PNG', 'WEBP') and image.mode == 'RGBA': background = self._Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') From deaf1fb6cf513db8119acfe70fd3e08f761f7ede Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Sat, 31 Dec 2022 16:11:45 -0300 Subject: [PATCH 237/237] Stop using setup.py (#5776) --- .github/workflows/publish.yml | 4 ++-- tox.ini | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 991b0b6e8..eee9a4f02 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,8 +24,8 @@ jobs: - name: Publish to PyPI if: steps.check-release-tag.outputs.release_tag == 'true' run: | - pip install --upgrade setuptools wheel twine - python setup.py sdist bdist_wheel + pip install --upgrade build twine + python -m build export TWINE_USERNAME=__token__ export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }} twine upload dist/* diff --git a/tox.ini b/tox.ini index 4d0f0291b..0e156d63b 100644 --- a/tox.ini +++ b/tox.ini @@ -75,8 +75,9 @@ commands = basepython = python3 deps = twine==4.0.1 + build==0.9.0 commands = - python setup.py sdist + python -m build --sdist twine check dist/* [pinned]