From 5a55c4269d3389df4e486657caa58985f8e37e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 31 Jan 2019 17:20:29 -0200 Subject: [PATCH 001/125] Adding GCSFeedStorage --- scrapy/extensions/feedexport.py | 26 ++++++++++++++++++ tests/test_feedexport.py | 48 ++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 22ebf3b3f..00d5d8025 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -146,6 +146,32 @@ class S3FeedStorage(BlockingFeedStorage): key.close() + +class GCSFeedStorage(BlockingFeedStorage): + + project_id = None + bucket_name = None + blob_name = None + + def __init__(self, uri, project_id): + self.project_id = project_id + u = urlparse(uri) + self.bucket_name = u.hostname + self.blob_name = u.path[1:] # remove first "/" + + @classmethod + def from_crawler(cls, crawler, uri): + return cls(uri, crawler.settings['GCS_PROJECT_ID']) + + def _store_in_thread(self, file): + file.seek(0) + from google.cloud.storage import Client + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file) + + class FTPFeedStorage(BlockingFeedStorage): def __init__(self, uri): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e46c8c14e..f3c499b3f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -21,7 +21,7 @@ from w3lib.url import path_to_file_uri import scrapy from scrapy.exporters import CsvItemExporter from scrapy.extensions.feedexport import ( - IFeedStorage, FileFeedStorage, FTPFeedStorage, + IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler @@ -187,6 +187,52 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(content, expected_content) +class GCSFeedStorageTest(unittest.TestCase): + + @mock.patch('scrapy.conf.settings', + new={'GCS_PROJECT_ID': 'conf_id' }, create=True) + def test_parse_settings(self): + try: + from google.cloud.storage import Client + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123' } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') + assert storage.project_id == '123' + assert storage.bucket_name == 'mybucket' + assert storage.blob_name == 'export.csv' + + @defer.inlineCallbacks + def test_store(self): + try: + from google.cloud.storage import Client, Bucket, Blob + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + uri = 'gcs://mybucket/export.csv' + project_id = 'myproject-123' + with mock.patch('google.cloud.storage.Client') as m: + client_mock = mock.create_autospec(Client) + m.return_value = client_mock + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id) + yield storage.store(f) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with('mybucket') + bucket_mock.blob.assert_called_once_with('export.csv') + blob_mock.upload_from_file.assert_called_once_with(f) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From a4059851e7b6c8d712b2bc73dbff99be2d569d21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 31 Jan 2019 18:29:15 -0200 Subject: [PATCH 002/125] Refactoring tests --- scrapy/utils/test.py | 16 ++++++++++++++++ tests/test_feedexport.py | 15 +++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..84eae97a7 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -7,6 +7,7 @@ import os from importlib import import_module from twisted.trial.unittest import SkipTest +from tests import mock from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore @@ -91,3 +92,18 @@ def assert_samelines(testcase, text1, text2, msg=None): line endings between platforms """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + +def mock_google_cloud_storage(): + """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob + classes and set their proper return values. + """ + from google.cloud.storage import Client, Bucket, Blob + client_mock = mock.create_autospec(Client) + + bucket_mock = mock.create_autospec(Bucket) + client_mock.get_bucket.return_value = bucket_mock + + blob_mock = mock.create_autospec(Blob) + bucket_mock.blob.return_value = blob_mock + + return (client_mock, bucket_mock, blob_mock) \ No newline at end of file diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f3c499b3f..6d23c68c3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -24,7 +24,8 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) -from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler +from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete, + get_crawler, mock_google_cloud_storage) from scrapy.utils.python import to_native_str @@ -207,22 +208,16 @@ class GCSFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): try: - from google.cloud.storage import Client, Bucket, Blob + from google.cloud.storage import Client except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") uri = 'gcs://mybucket/export.csv' project_id = 'myproject-123' - with mock.patch('google.cloud.storage.Client') as m: - client_mock = mock.create_autospec(Client) + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock - bucket_mock = mock.create_autospec(Bucket) - client_mock.get_bucket.return_value = bucket_mock - - blob_mock = mock.create_autospec(Blob) - bucket_mock.blob.return_value = blob_mock - f = mock.Mock() storage = GCSFeedStorage(uri, project_id) yield storage.store(f) From 1bb6c4154c43d03d00147d810147c4f3c807505e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:04:01 -0200 Subject: [PATCH 003/125] Turning into instance attributes --- scrapy/extensions/feedexport.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 00d5d8025..a81f44045 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,10 +149,6 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - project_id = None - bucket_name = None - blob_name = None - def __init__(self, uri, project_id): self.project_id = project_id u = urlparse(uri) From fc6809b024dc25663432b6c0c5780021a827ea20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:08:54 -0200 Subject: [PATCH 004/125] Add gcs schema to FEED_STORAGES_BASE --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3734a0a58..1a12f35a3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -145,6 +145,7 @@ FEED_STORAGES_BASE = { 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', + 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { From 4a53de165a53433a9a2a8cc4db34e5507c47fbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:09:56 -0200 Subject: [PATCH 005/125] Sorted schemas alphabetically --- scrapy/extensions/feedexport.py | 1 - scrapy/settings/default_settings.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a81f44045..8347b42ca 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -146,7 +146,6 @@ class S3FeedStorage(BlockingFeedStorage): key.close() - class GCSFeedStorage(BlockingFeedStorage): def __init__(self, uri, project_id): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1a12f35a3..8769c01ba 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -142,10 +142,10 @@ FEED_STORAGES = {} FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', - 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', - 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 's3': 'scrapy.extensions.feedexport.S3FeedStorage', + 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { From 2bbbd02bda368c2f052f3aa38f99498a632328bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 09:45:10 -0200 Subject: [PATCH 006/125] Adding an option to set ACL while uploading the blob to GCS --- scrapy/extensions/feedexport.py | 11 ++++++++--- scrapy/settings/default_settings.py | 2 ++ tests/test_feedexport.py | 10 ++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8347b42ca..fbbf9bb97 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage): class GCSFeedStorage(BlockingFeedStorage): - def __init__(self, uri, project_id): + def __init__(self, uri, project_id, acl): self.project_id = project_id + self.acl = acl u = urlparse(uri) self.bucket_name = u.hostname self.blob_name = u.path[1:] # remove first "/" @classmethod def from_crawler(cls, crawler, uri): - return cls(uri, crawler.settings['GCS_PROJECT_ID']) + return cls( + uri, + crawler.settings['GCS_PROJECT_ID'], + crawler.settings['FEED_STORAGE_GCS_ACL'] + ) def _store_in_thread(self, file): file.seek(0) @@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage): client = Client(project=self.project_id) bucket = client.get_bucket(self.bucket_name) blob = bucket.blob(self.blob_name) - blob.upload_from_file(file) + blob.upload_from_file(file, predefined_acl=self.acl) class FTPFeedStorage(BlockingFeedStorage): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8769c01ba..5d2862980 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_GCS_ACL = None + FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 6d23c68c3..5cbca6d28 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase): class GCSFeedStorageTest(unittest.TestCase): @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id' }, create=True) + new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True) def test_parse_settings(self): try: from google.cloud.storage import Client except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') assert storage.project_id == '123' + assert storage.acl == 'publicRead' assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' @@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase): uri = 'gcs://mybucket/export.csv' project_id = 'myproject-123' + acl = 'publicRead' (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock f = mock.Mock() - storage = GCSFeedStorage(uri, project_id) + storage = GCSFeedStorage(uri, project_id, acl) yield storage.store(f) f.seek.assert_called_once_with(0) m.assert_called_once_with(project=project_id) client_mock.get_bucket.assert_called_once_with('mybucket') bucket_mock.blob.assert_called_once_with('export.csv') - blob_mock.upload_from_file.assert_called_once_with(f) + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) class StdoutFeedStorageTest(unittest.TestCase): From cb5f800b0f7029d2bf2e09e9bc1391078c133a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 8 Feb 2019 11:26:33 -0200 Subject: [PATCH 007/125] Adding documentation about Google Cloud Storage Feed Export --- docs/topics/feed-exports.rst | 22 ++++++++++++++++++++++ docs/topics/settings.rst | 18 ++++++++++++++++++ scrapy/settings/default_settings.py | 2 ++ 3 files changed, 42 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index b64dbfbfd..efb63b0ba 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -185,6 +185,27 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` +.. _topics-feed-storage-gcs: + +Google Cloud Storage (GCS) +-------------------------- + +The feeds are stored on `Google Cloud Storage`_. + + * URI scheme: ``gcs`` + * Example URIs: + + * ``gcs://mybucket/path/to/export.csv`` + + * Required external libraries: `google-cloud-storage `_. + +For more information about authentication, please refer to `Google Cloud documentation `_. + +You can set a *Project ID* and *Access Control List (ACL)* through the following settings: + + * :setting:`FEED_STORAGE_GCS_ACL` + * :setting:`GCS_PROJECT_ID` + .. _topics-feed-storage-stdout: Standard output @@ -366,3 +387,4 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto: https://github.com/boto/boto .. _botocore: https://github.com/boto/botocore +.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0ac26a9bd..90ae8fd93 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -749,6 +749,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler temporary files before uploading with :ref:`FTP feed storage ` and :ref:`Amazon S3 `. +.. setting:: FEED_STORAGE_GCS_ACL + +FEED_STORAGE_GCS_ACL +-------------------- + +The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. +For more information on how to set this value, please refer to `Google Cloud documentation `_. + .. setting:: FTP_PASSIVE_MODE FTP_PASSIVE_MODE @@ -786,6 +794,15 @@ Default: ``"anonymous"`` The username to use for FTP connections when there is no ``"ftp_user"`` in ``Request`` meta. +.. setting:: GCS_PROJECT_ID + +GCS_PROJECT_ID +----------------- + +Default: ``None`` + +The Project ID that will be used when storing data on `Google Cloud Storage`_. + .. setting:: ITEM_PIPELINES ITEM_PIPELINES @@ -1371,3 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search +.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5d2862980..c17e94a64 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -168,6 +168,8 @@ FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' FTP_PASSIVE_MODE = True +GCS_PROJECT_ID = None + HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False From 0bb3d8ca93cf68e0ef231ee734f8a1a3c4075a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Wed, 27 Feb 2019 18:41:01 -0300 Subject: [PATCH 008/125] Updating Google Cloud Storage scheme to gs instead of gcs --- docs/topics/feed-exports.rst | 4 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index efb63b0ba..0957a5997 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -192,10 +192,10 @@ Google Cloud Storage (GCS) The feeds are stored on `Google Cloud Storage`_. - * URI scheme: ``gcs`` + * URI scheme: ``gs`` * Example URIs: - * ``gcs://mybucket/path/to/export.csv`` + * ``gs://mybucket/path/to/export.csv`` * Required external libraries: `google-cloud-storage `_. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c17e94a64..50fcd1d0a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -143,7 +143,7 @@ FEED_STORAGES_BASE = { '': 'scrapy.extensions.feedexport.FileFeedStorage', 'file': 'scrapy.extensions.feedexport.FileFeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', - 'gcs': 'scrapy.extensions.feedexport.GCSFeedStorage', + 'gs': 'scrapy.extensions.feedexport.GCSFeedStorage', 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage', } diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 5cbca6d28..41df7d7af 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -200,7 +200,7 @@ class GCSFeedStorageTest(unittest.TestCase): settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } crawler = get_crawler(settings_dict=settings) - storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv') + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.project_id == '123' assert storage.acl == 'publicRead' assert storage.bucket_name == 'mybucket' From 2cb4dc32052c306568206f0997de4b4e53069efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Fri, 22 Mar 2019 09:50:11 -0300 Subject: [PATCH 009/125] Mentioning to use JSON API for ACLs --- docs/topics/settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 90ae8fd93..fcdf31cac 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -755,7 +755,7 @@ FEED_STORAGE_GCS_ACL -------------------- The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage `. -For more information on how to set this value, please refer to `Google Cloud documentation `_. +For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation `_. .. setting:: FTP_PASSIVE_MODE @@ -1388,4 +1388,4 @@ case to see how to enable and use them. .. _Amazon web services: https://aws.amazon.com/ .. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search .. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search -.. _Google Cloud Storage: https://cloud.google.com/storage/ \ No newline at end of file +.. _Google Cloud Storage: https://cloud.google.com/storage/ From 110bc92e6b9c9c3ce1775a9ca1487df42a80d219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= Date: Thu, 29 Aug 2019 11:10:00 -0300 Subject: [PATCH 010/125] Fix default value of FEED_STORAGE_GCS_ACL --- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fbbf9bb97..1e982c684 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -160,7 +160,7 @@ class GCSFeedStorage(BlockingFeedStorage): return cls( uri, crawler.settings['GCS_PROJECT_ID'], - crawler.settings['FEED_STORAGE_GCS_ACL'] + crawler.settings['FEED_STORAGE_GCS_ACL'] or None ) def _store_in_thread(self, file): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 50fcd1d0a..45257a61c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 -FEED_STORAGE_GCS_ACL = None +FEED_STORAGE_GCS_ACL = '' FILES_STORE_S3_ACL = 'private' FILES_STORE_GCS_ACL = '' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 41df7d7af..69f144d07 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -206,6 +206,24 @@ class GCSFeedStorageTest(unittest.TestCase): assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' + @mock.patch('scrapy.conf.settings', + new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True) + def test_parse_empty_acl(self): + try: + from google.cloud.storage import Client + except ImportError: + raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.acl is None + + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None } + crawler = get_crawler(settings_dict=settings) + storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') + assert storage.acl is None + @defer.inlineCallbacks def test_store(self): try: From ad6075440c0285d903ec7238354d093ea300b8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Oct 2019 19:00:03 +0200 Subject: [PATCH 011/125] Fix references to Python types in parameter type fields --- docs/topics/contracts.rst | 2 +- docs/topics/email.rst | 10 ++++----- docs/topics/exporters.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/link-extractors.rst | 14 ++++++------- docs/topics/loaders.rst | 6 +++--- docs/topics/request-response.rst | 36 ++++++++++++++++---------------- scrapy/crawler.py | 2 +- scrapy/robotstxt.py | 4 ++-- scrapy/settings/__init__.py | 33 ++++++++++++----------------- scrapy/signalmanager.py | 2 +- 11 files changed, 53 insertions(+), 60 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 62f9a743b..15443f4cc 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -85,7 +85,7 @@ override three methods: .. class:: Contract(method, \*args) :param method: callback function to which the contract is associated - :type method: function + :type method: collections.abc.Callable :param args: list of arguments passed into the docstring (whitespace separated) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 949cdc638..73b1bdc3b 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -63,10 +63,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :type smtpport: int :param smtptls: enforce using SMTP STARTTLS - :type smtptls: boolean + :type smtptls: bool :param smtpssl: enforce using a secure SSL connection - :type smtpssl: boolean + :type smtpssl: bool .. classmethod:: from_settings(settings) @@ -81,13 +81,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. :param to: the e-mail recipients - :type to: str or list of str + :type to: str or list :param subject: the subject of the e-mail :type subject: str :param cc: the e-mails to CC - :type cc: str or list of str + :type cc: str or list :param body: the e-mail body :type body: str @@ -97,7 +97,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. appear on the e-mail's attachment, ``mimetype`` is the mimetype of the attachment and ``file_object`` is a readable file object with the contents of the attachment - :type attachs: iterable + :type attachs: collections.abc.Iterable :param mimetype: the MIME type of the e-mail :type mimetype: str diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index a698a6a4e..da304922d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -300,7 +300,7 @@ CsvItemExporter :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from :attr:`BaseItemExporter.fields_to_export` or the first exported item fields. - :type include_headers_line: boolean + :type include_headers_line: bool :param join_multivalued: The char (or chars) that will be used for joining multi-valued fields, if found. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 8278e9849..657f1cc61 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. :param ignore: if given, all objects from the specified class (or tuple of classes) will be ignored. - :type ignore: class or classes tuple + :type ignore: type or tuple .. function:: get_oldest(class_name) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 713a94e10..13b9ad7a5 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -59,13 +59,13 @@ LxmlLinkExtractor :param allow: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be extracted. If not given (or empty), it will match all links. - :type allow: a regular expression (or list of) + :type allow: str or list :param deny: a single regular expression (or list of regular expressions) that the (absolute) urls must match in order to be excluded (ie. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. - :type deny: a regular expression (or list of) + :type deny: str or list :param allow_domains: a single value or a list of string containing domains which will be considered for extracting the links @@ -97,7 +97,7 @@ LxmlLinkExtractor that the link's text must match in order to be extracted. If not given (or empty), it will match all links. If a list of regular expressions is given, the link will be extracted if it matches at least one. - :type restrict_text: a regular expression (or list of) + :type restrict_text: str or list :param tags: a tag or a list of tags to consider when extracting links. Defaults to ``('a', 'area')``. @@ -115,11 +115,11 @@ LxmlLinkExtractor different for requests with canonicalized and raw URLs. If you're using LinkExtractor to follow links it is more robust to keep the default ``canonicalize=False``. - :type canonicalize: boolean + :type canonicalize: bool :param unique: whether duplicate filtering should be applied to extracted links. - :type unique: boolean + :type unique: bool :param process_value: a function which receives each value extracted from the tag and attributes scanned and can modify the value and return a @@ -141,7 +141,7 @@ LxmlLinkExtractor if m: return m.group(1) - :type process_value: callable + :type process_value: collections.abc.Callable :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces @@ -150,6 +150,6 @@ LxmlLinkExtractor elements, etc., so LinkExtractor strips space chars by default. Set ``strip=False`` to turn it off (e.g. if you're extracting urls from elements or attributes which allow leading/trailing whitespaces). - :type strip: boolean + :type strip: bool .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 1c2f1da4d..4137fdd24 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -320,7 +320,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the given value using :meth:`~scrapy.utils.misc.extract_regex` method, applied before processors - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -365,7 +365,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected XPath region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: @@ -408,7 +408,7 @@ ItemLoader objects :param re: a regular expression to use for extracting data from the selected CSS region - :type re: str or compiled regex + :type re: str or typing.Pattern Examples:: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 727c67482..2f99a72f5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,7 +31,7 @@ Request objects a :class:`Response`. :param url: the URL of this request - :type url: string + :type url: str :param callback: the function that will be called with the response of this request (once its downloaded) as its first parameter. For more information @@ -40,10 +40,10 @@ Request objects :meth:`~scrapy.spiders.Spider.parse` method will be used. Note that if exceptions are raised during processing, errback is called instead. - :type callback: callable + :type callback: collections.abc.Callable :param method: the HTTP method of this request. Defaults to ``'GET'``. - :type method: string + :type method: str :param meta: the initial values for the :attr:`Request.meta` attribute. If given, the dict passed in this parameter will be shallow copied. @@ -54,7 +54,7 @@ Request objects ``body`` is not given, an empty string is stored. Regardless of the type of this argument, the final value stored will be a ``str`` (never ``unicode`` or ``None``). - :type body: str or unicode + :type body: str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -105,7 +105,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to ``str`` (if given as ``unicode``). - :type encoding: string + :type encoding: str :param priority: the priority of this request (defaults to ``0``). The priority is used by the scheduler to define the order used to process @@ -117,7 +117,7 @@ Request objects the scheduler. This is used when you want to perform an identical request multiple times, to ignore the duplicates filter. Use it with care, or you will get into crawling loops. Default to ``False``. - :type dont_filter: boolean + :type dont_filter: bool :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed @@ -125,7 +125,7 @@ Request objects as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. - :type errback: callable + :type errback: collections.abc.Callable :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -407,7 +407,7 @@ fields with form data from :class:`Response` objects. :param formdata: is a dictionary (or iterable of (key, value) tuples) containing HTML Form data which will be url-encoded and assigned to the body of the request. - :type formdata: dict or iterable of tuples + :type formdata: dict or collections.abc.Iterable The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: @@ -439,20 +439,20 @@ fields with form data from :class:`Response` objects. :type response: :class:`Response` object :param formname: if given, the form with name attribute set to this value will be used. - :type formname: string + :type formname: str :param formid: if given, the form with id attribute set to this value will be used. - :type formid: string + :type formid: str :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: string + :type formxpath: str :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: string + :type formcss: str :param formnumber: the number of form to use, when the response contains multiple forms. The first one (and also the default) is ``0``. - :type formnumber: integer + :type formnumber: int :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is @@ -470,7 +470,7 @@ fields with form data from :class:`Response` objects. :param dont_click: If True, the form data will be submitted without clicking in any element. - :type dont_click: boolean + :type dont_click: bool The other parameters of this class method are passed directly to the :class:`FormRequest` constructor. @@ -558,7 +558,7 @@ dealing with JSON requests. if :attr:`Request.body` argument is provided this parameter will be ignored. if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be set to ``'POST'`` automatically. - :type data: JSON serializable object + :type data: object :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize data into JSON format. @@ -587,10 +587,10 @@ Response objects downloaded (by the Downloader) and fed to the Spiders for processing. :param url: the URL of this response - :type url: string + :type url: str :param status: the HTTP status of the response. Defaults to ``200``. - :type status: integer + :type status: int :param headers: the headers of this response. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). @@ -730,7 +730,7 @@ TextResponse objects body, it will be encoded using this encoding (remember the body attribute is always a string). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. - :type encoding: string + :type encoding: str :class:`TextResponse` objects support the following attributes in addition to the standard :class:`Response` ones: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ded3c082b..84acf543f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -292,7 +292,7 @@ class CrawlerProcess(CrawlerRunner): If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. - :param boolean stop_after_crawl: stop or not the reactor when all + :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished """ if stop_after_crawl: diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..7faad308a 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -43,10 +43,10 @@ class RobotParser(with_metaclass(ABCMeta)): """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. :param url: Absolute URL - :type url: string + :type url: str :param user_agent: User agent - :type user_agent: string + :type user_agent: str """ pass diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28c7940d..95c02021e 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -105,10 +105,9 @@ class BaseSettings(MutableMapping): Get a setting value without affecting its original type. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return self[name] if self[name] is not None else default @@ -123,10 +122,9 @@ class BaseSettings(MutableMapping): ``'0'`` will return ``False`` when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ got = self.get(name, default) try: @@ -145,10 +143,9 @@ class BaseSettings(MutableMapping): Get a setting value as an int. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return int(self.get(name, default)) @@ -157,10 +154,9 @@ class BaseSettings(MutableMapping): Get a setting value as a float. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ return float(self.get(name, default)) @@ -173,10 +169,9 @@ class BaseSettings(MutableMapping): ``'one,two'`` will return a list ['one', 'two'] when using this method. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -194,10 +189,9 @@ class BaseSettings(MutableMapping): and losing all information about priority and mutability. :param name: the setting name - :type name: string + :type name: str :param default: the value to return if no setting is found - :type default: any """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -209,7 +203,7 @@ class BaseSettings(MutableMapping): counterpart. :param name: name of the dictionary-like setting - :type name: string + :type name: str """ compbs = BaseSettings() compbs.update(self[name + '_BASE']) @@ -222,7 +216,7 @@ class BaseSettings(MutableMapping): the given ``name`` does not exist. :param name: the setting name - :type name: string + :type name: str """ if name not in self: return None @@ -252,14 +246,13 @@ class BaseSettings(MutableMapping): otherwise they won't have any effect. :param name: the setting name - :type name: string + :type name: str :param value: the value to associate with the setting - :type value: any :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() priority = get_settings_priority(priority) @@ -283,11 +276,11 @@ class BaseSettings(MutableMapping): uppercase variable of ``module`` with the provided ``priority``. :param module: the module or the path of the module - :type module: module object or string + :type module: types.ModuleType or str :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(module, six.string_types): @@ -316,7 +309,7 @@ class BaseSettings(MutableMapping): :param priority: the priority of the settings. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer - :type priority: string or int + :type priority: str or int """ self._assert_mutability() if isinstance(values, six.string_types): diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..c24b16fcb 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -17,7 +17,7 @@ class SignalManager(object): section. :param receiver: the function to be connected - :type receiver: callable + :type receiver: collections.abc.Callable :param signal: the signal to connect to :type signal: object From 5479e7ecc7d30424dd2f3d9bbfb18abca765be92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:44 +0200 Subject: [PATCH 012/125] Indicate that lists of emails may be provided as a single string or as a list of strings --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 73b1bdc3b..284849c56 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -80,13 +80,13 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Send email to the given recipients. - :param to: the e-mail recipients + :param to: the e-mail recipients as a string or as a list of strings :type to: str or list :param subject: the subject of the e-mail :type subject: str - :param cc: the e-mails to CC + :param cc: the e-mails to CC as a string or as a list of strings :type cc: str or list :param body: the e-mail body From d96b9f860b01256a84bda641b190b423eb4910b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 15:24:59 +0200 Subject: [PATCH 013/125] Use object as type for parameters that allow any value --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 95c02021e..d1a5093a6 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -108,6 +108,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return self[name] if self[name] is not None else default @@ -125,6 +126,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ got = self.get(name, default) try: @@ -146,6 +148,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return int(self.get(name, default)) @@ -157,6 +160,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ return float(self.get(name, default)) @@ -172,6 +176,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or []) if isinstance(value, six.string_types): @@ -192,6 +197,7 @@ class BaseSettings(MutableMapping): :type name: str :param default: the value to return if no setting is found + :type default: object """ value = self.get(name, default or {}) if isinstance(value, six.string_types): @@ -249,6 +255,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting + :type default: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From b9a58798eed39bf543b9682d9ce43b13378a5074 Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 24 May 2014 20:24:01 +0000 Subject: [PATCH 014/125] change Scraper API to call internal `_parse` method A Spider class using internal pre-processing can have first dibs at this and then call a public `parse` method for subclass hooking. --- scrapy/core/scraper.py | 2 +- scrapy/spiders/__init__.py | 3 +++ scrapy/spiders/crawl.py | 2 +- scrapy/spiders/feed.py | 4 ++-- tests/test_spider.py | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 99114d3bb..ad6649134 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -145,7 +145,7 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - callback = request.callback or spider.parse + callback = request.callback or spider._parse warn_on_generator_with_return_value(spider, callback) warn_on_generator_with_return_value(spider, request.errback) dfd.addCallbacks(callback=callback, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9429f6cb2..1011eb870 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -80,6 +80,9 @@ class Spider(object_ref): """ This method is deprecated. """ return Request(url, dont_filter=True) + def _parse(self, response): + return self.parse(response) + def parse(self, response): raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a2c364c0e..e28d17dcd 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -74,7 +74,7 @@ class CrawlSpider(Spider): super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() - def parse(self, response): + def _parse(self, response): return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True) def parse_start_url(self, response): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index c566f0236..11bd17db4 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -61,7 +61,7 @@ class XMLFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def parse(self, response): + def _parse(self, response): if not hasattr(self, 'parse_node'): raise NotConfigured('You must define parse_node method in order to scrape this XML feed') @@ -128,7 +128,7 @@ class CSVFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def parse(self, response): + def _parse(self, response): if not hasattr(self, 'parse_row'): raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') response = self.adapt_response(response) diff --git a/tests/test_spider.py b/tests/test_spider.py index 317a27076..6fbec7e58 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -142,7 +142,7 @@ class XMLFeedSpiderTest(SpiderTest): for iterator in ('iternodes', 'xml'): spider = _XMLSpider('example', iterator=iterator) - output = list(spider.parse(response)) + output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ {'loc': [u'http://www.example.com/Special-Offers.html'], From 5982e3477c732f4dff5accea6eab486e4ab52c3e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 23 Dec 2019 14:12:21 -0300 Subject: [PATCH 015/125] Take keyword arguments in base parsing methods --- docs/topics/spiders.rst | 12 +++++++----- scrapy/spiders/__init__.py | 6 +++--- scrapy/spiders/crawl.py | 11 ++++++++--- scrapy/spiders/feed.py | 4 ++-- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0fb14e24..dd763b607 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -362,12 +362,14 @@ CrawlSpider This spider also exposes an overrideable method: - .. method:: parse_start_url(response) + .. method:: parse_start_url(response, **kwargs) - This method is called for the start_urls responses. It allows to parse - the initial responses and must return either an - :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request` - object, or an iterable containing any of them. + This method is called for each response produced for the URLs in + the spider's ``start_urls`` attribute. It allows to parse + the initial responses and must return either an item + (:class:`scrapy.item.Item` or :class:`dict`), + a :class:`~scrapy.http.Request`, + or an iterable containing any of them. Crawling rules ~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 1011eb870..3e19f1e23 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -80,10 +80,10 @@ class Spider(object_ref): """ This method is deprecated. """ return Request(url, dont_filter=True) - def _parse(self, response): - return self.parse(response) + def _parse(self, response, **kwargs): + return self.parse(response, **kwargs) - def parse(self, response): + def parse(self, response, **kwargs): raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) @classmethod diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index e28d17dcd..4ec0de78c 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -74,10 +74,15 @@ class CrawlSpider(Spider): super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() - def _parse(self, response): - return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True) + def _parse(self, response, **kwargs): + return self._parse_response( + response=response, + callback=self.parse_start_url, + cb_kwargs=kwargs, + follow=True, + ) - def parse_start_url(self, response): + def parse_start_url(self, response, **kwargs): return [] def process_results(self, response, results): diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 11bd17db4..4fa6009a5 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -61,7 +61,7 @@ class XMLFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def _parse(self, response): + def _parse(self, response, **kwargs): if not hasattr(self, 'parse_node'): raise NotConfigured('You must define parse_node method in order to scrape this XML feed') @@ -128,7 +128,7 @@ class CSVFeedSpider(Spider): for result_item in self.process_results(response, ret): yield result_item - def _parse(self, response): + def _parse(self, response, **kwargs): if not hasattr(self, 'parse_row'): raise NotConfigured('You must define parse_row method in order to scrape this CSV feed') response = self.adapt_response(response) From 8d4948f6ca44a76ee7714c8b4b1c46ef73a8845e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 26 Dec 2019 14:38:11 -0300 Subject: [PATCH 016/125] [test] Override CrawlSpider.parse --- tests/spiders.py | 35 +++++++++++++++++++++++++++++------ tests/test_crawl.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 39c8da0b6..dcc475ca7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -186,13 +186,39 @@ class DuplicateStartRequestsSpider(MockServerSpider): self.visited += 1 -class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider): - name = 'crawl_spider_with_errback' +class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): + """ + A CrawlSpider which overrides the 'parse' method + """ + name = 'crawl_spider_with_parse_method' custom_settings = { 'RETRY_HTTP_CODES': [], # no need to retry } rules = ( - Rule(LinkExtractor(), callback='callback', errback='errback', follow=True), + Rule(LinkExtractor(), callback='parse', follow=True), + ) + + def start_requests(self): + test_body = b""" + + Page title<title></head> + <body> + <p><a href="/status?n=200">Item 200</a></p> <!-- callback --> + <p><a href="/status?n=201">Item 201</a></p> <!-- callback --> + </body> + </html> + """ + url = self.mockserver.url("/alpayload") + yield Request(url, method="POST", body=test_body) + + def parse(self, response): + self.logger.info('[parse] status %i', response.status) + + +class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): + name = 'crawl_spider_with_errback' + rules = ( + Rule(LinkExtractor(), callback='parse', errback='errback', follow=True), ) def start_requests(self): @@ -211,8 +237,5 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider): url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def callback(self, response): - self.logger.info('[callback] status %i', response.status) - def errback(self, failure): self.logger.info('[errback] status %i', failure.value.response.status) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f433fcea6..4299e4bbb 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.utils.python import to_unicode from tests.mockserver import MockServer -from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, - SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback) +from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback, + CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider, + DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider) class CrawlTestCase(TestCase): @@ -297,6 +298,27 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + +class CrawlSpiderTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_crawlspider_with_parse(self): + self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver) + + with LogCapture() as log: + yield self.runner.join() + + self.assertIn("[parse] status 200", str(log)) + self.assertIn("[parse] status 201", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver) @@ -304,7 +326,8 @@ with multiples lines with LogCapture() as log: yield self.runner.join() - self.assertIn("[callback] status 200", str(log)) - self.assertIn("[callback] status 201", str(log)) + self.assertIn("[parse] status 200", str(log)) + self.assertIn("[parse] status 201", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + self.assertIn("[errback] status 501", str(log)) From c54df8253a67bb6863a25bf7d667096adc73a040 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 26 Dec 2019 15:12:19 -0300 Subject: [PATCH 017/125] [test] Handle keyword args in CrawlSpider.parse --- tests/spiders.py | 5 +++-- tests/test_crawl.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index dcc475ca7..c042eb7fe 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -211,8 +211,9 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): url = self.mockserver.url("/alpayload") yield Request(url, method="POST", body=test_body) - def parse(self, response): - self.logger.info('[parse] status %i', response.status) + def parse(self, response, foo=None): + self.logger.info('[parse] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4299e4bbb..6247ced35 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -316,8 +316,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield self.runner.join() - self.assertIn("[parse] status 200", str(log)) - self.assertIn("[parse] status 201", str(log)) + self.assertIn("[parse] status 200 (foo: None)", str(log)) + self.assertIn("[parse] status 201 (foo: None)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) @defer.inlineCallbacks def test_crawlspider_with_errback(self): @@ -326,8 +327,9 @@ class CrawlSpiderTestCase(TestCase): with LogCapture() as log: yield self.runner.join() - self.assertIn("[parse] status 200", str(log)) - self.assertIn("[parse] status 201", str(log)) + self.assertIn("[parse] status 200 (foo: None)", str(log)) + self.assertIn("[parse] status 201 (foo: None)", str(log)) + self.assertIn("[parse] status 202 (foo: bar)", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) self.assertIn("[errback] status 501", str(log)) From 8a1dc26d4662ec0c81d22c13cffa44255a7c29ca Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 26 Dec 2019 15:14:47 -0300 Subject: [PATCH 018/125] [doc] Note about the 'parse' method for CrawlSpider/XMLFeedSpider --- docs/topics/spiders.rst | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index dd763b607..406f50fb3 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -392,11 +392,6 @@ Crawling rules object will contain the text of the link that produced the :class:`~scrapy.http.Request` in its ``meta`` dictionary (under the ``link_text`` key) - .. warning:: When writing crawl spider rules, avoid using ``parse`` as - callback, since the :class:`CrawlSpider` uses the ``parse`` method - itself to implement its logic. So if you override the ``parse`` method, - the crawl spider will no longer work. - ``cb_kwargs`` is a dict containing the keyword arguments to be passed to the callback function. @@ -422,6 +417,12 @@ Crawling rules It receives a :class:`Twisted Failure <twisted.python.failure.Failure>` instance as first parameter. + +.. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`CrawlSpider`-based spiders; + unexpected behaviour can occur otherwise. + + CrawlSpider example ~~~~~~~~~~~~~~~~~~~ @@ -452,6 +453,11 @@ Let's now take a look at an example CrawlSpider with rules:: item['name'] = response.xpath('//td[@id="item_name"]/text()').get() item['description'] = response.xpath('//td[@id="item_description"]/text()').get() item['link_text'] = response.meta['link_text'] + url = response.xpath('//td[@id="additional_data"]/@href').get() + return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item)) + + def parse_additional_page(self, response, item): + item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get() return item @@ -545,6 +551,11 @@ XMLFeedSpider those results. It must return a list of results (Items or Requests). +.. warning:: Because of its internal implementation, you must explicitly set + callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders; + unexpected behaviour can occur otherwise. + + XMLFeedSpider example ~~~~~~~~~~~~~~~~~~~~~ From 0641ba0faa97498ca5bee39c4e8faec58d5f0522 Mon Sep 17 00:00:00 2001 From: faizan2700 <syedfaizan824@gmail.com> Date: Sun, 2 Feb 2020 16:54:22 +0530 Subject: [PATCH 019/125] SCRAPY_CHECK will be set while running contact --- scrapy/commands/check.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 9d4437a47..09a76ca7a 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -78,19 +78,19 @@ class Command(ScrapyCommand): elif tested_methods: self.crawler_process.crawl(spidercls) - # start checks - if opts.list: - for spider, methods in sorted(contract_reqs.items()): - if not methods and not opts.verbose: - continue - print(spider) - for method in sorted(methods): - print(' * %s' % method) - else: - start = time.time() - self.crawler_process.start() - stop = time.time() + # start checks + if opts.list: + for spider, methods in sorted(contract_reqs.items()): + if not methods and not opts.verbose: + continue + print(spider) + for method in sorted(methods): + print(' * %s' % method) + else: + start = time.time() + self.crawler_process.start() + stop = time.time() - result.printErrors() - result.printSummary(start, stop) - self.exitcode = int(not result.wasSuccessful()) + result.printErrors() + result.printSummary(start, stop) + self.exitcode = int(not result.wasSuccessful()) From 7f2d3051feb0bc8f868a3eea8310e2fc8c461287 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 6 Feb 2020 18:19:40 -0300 Subject: [PATCH 020/125] Fix Flake8 issue --- tests/test_crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f1d502ff7..225fe7a0e 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -14,8 +14,8 @@ from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioReturnSpider, AsyncDefAsyncioSpider, - AsyncDefSpider, - BrokenStartRequestsSpider, + AsyncDefSpider, + BrokenStartRequestsSpider, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, From 7025c18b159f1ce14b5732dc7d262efbdc72cf5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 10 Feb 2020 19:43:23 +0100 Subject: [PATCH 021/125] Clear line of spaces --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9f837221e..8dc08710f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -184,7 +184,7 @@ passed through the following settings: * :setting:`AWS_ACCESS_KEY_ID` * :setting:`AWS_SECRET_ACCESS_KEY` - + You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` From 73e88d036c72aa628af50371aace4cfce7e286e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 12 Feb 2020 17:17:38 +0100 Subject: [PATCH 022/125] Import mock from unittest --- scrapy/utils/test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4ff6d73a5..00002c303 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,14 +2,13 @@ This module contains some assorted functions used in tests """ -from __future__ import absolute_import -from posixpath import split import asyncio import os +from posixpath import split +from unittest import mock from importlib import import_module from twisted.trial.unittest import SkipTest -from tests import mock from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore From e1be078eaa17a8df72716932fde07e225f79745f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 12 Feb 2020 17:38:06 +0100 Subject: [PATCH 023/125] Fix Flake8-reported issues --- scrapy/utils/test.py | 2 ++ tests/test_feedexport.py | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 00002c303..7442a2f33 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -120,12 +120,14 @@ def assert_samelines(testcase, text1, text2, msg=None): """ testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + def get_from_asyncio_queue(value): q = asyncio.Queue() getter = q.get() q.put_nowait(value) return getter + def mock_google_cloud_storage(): """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob classes and set their proper return values. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d97b199fe..2b299503b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -25,9 +25,13 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage, S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) -from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete, - get_crawler, mock_google_cloud_storage) from scrapy.utils.python import to_unicode +from scrapy.utils.test import ( + assert_aws_environ, + get_s3_content_and_delete, + get_crawler, + mock_google_cloud_storage, +) class FileFeedStorageTest(unittest.TestCase): @@ -362,15 +366,13 @@ class S3FeedStorageTest(unittest.TestCase): class GCSFeedStorageTest(unittest.TestCase): - @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True) def test_parse_settings(self): try: - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.project_id == '123' @@ -378,20 +380,18 @@ class GCSFeedStorageTest(unittest.TestCase): assert storage.bucket_name == 'mybucket' assert storage.blob_name == 'export.csv' - @mock.patch('scrapy.conf.settings', - new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True) def test_parse_empty_acl(self): try: - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.acl is None - settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None } + settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None} crawler = get_crawler(settings_dict=settings) storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv') assert storage.acl is None @@ -399,7 +399,7 @@ class GCSFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): try: - from google.cloud.storage import Client + from google.cloud.storage import Client # noqa except ImportError: raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") @@ -407,7 +407,7 @@ class GCSFeedStorageTest(unittest.TestCase): project_id = 'myproject-123' acl = 'publicRead' (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() - with mock.patch('google.cloud.storage.Client') as m: + with mock.patch('google.cloud.storage.Client') as m: m.return_value = client_mock f = mock.Mock() @@ -420,6 +420,7 @@ class GCSFeedStorageTest(unittest.TestCase): bucket_mock.blob.assert_called_once_with('export.csv') blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks From a175b6efc319ecbfd03b19705c25bd8bd44ca339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 27 Mar 2020 02:10:10 -0300 Subject: [PATCH 024/125] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..9bc324c32 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,32 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + + +pool: + vmImage: 'windows-2019' +strategy: + matrix: + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From 02206e5ffe74fe8107c272ad35920928544323b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 27 Mar 2020 02:20:39 -0300 Subject: [PATCH 025/125] Run tox --- azure-pipelines.yml | 9 ++++++--- tests/requirements-py3.txt | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9bc324c32..489cfe53b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,10 +10,13 @@ strategy: matrix: Python35: python.version: '3.5' + TOXENV: py35 Python36: python.version: '3.6' + TOXENV: py36 Python37: python.version: '3.7' + TOXENV: py37 steps: - task: UsePythonVersion@0 @@ -27,6 +30,6 @@ steps: displayName: 'Install dependencies' - script: | - pip install pytest pytest-azurepipelines - pytest - displayName: 'pytest' + pip install -U tox twine wheel codecov + tox + displayName: 'Run test suite' diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..8896f4614 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,6 +6,7 @@ pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist +pytest-azurepipelines sybil testfixtures From 0699e6bb1600ff943e131a3b7a299aa038145198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com> Date: Fri, 27 Mar 2020 02:22:05 -0300 Subject: [PATCH 026/125] no need to install requirements.txt --- azure-pipelines.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 489cfe53b..ffc4d549b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,11 +24,6 @@ steps: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' -- script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' - - script: | pip install -U tox twine wheel codecov tox From 3fb0027138ab44b16be6e11677626d69b8c90c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Sat, 28 Mar 2020 17:36:50 +0100 Subject: [PATCH 027/125] =?UTF-8?q?Require=20sybil=20=E2=89=A5=201.3.0=20f?= =?UTF-8?q?or=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..e7c86e0e9 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -6,7 +6,7 @@ pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist -sybil +sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures # optional for shell wrapper tests From 2f510fd47d8d10217f7b18b531205dd8c252eaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 15 Apr 2020 21:10:05 +0200 Subject: [PATCH 028/125] Fix ShellTest.test_local_file on Windows --- scrapy/utils/url.py | 13 +++++++------ tests/test_command_shell.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c9abb12d5..c29ed4461 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -85,11 +85,9 @@ def add_http_if_no_scheme(url): def guess_scheme(url): - """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - parts = urlparse(url) - if parts.scheme: - return url - # Note: this does not match Windows filepath + """Add an URL scheme if missing: file:// for filepath-like input or + http:// otherwise.""" + # POSIX path if re.match(r'''^ # start with... ( \. # ...a single dot, @@ -99,7 +97,10 @@ def guess_scheme(url): )? # optional match of ".", ".." or ".blabla" / # at least one "/" for a file path, . # and something after the "/" - ''', parts.path, flags=re.VERBOSE): + ''', url, flags=re.VERBOSE): + return any_to_uri(url) + # Windows drive-letter path + elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE): return any_to_uri(url) else: return add_http_if_no_scheme(url) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..acf8e9f71 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -94,7 +94,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_file(self): - filepath = join(tests_datadir, 'test_site/index.html') + filepath = join(tests_datadir, 'test_site', 'index.html') _, out, _ = yield self.execute([filepath, '-c', 'item']) assert b'{}' in out From e5b23f4b00962df76d8302ebf869ef4a4319e142 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 18 Mar 2020 11:26:59 +0300 Subject: [PATCH 029/125] fix #4250: add batch deliveries --- scrapy/extensions/feedexport.py | 54 +++++++++++++++++++++-------- scrapy/settings/default_settings.py | 1 + 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 998d2a5d1..906f99fee 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -241,6 +241,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') + self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -250,19 +251,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): uri = uri % self._get_uri_params(spider, feed['uri_params']) - storage = self._get_storage(uri) - file = storage.open(spider) - exporter = self._get_exporter( - file=file, - format=feed['format'], - fields_to_export=feed['fields'], - encoding=feed['encoding'], - indent=feed['indent'], - ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) - self.slots.append(slot) - if slot.store_empty: - slot.start_exporting() + self.slots.append(self._start_new_batch(None, uri, feed, spider)) def close_spider(self, spider): deferred_list = [] @@ -285,11 +274,48 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None + def _start_new_batch(self, previous_batch_slot, uri, feed, spider): + """ + Redirect the output data stream to a new file. + Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + """ + if previous_batch_slot is not None: + previous_batch_slot.exporter.finish_exporting() + previous_batch_slot.storage.store(previous_batch_slot.file) + storage = self._get_storage(uri) + file = storage.open(spider) + exporter = self._get_exporter( + file=file, + format=feed['format'], + fields_to_export=feed['fields'], + encoding=feed['encoding'], + indent=feed['indent'] + ) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + if slot.store_empty: + slot.start_exporting() + return slot + + def _get_uri_of_partial(self, slot, feed, spider): + """Get uri for each partial using datetime.now().isoformat()""" + uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.' + uri = uri + datetime.now().isoformat() + '.' + feed['format'] + return uri + def item_scraped(self, item, spider): - for slot in self.slots: + slots = [] + for idx, slot in enumerate(self.slots): slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 + if self.storage_batch and slot.itemcount % self.storage_batch == 0: + uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider) + slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) + self.feeds[uri] = self.feeds[slot.uri] + self.feeds.pop(slot.uri) + self.slots[idx] = None + self.slots = [slot for slot in self.slots if slot is not None] + self.slots.extend(slots) def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 077317c81..690e044c5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,6 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } +FEED_STORAGE_BATCH = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From 8b4566ff93843cdf17ada069dc09261a99971d26 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 18 Mar 2020 14:21:21 +0300 Subject: [PATCH 030/125] fix wrong name of first file in partial deliveries --- scrapy/extensions/feedexport.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 906f99fee..4f7c6bf07 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -249,6 +249,8 @@ class FeedExporter: raise NotConfigured def open_spider(self, spider): + if self.storage_batch: + self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()} for uri, feed in self.feeds.items(): uri = uri % self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch(None, uri, feed, spider)) @@ -296,11 +298,11 @@ class FeedExporter: slot.start_exporting() return slot - def _get_uri_of_partial(self, slot, feed, spider): + def _get_uri_of_partial(self, template_uri, feed, spider): """Get uri for each partial using datetime.now().isoformat()""" - uri = (slot.uri % self._get_uri_params(spider, feed['uri_params'])).split('.')[0] + '.' - uri = uri + datetime.now().isoformat() + '.' + feed['format'] - return uri + template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params'])) + uri_name = template_uri.split('.')[0] + return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"]) def item_scraped(self, item, spider): slots = [] @@ -309,11 +311,12 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 if self.storage_batch and slot.itemcount % self.storage_batch == 0: - uri = self._get_uri_of_partial(slot, self.feeds[slot.uri], spider) + uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider) slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) self.feeds[uri] = self.feeds[slot.uri] self.feeds.pop(slot.uri) self.slots[idx] = None + self.slots = [slot for slot in self.slots if slot is not None] self.slots.extend(slots) From 0723e3f4f9777a87d0df3b2e2fddfeac9099dd3b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 19 Mar 2020 21:17:02 +0300 Subject: [PATCH 031/125] add batch_id, add error if uri is specified incorrectly --- scrapy/extensions/feedexport.py | 73 ++++++++++++++++++++--------- scrapy/settings/default_settings.py | 2 +- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4f7c6bf07..38b25bf4a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -180,14 +180,16 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri): self.file = file self.exporter = exporter self.storage = storage # feed params - self.uri = uri + self.batch_id = batch_id self.format = format self.store_empty = store_empty + self.template_uri = template_uri + self.uri = uri # flags self.itemcount = 0 self._exporting = False @@ -241,19 +243,28 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch = self.settings.getint('FEED_STORAGE_BATCH') + self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured + if not self._batch_deliveries_supported(uri): + raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured def open_spider(self, spider): - if self.storage_batch: - self.feeds = {self._get_uri_of_partial(uri, feed, spider): feed for uri, feed in self.feeds.items()} for uri, feed in self.feeds.items(): - uri = uri % self._get_uri_params(spider, feed['uri_params']) - self.slots.append(self._start_new_batch(None, uri, feed, spider)) + batch_id = 1 + uri_params = self._get_uri_params(spider, feed['uri_params']) + uri_params['batch_id'] = batch_id + self.slots.append(self._start_new_batch( + previous_batch_slot=None, + uri=uri % uri_params, + feed=feed, + spider=spider, + batch_id=batch_id, + template_uri=uri + )) def close_spider(self, spider): deferred_list = [] @@ -276,10 +287,17 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider): + def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store + to get the file properly closed. + :param uri: uri of the new batch to start + :param feed: dict with parameters of feed + :param spider: user spider + :param batch_id: sequential batch id starting at 1 + :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: previous_batch_slot.exporter.finish_exporting() @@ -293,30 +311,30 @@ class FeedExporter: encoding=feed['encoding'], indent=feed['indent'] ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri) if slot.store_empty: slot.start_exporting() return slot - def _get_uri_of_partial(self, template_uri, feed, spider): - """Get uri for each partial using datetime.now().isoformat()""" - template_uri = (template_uri % self._get_uri_params(spider, feed['uri_params'])) - uri_name = template_uri.split('.')[0] - return '{}.{}.{}'.format(uri_name, datetime.now().isoformat(), feed["format"]) - def item_scraped(self, item, spider): slots = [] for idx, slot in enumerate(self.slots): slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - if self.storage_batch and slot.itemcount % self.storage_batch == 0: - uri = self._get_uri_of_partial(slot.uri, self.feeds[slot.uri], spider) - slots.append(self._start_new_batch(slot, uri, self.feeds[slot.uri], spider)) - self.feeds[uri] = self.feeds[slot.uri] - self.feeds.pop(slot.uri) + if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: + batch_id = slot.batch_id + 1 + uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) + uri_params['batch_id'] = batch_id + self.slots.append(self._start_new_batch( + previous_batch_slot=slot, + uri=slot.template_uri % uri_params, + feed=self.feeds[slot.template_uri], + spider=spider, + batch_id=batch_id, + template_uri=slot.template_uri + )) self.slots[idx] = None - self.slots = [slot for slot in self.slots if slot is not None] self.slots.extend(slots) @@ -335,6 +353,17 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) + def _batch_deliveries_supported(self, uri): + """ + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s + to distinguish different files of partial output + """ + if not self.storage_batch_size: + return True + if '%(time)s' in uri or '%(batch_id)s' in uri: + return True + logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + def _storage_supported(self, uri): scheme = urlparse(uri).scheme if scheme in self.storages: @@ -364,7 +393,7 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') + ts = datetime.utcnow().isoformat().replace(':', '-') params['time'] = ts uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 690e044c5..7f90a2280 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH = 0 +FEED_STORAGE_BATCH_SIZE = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From d11411b402ae68874c6ccc2883836be0b9cf8326 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 21 Mar 2020 10:48:13 +0300 Subject: [PATCH 032/125] fix comments --- scrapy/extensions/feedexport.py | 31 ++++++++++++++++++----------- scrapy/settings/default_settings.py | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 38b25bf4a..ab0a0de37 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) @@ -243,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.getint('FEED_STORAGE_BATCH_SIZE') + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -263,7 +262,7 @@ class FeedExporter: feed=feed, spider=spider, batch_id=batch_id, - template_uri=uri + template_uri=uri, )) def close_spider(self, spider): @@ -290,7 +289,7 @@ class FeedExporter: def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is greater than zero. + Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store to get the file properly closed. :param uri: uri of the new batch to start @@ -309,9 +308,18 @@ class FeedExporter: format=feed['format'], fields_to_export=feed['fields'], encoding=feed['encoding'], - indent=feed['indent'] + indent=feed['indent'], + ) + slot = _FeedSlot( + file=file, + exporter=exporter, + storage=storage, + uri=uri, + format=feed['format'], + store_empty=feed['store_empty'], + batch_id=batch_id, + template_uri=template_uri, ) - slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'], batch_id, template_uri) if slot.store_empty: slot.start_exporting() return slot @@ -326,13 +334,13 @@ class FeedExporter: batch_id = slot.batch_id + 1 uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) uri_params['batch_id'] = batch_id - self.slots.append(self._start_new_batch( + slots.append(self._start_new_batch( previous_batch_slot=slot, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, batch_id=batch_id, - template_uri=slot.template_uri + template_uri=slot.template_uri, )) self.slots[idx] = None self.slots = [slot for slot in self.slots if slot is not None] @@ -358,11 +366,10 @@ class FeedExporter: If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_size: + if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri: return True - if '%(time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + return False def _storage_supported(self, uri): scheme = urlparse(uri).scheme diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7f90a2280..c3463a505 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = 0 +FEED_STORAGE_BATCH_SIZE = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', From 39d0d13d3f7bd671d5b29646b209c62e23373fab Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 26 Mar 2020 14:18:35 +0300 Subject: [PATCH 033/125] Add partial deliveries tests --- tests/test_feedexport.py | 195 +++++++++++++++++++++++++++++++-------- 1 file changed, 159 insertions(+), 36 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c5589e52f..1ebe44e12 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -6,6 +6,7 @@ import shutil import string import tempfile import warnings +from abc import ABC, abstractmethod from io import BytesIO from pathlib import Path from string import ascii_letters, digits @@ -21,8 +22,9 @@ from zope.interface.verify import verifyObject import scrapy from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.exporters import CsvItemExporter -from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage, +from scrapy.extensions.feedexport import (BlockingFeedStorage, FeedExporter, FileFeedStorage, FTPFeedStorage, IFeedStorage, S3FeedStorage, StdoutFeedStorage) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -76,6 +78,7 @@ class FTPFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -129,6 +132,7 @@ class BlockingFeedStorageTest(unittest.TestCase): def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): name = 'test_spider' + crawler = get_crawler(settings_dict=settings) spider = TestSpider.from_crawler(crawler) return spider @@ -390,23 +394,63 @@ class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin): pass -class FeedExportTest(unittest.TestCase): +class FeedExportTestBase(ABC, unittest.TestCase): + __test__ = False class MyItem(scrapy.Item): foo = scrapy.Field() egg = scrapy.Field() baz = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + return os.path.join(self.temp_dir, inter_dir, filename) + def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - def _random_temp_filename(self): - chars = [random.choice(ascii_letters + digits) for _ in range(15)] - filename = ''.join(chars) - return os.path.join(self.temp_dir, filename) + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @defer.inlineCallbacks + def exported_no_data(self, settings): + """ + Return exported data which a spider yielding no ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + pass + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + + @abstractmethod + def run_and_export(self, spider_cls, settings): + pass + + +class FeedExportTest(FeedExportTestBase): + __test__ = True @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -417,7 +461,6 @@ class FeedExportTest(unittest.TestCase): urljoin('file:', pathname2url(str(file_path))): feed for file_path, feed in FEEDS.items() } - content = {} try: with MockServer() as s: @@ -435,35 +478,6 @@ class FeedExportTest(unittest.TestCase): defer.returnValue(content) - @defer.inlineCallbacks - def exported_data(self, items, settings): - """ - Return exported data which a spider yielding ``items`` would return. - """ - class TestSpider(scrapy.Spider): - name = 'testspider' - - def parse(self, response): - for item in items: - yield item - - data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) - - @defer.inlineCallbacks - def exported_no_data(self, settings): - """ - Return exported data which a spider yielding no ``items`` would return. - """ - class TestSpider(scrapy.Spider): - name = 'testspider' - - def parse(self, response): - pass - - data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) - @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -970,3 +984,112 @@ class FeedExportTest(unittest.TestCase): } data = yield self.exported_no_data(settings) self.assertEqual(data['csv'], b'') + + +class PartialDeliveriesTest(FeedExportTestBase): + __test__ = True + _file_mark = '_%(time)s_#%(batch_id)s' + + @defer.inlineCallbacks + def run_and_export(self, spider_cls, settings): + """ Run spider with specified settings; return exported data. """ + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + urljoin('file:', file_path): feed + for file_path, feed in FEEDS.items() + } + from collections import defaultdict + content = defaultdict(list) + try: + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + spider_cls.start_urls = [s.url('/')] + yield runner.crawl(spider_cls) + + for path, feed in FEEDS.items(): + dir_name = os.path.dirname(path) + for file in sorted(os.listdir(dir_name)): + with open(os.path.join(dir_name, file), 'rb') as f: + data = f.read() + content[feed['format']].append(data) + finally: + pass + defer.returnValue(content) + + @defer.inlineCallbacks + def assertPartialExported(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, + }, + }) + data = yield self.exported_data(items, settings) + data['jl'] = b''.join(data['jl']) + parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] + + rows = [{k: v for k, v in row.items() if v} for row in rows] + self.assertEqual(rows, parsed) + + @defer.inlineCallbacks + def test_partial_deliveries(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + rows = [ + {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, + {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, + {'foo': 'bar3', 'baz': 'quux3'} + ] + settings = { + 'FEED_STORAGE_BATCH_SIZE': 1 + } + yield self.assertPartialExported(items, rows, settings=settings) + + def test_wrong_path(self): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': 'xml'}, + }, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + crawler = get_crawler(settings_dict=settings) + self.assertRaises(NotConfigured, FeedExporter, crawler) + + @defer.inlineCallbacks + def test_export_no_items_not_store_empty(self): + for fmt in ('json', 'jsonlines', 'xml', 'csv'): + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + data = yield self.exported_no_data(settings) + data[fmt] = b''.join(data[fmt]) + self.assertEqual(data[fmt], b'') + + @defer.inlineCallbacks + def test_export_no_items_store_empty(self): + formats = ( + ('json', b'[]'), + ('jsonlines', b''), + ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'), + ('csv', b''), + ) + + for fmt, expctd in formats: + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, + }, + 'FEED_STORE_EMPTY': True, + 'FEED_EXPORT_INDENT': None, + 'FEED_STORAGE_BATCH_SIZE': 1 + } + data = yield self.exported_no_data(settings) + data[fmt] = b''.join(data[fmt]) + self.assertEqual(data[fmt], expctd) From ffa8a533e74478a5c81fbf453f2c65601bb1d244 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 28 Mar 2020 11:40:16 +0300 Subject: [PATCH 034/125] Set batch_id in _get_uri_params --- scrapy/extensions/feedexport.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ab0a0de37..06ea6c5b2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -253,15 +253,12 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - batch_id = 1 - uri_params = self._get_uri_params(spider, feed['uri_params']) - uri_params['batch_id'] = batch_id + uri_params = self._get_uri_params(spider, feed['uri_params'], None) self.slots.append(self._start_new_batch( previous_batch_slot=None, uri=uri % uri_params, feed=feed, spider=spider, - batch_id=batch_id, template_uri=uri, )) @@ -286,7 +283,7 @@ class FeedExporter: deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider, batch_id, template_uri): + def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. @@ -295,12 +292,15 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param batch_id: sequential batch id starting at 1 :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: + previous_batch_id = previous_batch_slot.batch_id previous_batch_slot.exporter.finish_exporting() previous_batch_slot.storage.store(previous_batch_slot.file) + else: + previous_batch_id = 0 + storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter( @@ -317,7 +317,7 @@ class FeedExporter: uri=uri, format=feed['format'], store_empty=feed['store_empty'], - batch_id=batch_id, + batch_id=previous_batch_id + 1, template_uri=template_uri, ) if slot.store_empty: @@ -331,15 +331,12 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: - batch_id = slot.batch_id + 1 - uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params']) - uri_params['batch_id'] = batch_id + uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) slots.append(self._start_new_batch( previous_batch_slot=slot, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, - batch_id=batch_id, template_uri=slot.template_uri, )) self.slots[idx] = None @@ -396,12 +393,12 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params): + def _get_uri_params(self, spider, uri_params, slot): params = {} for k in dir(spider): params[k] = getattr(spider, k) - ts = datetime.utcnow().isoformat().replace(':', '-') - params['time'] = ts + params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 + params['time'] = datetime.utcnow().isoformat().replace(':', '-') uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params From 963580463b96315eb58319e6d35b4cd52672371a Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 15 Apr 2020 20:14:33 +0300 Subject: [PATCH 035/125] Update tests --- tests/test_feedexport.py | 203 ++++++++++++++++++++++++++++++--------- 1 file changed, 159 insertions(+), 44 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1ebe44e12..c6cd867b1 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ import string import tempfile import warnings from abc import ABC, abstractmethod +from collections import defaultdict from io import BytesIO from pathlib import Path from string import ascii_letters, digits @@ -444,10 +445,31 @@ class FeedExportTestBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) defer.returnValue(data) + @defer.inlineCallbacks + def assertExported(self, items, header, rows, settings=None, ordered=True): + yield self.assertExportedCsv(items, header, rows, settings, ordered) + yield self.assertExportedJsonLines(items, rows, settings) + yield self.assertExportedXml(items, rows, settings) + yield self.assertExportedPickle(items, rows, settings) + yield self.assertExportedMarshal(items, rows, settings) + yield self.assertExportedMultiple(items, rows, settings) + @abstractmethod def run_and_export(self, spider_cls, settings): pass + def _load_until_eof(self, data, load_func): + result = [] + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break + return result + class FeedExportTest(FeedExportTestBase): __test__ = True @@ -478,6 +500,22 @@ class FeedExportTest(FeedExportTestBase): defer.returnValue(content) + @defer.inlineCallbacks + def exported_data(self, items, settings): + """ + Return exported data which a spider yielding ``items`` would return. + """ + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + data = yield self.run_and_export(TestSpider, settings) + defer.returnValue(data) + @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} @@ -543,18 +581,6 @@ class FeedExportTest(FeedExportTestBase): json_rows = json.loads(to_unicode(data['json'])) self.assertEqual(rows, json_rows) - def _load_until_eof(self, data, load_func): - result = [] - with tempfile.TemporaryFile() as temp: - temp.write(data) - temp.seek(0) - while True: - try: - result.append(load_func(temp)) - except EOFError: - break - return result - @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} @@ -583,15 +609,6 @@ class FeedExportTest(FeedExportTestBase): result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) - @defer.inlineCallbacks - def assertExported(self, items, header, rows, settings=None, ordered=True): - yield self.assertExportedCsv(items, header, rows, settings, ordered) - yield self.assertExportedJsonLines(items, rows, settings) - yield self.assertExportedXml(items, rows, settings) - yield self.assertExportedPickle(items, rows, settings) - yield self.assertExportedMarshal(items, rows, settings) - yield self.assertExportedMultiple(items, rows, settings) - @defer.inlineCallbacks def test_export_items(self): # feed exporters use field names from Item @@ -615,7 +632,7 @@ class FeedExportTest(FeedExportTestBase): }, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], b'') + self.assertEqual(b'', data[fmt]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -635,7 +652,7 @@ class FeedExportTest(FeedExportTestBase): 'FEED_EXPORT_INDENT': None, } data = yield self.exported_no_data(settings) - self.assertEqual(data[fmt], expctd) + self.assertEqual(expctd, data[fmt]) @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -734,7 +751,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'), + 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( + 'utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } @@ -751,7 +769,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( + 'latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } @@ -772,7 +791,8 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode( + 'latin-1'), 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } @@ -988,7 +1008,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time)s_#%(batch_id)s' + _file_mark = '_%(time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -999,7 +1019,6 @@ class PartialDeliveriesTest(FeedExportTestBase): urljoin('file:', file_path): feed for file_path, feed in FEEDS.items() } - from collections import defaultdict content = defaultdict(list) try: with MockServer() as s: @@ -1014,26 +1033,120 @@ class PartialDeliveriesTest(FeedExportTestBase): data = f.read() content[feed['format']].append(data) finally: - pass + self.tearDown() defer.returnValue(content) @defer.inlineCallbacks - def assertPartialExported(self, items, rows, settings=None): + def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update({ 'FEEDS': { os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - data = yield self.exported_data(items, settings) - data['jl'] = b''.join(data['jl']) - parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] - + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] rows = [{k: v for k, v in row.items() if v} for row in rows] - self.assertEqual(rows, parsed) + data = yield self.exported_data(items, settings) + for batch in data['jl']: + got_batch = [json.loads(to_unicode(batch_item)) for batch_item in batch.splitlines()] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) @defer.inlineCallbacks - def test_partial_deliveries(self): + def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + data = yield self.exported_data(items, settings) + for batch in data['csv']: + got_batch = csv.DictReader(to_unicode(batch).splitlines()) + self.assertEqual(list(header), got_batch.fieldnames) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, list(got_batch)) + + @defer.inlineCallbacks + def assertExportedXml(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, + os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + # XML + xml_rows = rows.copy() + for batch in data['xml']: + root = lxml.etree.fromstring(batch) + got_batch = [{e.tag: e.text for e in it} for it in root.findall('item')] + expected_batch, xml_rows = xml_rows[:batch_size], xml_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + # JSON + json_rows = rows.copy() + for batch in data['json']: + got_batch = json.loads(batch) + expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedPickle(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import pickle + for batch in data['pickle']: + got_batch = self._load_until_eof(batch, load_func=pickle.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def assertExportedMarshal(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, + }, + }) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + rows = [{k: v for k, v in row.items() if v} for row in rows] + data = yield self.exported_data(items, settings) + import marshal + for batch in data['marshal']: + got_batch = self._load_until_eof(batch, load_func=marshal.load) + expected_batch, rows = rows[:batch_size], rows[batch_size:] + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_export_items(self): items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1042,14 +1155,16 @@ class PartialDeliveriesTest(FeedExportTestBase): rows = [ {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}, - {'foo': 'bar3', 'baz': 'quux3'} + {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_SIZE': 2 } - yield self.assertPartialExported(items, rows, settings=settings) + header = self.MyItem.fields.keys() + yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): + """If path without %(time)s or %(batch_id)s an exception must be raised""" settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1069,8 +1184,8 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEED_STORAGE_BATCH_SIZE': 1 } data = yield self.exported_no_data(settings) - data[fmt] = b''.join(data[fmt]) - self.assertEqual(data[fmt], b'') + data = dict(data) + self.assertEqual(b'', data[fmt][0]) @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -1088,8 +1203,8 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_SIZE': 1, } data = yield self.exported_no_data(settings) - data[fmt] = b''.join(data[fmt]) - self.assertEqual(data[fmt], expctd) + data = dict(data) + self.assertEqual(expctd, data[fmt][0]) From cac1f3a6adedc32977e0fb1830917a5e7d758bef Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Apr 2020 10:06:56 +0300 Subject: [PATCH 036/125] Update documentation --- docs/topics/feed-exports.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 9e5968a29..0bba03a7c 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,6 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` + * :setting:`FEED_EXPORT_BATCH_SIZE` .. currentmodule:: scrapy.extensions.feedexport @@ -429,3 +430,37 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + +.. setting:: FEED_EXPORT_BATCH_SIZE + +FEED_EXPORT_BATCH_SIZE +---------------------- +Default: ``None`` + +An integer number which represent number of scraped items stored in each output +file. Whenever the number of items exceeds this setting, a new file +creates and output redirects to it. The name of the new file will be selected +based on timestamp when the feed is being created and/or batch sequence number. +Therefore you must specify %(time)s or %(batch_id)s or both in the file path. + +* ``%(time)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_id)s`` - gets replaced by sequence number of batch + +For instance:: + + FEED_EXPORT_BATCH_SIZE=100 + +Your request can be like:: + + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json + +The result directory tree of above can be like:: + +->projectname +-->dirname +--->1-filename2020-03-28T14-45-08.237134.json +--->2-filename2020-03-28T14-45-09.148903.json +--->3-filename2020-03-28T14-45-10.046092.json + +Where first and second files contain exactly 100 items. The last one contains +<= 100 items. \ No newline at end of file From 5980ae72c6cb177f47fbb41d17837e8d98d50025 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Apr 2020 10:13:39 +0300 Subject: [PATCH 037/125] Some minor fixes and refactoring --- tests/test_feedexport.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 60e19d1df..e97e50e8e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -428,7 +428,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): yield item data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) + return data @defer.inlineCallbacks def exported_no_data(self, settings): @@ -443,7 +443,7 @@ class FeedExportTestBase(ABC, unittest.TestCase): pass data = yield self.run_and_export(TestSpider, settings) - defer.returnValue(data) + return data @defer.inlineCallbacks def assertExported(self, items, header, rows, settings=None, ordered=True): @@ -735,8 +735,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( - 'utf-8'), + 'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } @@ -753,8 +752,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode( - 'latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } @@ -775,8 +773,7 @@ class FeedExportTest(FeedExportTestBase): formats = { 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode( - 'latin-1'), + 'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), } @@ -1148,7 +1145,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """If path without %(time)s or %(batch_id)s an exception must be raised""" + """If path is without %(time)s or %(batch_id)s an exception must be raised""" settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, From 94ee68695a42ec8d3ccdff71bfc2fa33d5ea7049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 15:36:56 +0200 Subject: [PATCH 038/125] Mock server: use 127.0.0.1 also for HTTPS Windows throws an error about 0.0.0.0 being external: https://stackoverflow.com/a/23857995/939364 --- tests/mockserver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..e3dbdcc68 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -218,9 +218,8 @@ class MockServer(): self.proc.communicate() def url(self, path, is_secure=False): - host = self.http_address.replace('0.0.0.0', '127.0.0.1') - if is_secure: - host = self.https_address + host = self.https_address if is_secure else self.http_address + host = host.replace('0.0.0.0', '127.0.0.1') return host + path From 7cc9601029274124804e63ebebfa8783ac175205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 16:57:48 +0200 Subject: [PATCH 039/125] Improve reporting on test_ipv6_alternative_name_resolver --- tests/test_crawler.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 169e763f0..a6b079395 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -309,14 +309,8 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertTrue(any([ - "twisted.internet.error.ConnectionRefusedError" in log, - "twisted.internet.error.ConnectError" in log, - ])) - self.assertTrue(any([ - "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, - "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, - ])) + self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)") + self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,") def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From cf4180308982d9cd017167238a3df8de7900646d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 17:07:29 +0200 Subject: [PATCH 040/125] Skip test_reactor_poll on Windows --- tests/test_crawler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a6b079395..3d166e14c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,6 @@ import logging import os +import platform import subprocess import sys import warnings @@ -317,6 +318,7 @@ class CrawlerProcessSubprocess(unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows") def test_reactor_poll(self): log = self.run_script("twisted_reactor_poll.py") self.assertIn("Spider closed (finished)", log) From ea3e675801fe41c7517c58e21b46831940fbd064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Apr 2020 17:10:45 +0200 Subject: [PATCH 041/125] test_utils_iterators: use os.linesep --- tests/test_utils_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 33fc4d570..5b0073fd1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -8,7 +8,7 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = u"foo\nbar" +FOOBAR_NL = "foo{}bar".format(os.linesep) class XmliterTestCase(unittest.TestCase): From ec76445dd6753074c1531571f66467eecf22b498 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 18 Apr 2020 09:29:23 +0300 Subject: [PATCH 042/125] Update tests --- tests/test_feedexport.py | 66 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e97e50e8e..8e03a91c8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1128,6 +1128,7 @@ class PartialDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_items(self): + """ Test partial deliveries in all supported formats """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1145,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """If path is without %(time)s or %(batch_id)s an exception must be raised""" + """ If path is without %(time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1189,3 +1190,66 @@ class PartialDeliveriesTest(FeedExportTestBase): data = yield self.exported_no_data(settings) data = dict(data) self.assertEqual(expctd, data[fmt][0]) + + @defer.inlineCallbacks + def test_export_multiple_configs(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + + formats = { + 'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), + u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), + u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'.encode('latin-1')], + 'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', + 'indent': 0, + 'fields': ['bar'], + 'encoding': 'utf-8', + }, + os.path.join(self._random_temp_filename(), 'xml', self._file_mark): { + 'format': 'xml', + 'indent': 2, + 'fields': ['foo'], + 'encoding': 'latin-1', + }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'csv', + 'indent': None, + 'fields': ['bar', 'foo'], + 'encoding': 'utf-8', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + + @defer.inlineCallbacks + def test_batch_path_differ(self): + """ + Test that the name of all batch files differ from each other. + So %(time)s replaced with the current date. + """ + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), '%(time)s'): { + 'format': 'json', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + data = yield self.exported_data(items, settings) + self.assertEqual(len(items) + 1, len(data['json'])) From 1fecacbb1a3c2c4c61c202561f9c51f7f6b191ba Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Mon, 20 Apr 2020 12:05:15 -0300 Subject: [PATCH 043/125] IPv6 test: check for the absence of DNSLookupError --- tests/test_crawler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 3d166e14c..d6756c266 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -310,8 +310,7 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_ipv6_alternative_name_resolver(self): log = self.run_script('alternative_name_resolver.py') self.assertIn('Spider closed (finished)', log) - self.assertRegex(log, r"twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)") - self.assertRegex(log, r"'downloader/exception_type_count/twisted\.internet\.error\.(?:ConnectionRefusedError|ConnectError)': 1,") + self.assertNotIn("twisted.internet.error.DNSLookupError", log) def test_reactor_select(self): log = self.run_script("twisted_reactor_select.py") From f0f1be76d1e6cef65ac9a01d13c5d5060a03f648 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 27 Apr 2020 09:56:57 +0300 Subject: [PATCH 044/125] Using time_id instead of time as a timestamp --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 11 ++++++----- tests/test_feedexport.py | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0bba03a7c..2017be78f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file creates and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time)s or %(batch_id)s or both in the file path. +Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path. -* ``%(time)s`` - gets replaced by a timestamp when the feed is being created +* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 06ea6c5b2..72baa6269 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -292,7 +292,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri """ if previous_batch_slot is not None: previous_batch_id = previous_batch_slot.batch_id @@ -360,12 +360,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -397,8 +397,9 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) + params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') + params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 - params['time'] = datetime.utcnow().isoformat().replace(':', '-') uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8e03a91c8..da759917a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time)s_#%(batch_id)s_' + _file_mark = '_%(time_id)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time)s replaced with the current date. + So %(time_id)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time)s'): { + os.path.join(self._random_temp_filename(), '%(time_id)s'): { 'format': 'json', }, }, From 2eee6c81017e08bb492da560bc73c03f4f375fcc Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 27 Apr 2020 09:58:14 +0300 Subject: [PATCH 045/125] Documentation spelling fix --- docs/topics/feed-exports.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2017be78f..6c463fc27 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -438,10 +438,10 @@ FEED_EXPORT_BATCH_SIZE Default: ``None`` An integer number which represent number of scraped items stored in each output -file. Whenever the number of items exceeds this setting, a new file -creates and output redirects to it. The name of the new file will be selected +file. Whenever the number of items exceeds this setting, a new file is +created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in the file path. +Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. * ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch From 204737042ac6672eee73c975d0bd6735893d684c Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 27 Apr 2020 12:52:18 +0300 Subject: [PATCH 046/125] Extract the slot closing functionality to the function; minor changes --- scrapy/extensions/feedexport.py | 56 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 72baa6269..fe6061c33 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -255,7 +255,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): uri_params = self._get_uri_params(spider, feed['uri_params'], None) self.slots.append(self._start_new_batch( - previous_batch_slot=None, + batch_id=1, uri=uri % uri_params, feed=feed, spider=spider, @@ -265,42 +265,38 @@ class FeedExporter: def close_spider(self, spider): deferred_list = [] for slot in self.slots: - if not slot.itemcount and not slot.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) - slot.finish_exporting() - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': slot.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} - d = defer.maybeDeferred(slot.storage.store, slot.file) - d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, - extra={'spider': spider})) - d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) + d = self._close_slot(slot, spider) deferred_list.append(d) return defer.DeferredList(deferred_list) if deferred_list else None - def _start_new_batch(self, previous_batch_slot, uri, feed, spider, template_uri): + def _close_slot(self, slot, spider): + if not slot.itemcount and not slot.store_empty: + # We need to call slot.storage.store nonetheless to get the file + # properly closed. + return defer.maybeDeferred(slot.storage.store, slot.file) + slot.finish_exporting() + logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" + log_args = {'format': slot.format, + 'itemcount': slot.itemcount, + 'uri': slot.uri} + d = defer.maybeDeferred(slot.storage.store, slot.file) + d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, + extra={'spider': spider})) + d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, + exc_info=failure_to_exc_info(f), + extra={'spider': spider})) + return d + + def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. - :param previous_batch_slot: slot of previous batch. We need to call slot.storage.store - to get the file properly closed. + :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri """ - if previous_batch_slot is not None: - previous_batch_id = previous_batch_slot.batch_id - previous_batch_slot.exporter.finish_exporting() - previous_batch_slot.storage.store(previous_batch_slot.file) - else: - previous_batch_id = 0 - storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter( @@ -317,7 +313,7 @@ class FeedExporter: uri=uri, format=feed['format'], store_empty=feed['store_empty'], - batch_id=previous_batch_id + 1, + batch_id=batch_id, template_uri=template_uri, ) if slot.store_empty: @@ -330,10 +326,12 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - if self.storage_batch_size and slot.itemcount % self.storage_batch_size == 0: + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) + self._close_slot(slot, spider) slots.append(self._start_new_batch( - previous_batch_slot=slot, + batch_id=slot.batch_id + 1, uri=slot.template_uri % uri_params, feed=self.feeds[slot.template_uri], spider=spider, From 3f9874fac9f93c0956afa5975d7b2bbb21816894 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 1 May 2020 11:52:16 +0300 Subject: [PATCH 047/125] Add test s3 export --- tests/test_feedexport.py | 70 ++++++++++++++++++++++++++++++++++++++++ tox.ini | 1 + 2 files changed, 71 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index da759917a..9fc39c3a6 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1253,3 +1253,73 @@ class PartialDeliveriesTest(FeedExportTestBase): } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) + + @defer.inlineCallbacks + def test_s3_export(self): + """ + Test export of items into s3 bucket. + S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini + to perform this test: + [testenv] + setenv = + AWS_SECRET_ACCESS_KEY = ABCD + AWS_ACCESS_KEY_ID = ABCD + S3_TEST_BUCKET_NAME = ABCD + """ + try: + import boto3 + except ImportError: + raise unittest.SkipTest("S3FeedStorage requires boto3") + + assert_aws_environ() + s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME') + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + if not s3_test_bucket_name: + raise unittest.SkipTest("No S3 BUCKET available for testing") + + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + prefix = 'tmp/{filename}'.format(filename=filename) + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + bucket_name=s3_test_bucket_name, prefix=prefix + ) + storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) + settings = { + 'FEEDS': { + s3_test_file_uri: { + 'format': 'json', + }, + }, + 'FEED_STORAGE_BATCH_SIZE': 1, + } + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), + self.MyItem({'foo': 'bar3', 'baz': 'quux3'}), + ] + verifyObject(IFeedStorage, storage) + + class TestSpider(scrapy.Spider): + name = 'testspider' + + def parse(self, response): + for item in items: + yield item + + s3 = boto3.resource('s3') + my_bucket = s3.Bucket(s3_test_bucket_name) + batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + + with MockServer() as s: + runner = CrawlerRunner(Settings(settings)) + TestSpider.start_urls = [s.url('/')] + yield runner.crawl(TestSpider) + + for file_uri in my_bucket.objects.filter(Prefix=prefix): + content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key) + if not content and not items: + break + content = json.loads(content.decode('utf-8')) + expected_batch, items = items[:batch_size], items[batch_size:] + self.assertEqual(expected_batch, content) diff --git a/tox.ini b/tox.ini index cd118c921..c77fae1f0 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = # Extras botocore>=1.3.23 Pillow>=3.4.2 + boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From dad2ea75222d6240c569440d3221f5fc00925682 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 2 May 2020 01:21:03 +0300 Subject: [PATCH 048/125] Change time_id to batch_time --- docs/topics/feed-exports.rst | 6 +++--- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6c463fc27..2106b41f5 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is created and output redirects to it. The name of the new file will be selected based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. -* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created +* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created * ``%(batch_id)s`` - gets replaced by sequence number of batch For instance:: @@ -452,7 +452,7 @@ For instance:: Your request can be like:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The result directory tree of above can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fe6061c33..a262f5d18 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -295,7 +295,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri + :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s + If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri: + if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') return False def _storage_supported(self, uri): @@ -396,7 +396,7 @@ class FeedExporter: for k in dir(spider): params[k] = getattr(spider, k) params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['time_id'] = datetime.utcnow().isoformat().replace(':', '-') + params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9fc39c3a6..2217bb4ed 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase): class PartialDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(time_id)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)s_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): - """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase): def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. - So %(time_id)s replaced with the current date. + So %(batch_time)s replaced with the current date. """ items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase): ] settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), '%(time_id)s'): { + os.path.join(self._random_temp_filename(), '%(batch_time)s'): { 'format': 'json', }, }, @@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) prefix = 'tmp/{filename}'.format(filename=filename) - s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format( + s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format( bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) From 286fca733f23fa41165edcdcc7ab7593cc6b074f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 6 May 2020 16:20:33 +0200 Subject: [PATCH 049/125] Fix parameter name, broken by copy-pasting --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 8f6fd3e6a..99ffa0dc9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -248,7 +248,7 @@ class BaseSettings(MutableMapping): :type name: str :param value: the value to associate with the setting - :type default: object + :type value: object :param priority: the priority of the setting. Should be a key of :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer From bbd9d05dbff7abb6ab3a5ea575d75cfe88cb2ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 7 May 2020 11:44:43 +0200 Subject: [PATCH 050/125] request-response.rst: review type references around body mentions --- docs/topics/request-response.rst | 38 +++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1638c202d..4fec70e13 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,10 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str + :param body: the request body. If a string is passed, it is converted to + bytes using *encoding*, which defaults to ``utf-8``. If not passed or + ``None`` is passed, an empty bytes array is stored. + :type body: bytes :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +104,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes if given as a string. :type encoding: str :param priority: the priority of this request (defaults to ``0``). @@ -159,7 +157,7 @@ Request objects .. attribute:: Request.body - A str that contains the request body. + The request body as bytes. This attribute is read-only. To change the body of a Request use :meth:`replace`. @@ -598,7 +596,7 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. To access the decoded text as str you can use + :param body: the response body. To access the decoded text as a string, use ``response.text`` from an encoding-aware :ref:`Response subclass <topics-request-response-ref-response-subclasses>`, such as :class:`TextResponse`. @@ -646,10 +644,10 @@ Response objects .. attribute:: Response.body - The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use - :attr:`TextResponse.text` (only available in :class:`TextResponse` - and subclasses). + The response body as bytes. + + If you want the body as a string, use :attr:`TextResponse.text` (only + available in :class:`TextResponse` and subclasses). This attribute is read-only. To change the body of a Response use :meth:`replace`. @@ -768,10 +766,10 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode - body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the - encoding will be looked up in the response headers and body instead. + response. If you create a :class:`TextResponse` object with a string as + body, it will be converted to bytes encoded using this encoding. If + *encoding* is ``None`` (default), the encoding will be looked up in the + response headers and body instead. :type encoding: str :class:`TextResponse` objects support the following attributes in addition @@ -779,7 +777,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -787,8 +785,8 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding + ``str(response.body)`` is not a correct way to convert the response + body into a string: you would be using the system default encoding (typically ``ascii``) instead of the response encoding. From c6746f0e381a44f3d66efad86a4aca87805138bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Sat, 9 May 2020 15:51:11 +0200 Subject: [PATCH 051/125] =?UTF-8?q?bytes=20array=20=E2=86=92=20bytes=20obj?= =?UTF-8?q?ect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4fec70e13..ad6c10b6e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -53,7 +53,7 @@ Request objects :param body: the request body. If a string is passed, it is converted to bytes using *encoding*, which defaults to ``utf-8``. If not passed or - ``None`` is passed, an empty bytes array is stored. + ``None`` is passed, an empty :class:`bytes` object is stored. :type body: bytes :param headers: the headers of this request. The dict values can be strings From e07708e3744fb26fa72042720b33deab59331cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Sat, 9 May 2020 15:54:31 +0200 Subject: [PATCH 052/125] request-response: update the consequences of str(b'') --- docs/topics/request-response.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ad6c10b6e..397632932 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -786,8 +786,7 @@ TextResponse objects .. note:: ``str(response.body)`` is not a correct way to convert the response - body into a string: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + body into a string: ``str(b'')`` returns ``"b''"``. .. attribute:: TextResponse.encoding From b5684909d1cb01ad138a389caa750485b51f79cf Mon Sep 17 00:00:00 2001 From: Jacty <jacty@users.noreply.github.com> Date: Mon, 11 May 2020 11:18:25 +0800 Subject: [PATCH 053/125] Unnecessary update when value is None When value is None, it is not necessary to invoke update and run other methods and conditions to make the code complicated there. --- scrapy/settings/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b9a13c018..f28fbfaf9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -83,7 +83,8 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - self.update(values, priority) + if values is not None: + self.update(values, priority) def __getitem__(self, opt_name): if opt_name not in self: From 33ab0a36635fbd45debbc44584002bd7a4ef7fed Mon Sep 17 00:00:00 2001 From: Jacty <jacty@users.noreply.github.com> Date: Wed, 13 May 2020 06:11:07 +0800 Subject: [PATCH 054/125] Update __init__.py --- scrapy/settings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28fbfaf9..0425b48b3 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -83,7 +83,7 @@ class BaseSettings(MutableMapping): def __init__(self, values=None, priority='project'): self.frozen = False self.attributes = {} - if values is not None: + if values: self.update(values, priority) def __getitem__(self, opt_name): From 2327ecead085a41d1a71a70a12eb988bbf982268 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 13 May 2020 22:50:04 +0300 Subject: [PATCH 055/125] Rename FEED_STORAGE_BATCH_SIZE to FEED_STORAGE_BATCH_ITEM_COUNT --- docs/topics/feed-exports.rst | 8 ++++---- scrapy/extensions/feedexport.py | 10 +++++----- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 28 ++++++++++++++-------------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2106b41f5..917240d4d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_EXPORT_BATCH_SIZE` + * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_EXPORT_BATCH_SIZE +.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT -FEED_EXPORT_BATCH_SIZE +FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` @@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. For instance:: - FEED_EXPORT_BATCH_SIZE=100 + FEED_STORAGE_BATCH_ITEM_COUNT=100 Your request can be like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index a262f5d18..5bc946634 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None) + self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -290,7 +290,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified. + Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -326,7 +326,7 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one + # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) self._close_slot(slot, spider) @@ -358,12 +358,12 @@ class FeedExporter: def _batch_deliveries_supported(self, uri): """ - If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s + If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified') + logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c3463a505..5a7dc533e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_SIZE = None +FEED_STORAGE_BATCH_ITEM_COUNT = None FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2217bb4ed..1a21eeba9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_SIZE': 2 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=settings) @@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_SIZE': 1 + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_SIZE': 1, + 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_SIZE'] + batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] with MockServer() as s: runner = CrawlerRunner(Settings(settings)) From 8662d3587df74841d4ea640c0432446569e59262 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 13 May 2020 23:41:01 +0300 Subject: [PATCH 056/125] Documentation and code refactoring --- docs/topics/feed-exports.rst | 21 ++++++++++++--------- scrapy/extensions/feedexport.py | 7 ++++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 917240d4d..0f15044b3 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -437,24 +437,27 @@ FEED_STORAGE_BATCH_ITEM_COUNT ---------------------- Default: ``None`` -An integer number which represent number of scraped items stored in each output +An integer number that represents the number of scraped items stored in each output file. Whenever the number of items exceeds this setting, a new file is -created and output redirects to it. The name of the new file will be selected -based on timestamp when the feed is being created and/or batch sequence number. -Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI. +created and the output is redirected to it. The name of the new file will be selected +based on the timestamp when the feed is being created and/or on the batch sequence number. +Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created -* ``%(batch_id)s`` - gets replaced by sequence number of batch +(e.g. `2020-03-28T14-45-08.237134`) -For instance:: +* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch +(e.g. `2` for the second file) + +For instance, if your settings include:: FEED_STORAGE_BATCH_ITEM_COUNT=100 -Your request can be like:: +And your :command:`crawl` command line is:: scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json -The result directory tree of above can be like:: +The resulting directory tree of above can be like:: ->projectname -->dirname @@ -462,5 +465,5 @@ The result directory tree of above can be like:: --->2-filename2020-03-28T14-45-09.148903.json --->3-filename2020-03-28T14-45-10.046092.json -Where first and second files contain exactly 100 items. The last one contains +Where the first and second files contain exactly 100 items. The last one contains <= 100 items. \ No newline at end of file diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5bc946634..4c9362f3a 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,6 +25,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -337,9 +338,9 @@ class FeedExporter: spider=spider, template_uri=slot.template_uri, )) - self.slots[idx] = None - self.slots = [slot for slot in self.slots if slot is not None] - self.slots.extend(slots) + else: + slots.append(slot) + self.slots = slots def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) From 69c005f013eb0dc000611853e66371fad17dea9d Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 14 May 2020 10:35:56 +0300 Subject: [PATCH 057/125] Documentation indent fix --- docs/topics/feed-exports.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 42c4e2267..dfeea5b7f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -444,10 +444,10 @@ based on the timestamp when the feed is being created and/or on the batch sequen Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created -(e.g. `2020-03-28T14-45-08.237134`) + (e.g. `2020-03-28T14-45-08.237134`) * ``%(batch_id)s`` - gets replaced by the batch sequence number of batch -(e.g. `2` for the second file) + (e.g. `2` for the second file) For instance, if your settings include:: From 1cdcf8b08b8f1e68c5b107b6ae39b2da1aedd245 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 15 May 2020 19:46:36 +0300 Subject: [PATCH 058/125] Minor fixes --- docs/topics/feed-exports.rst | 19 ++++++++++--------- scrapy/extensions/feedexport.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dfeea5b7f..638733b6a 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT ----------------------------- Default: ``None`` -An integer number that represents the number of scraped items stored in each output -file. Whenever the number of items exceeds this setting, a new file is -created and the output is redirected to it. The name of the new file will be selected -based on the timestamp when the feed is being created and/or on the batch sequence number. -Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`. +If assigned an integer number higher than ``0``, Scrapy generates multiple output files +storing up to the specified number of items in each output file. + +When generating multiple output files, you must use at least one of the following +placeholders in :setting:`FEED_URI` to indicate how the different output file names are +generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created - (e.g. `2020-03-28T14-45-08.237134`) + (e.g. ``2020-03-28T14-45-08.237134``) * ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. `2` for the second file) + (e.g. ``2`` for the second file) For instance, if your settings include:: @@ -457,7 +458,7 @@ And your :command:`crawl` command line is:: scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json -The resulting directory tree of above can be like:: +The command line above can generate a directory tree like:: ->projectname -->dirname @@ -466,4 +467,4 @@ The resulting directory tree of above can be like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -<= 100 items. \ No newline at end of file +100 items or fever. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4c9362f3a..3d691c580 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): self.file = file self.exporter = exporter self.storage = storage @@ -188,7 +188,7 @@ class _FeedSlot: self.batch_id = batch_id self.format = format self.store_empty = store_empty - self.template_uri = template_uri + self.uri_template = uri_template self.uri = uri # flags self.itemcount = 0 @@ -260,7 +260,7 @@ class FeedExporter: uri=uri % uri_params, feed=feed, spider=spider, - template_uri=uri, + uri_template=uri, )) def close_spider(self, spider): @@ -288,7 +288,7 @@ class FeedExporter: extra={'spider': spider})) return d - def _start_new_batch(self, batch_id, uri, feed, spider, template_uri): + def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. @@ -296,7 +296,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -315,7 +315,7 @@ class FeedExporter: format=feed['format'], store_empty=feed['store_empty'], batch_id=batch_id, - template_uri=template_uri, + uri_template=uri_template, ) if slot.store_empty: slot.start_exporting() @@ -329,14 +329,14 @@ class FeedExporter: slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one if self.storage_batch_size and slot.itemcount == self.storage_batch_size: - uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot) + uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( batch_id=slot.batch_id + 1, - uri=slot.template_uri % uri_params, - feed=self.feeds[slot.template_uri], + uri=slot.uri_template % uri_params, + feed=self.feeds[slot.uri_template], spider=spider, - template_uri=slot.template_uri, + uri_template=slot.uri_template, )) else: slots.append(slot) From 10ae1a284f759b541d086e3d1a13cda96b6e2040 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 15 May 2020 22:50:54 +0300 Subject: [PATCH 059/125] Minor fixes --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 10 +++++----- tests/test_feedexport.py | 2 +- tox.ini | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 638733b6a..6f7db20c4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -467,4 +467,4 @@ The command line above can generate a directory tree like:: --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains -100 items or fever. +100 items or fewer. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3d691c580..cc26ae173 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,11 +243,11 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._batch_deliveries_supported(uri): + if not self._settings_are_valid(uri): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -328,7 +328,7 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_size and slot.itemcount == self.storage_batch_size: + if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -357,12 +357,12 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _batch_deliveries_supported(self, uri): + def _settings_are_valid(self, uri): """ If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri: + if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d1374f291..88f9a5933 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -986,7 +986,7 @@ class FeedExportTest(FeedExportTestBase): self.assertEqual(data['csv'], b'') -class PartialDeliveriesTest(FeedExportTestBase): +class BatchDeliveriesTest(FeedExportTestBase): __test__ = True _file_mark = '_%(batch_time)s_#%(batch_id)s_' diff --git a/tox.ini b/tox.ini index 6dd944dff..7507a14a6 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ deps = -ctests/constraints.txt -rtests/requirements-py3.txt # Extras + boto3>=1.13.0 botocore>=1.3.23 Pillow>=3.4.2 - boto3>=1.13.0 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID From a7d070f3bb350cbe1f7b580350d5f491f59d47d8 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 18 May 2020 22:25:29 +0300 Subject: [PATCH 060/125] Change log level to error --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cc26ae173..ce7fc372d 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -364,7 +364,7 @@ class FeedExporter: """ if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') + logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') return False def _storage_supported(self, uri): From 677e619d3761e6669c247786bb95822ce38c8080 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 21 May 2020 14:57:03 +0300 Subject: [PATCH 061/125] Fix too long lines --- scrapy/extensions/feedexport.py | 4 +++- tests/test_feedexport.py | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ce7fc372d..1f745be98 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -364,7 +364,9 @@ class FeedExporter: """ if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: return True - logger.error('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified') + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' + ) return False def _storage_supported(self, uri): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08ee24768..fecb17e29 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1204,12 +1204,20 @@ class BatchDeliveriesTest(FeedExportTestBase): items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] formats = { - 'json': [u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), - u'[\n{"bar": "BAR1"}\n]'.encode('utf-8')], - 'xml': [u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>'.encode('latin-1'), - u'<?xml version="1.0" encoding="latin-1"?>\n<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'.encode('latin-1')], - 'csv': [u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), - u'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), + '[\n{"bar": "BAR1"}\n]'.encode('utf-8')], + 'xml': [ + ( + '<?xml version="1.0" encoding="latin-1"?>\n' + '<items>\n <item>\n <foo>FOO</foo>\n </item>\n</items>' + ).encode('latin-1'), + ( + '<?xml version="1.0" encoding="latin-1"?>\n' + '<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>' + ).encode('latin-1') + ], + 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], } settings = { From dd96f94e8cc1517b7021e35e46cbdc92580c6333 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 22 May 2020 23:30:33 +0300 Subject: [PATCH 062/125] Push datetime.utcnow() to its own variable --- scrapy/extensions/feedexport.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1f745be98..45c2971a6 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -398,8 +398,9 @@ class FeedExporter: params = {} for k in dir(spider): params[k] = getattr(spider, k) - params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') - params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-') + utc_now = datetime.utcnow() + params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-') + params['batch_time'] = utc_now.isoformat().replace(':', '-') params['batch_id'] = slot.batch_id + 1 if slot is not None else 1 uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) From b99fe4aa4c35ff8f5c2355a09988ddb1e90d6b70 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 19 Jun 2020 21:41:15 -0300 Subject: [PATCH 063/125] Add google-cloud-storage to the 'pinned' tox environment --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 4c790158d..d28eab131 100644 --- a/tox.ini +++ b/tox.ini @@ -76,6 +76,7 @@ deps = -rtests/requirements-py3.txt # Extras botocore==1.3.23 + google-cloud-storage==1.29.0 Pillow==3.4.2 [testenv:extra-deps] From cfd039aeb6d46f7c120edd82ce06d6b9f4f8e7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 22 Jun 2020 19:28:33 +0200 Subject: [PATCH 064/125] Remove a duplicate GCS_PROJECT_ID reference target --- docs/topics/media-pipeline.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 01de3dedb..096618648 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -204,7 +204,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S Google Cloud Storage --------------------- -.. setting:: GCS_PROJECT_ID .. setting:: FILES_STORE_GCS_ACL .. setting:: IMAGES_STORE_GCS_ACL From c3cee74fd401e6a6307b5eb1786e532bb2cd5aa8 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 26 Jun 2020 18:45:21 +0300 Subject: [PATCH 065/125] Change default value of FEED_STORAGE_BATCH_ITEM_COUNT to 0 --- docs/topics/feed-exports.rst | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 20 ++++++++++---------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 866ce78eb..0b37e9a7d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_STORAGE_BATCH_ITEM_COUNT ----------------------------- -Default: ``None`` +Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files storing up to the specified number of items in each output file. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 1331782e3..e06116acd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -243,7 +243,7 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None) + self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5a7dc533e..810acd5a3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = None +FEED_STORAGE_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 1a6a5624b..578cd396b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() - yield self.assertExported(items, header, rows, settings=settings) + yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ @@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase): bucket_name=s3_test_bucket_name, prefix=prefix ) storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key) - settings = { + settings = Settings({ 'FEEDS': { s3_test_file_uri: { 'format': 'json', }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, - } + }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}), @@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT'] + batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) From 88a52198b90faa0129c8e05072197cdffbb9653b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sat, 27 Jun 2020 11:50:26 +0300 Subject: [PATCH 066/125] Add batch_item_count support in FEEDS setting --- scrapy/extensions/feedexport.py | 5 +++-- tests/test_feedexport.py | 39 ++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e06116acd..2312c994e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values - logger = logging.getLogger(__name__) @@ -337,7 +336,9 @@ class FeedExporter: slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count: + if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ + and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', + self.storage_batch_item_count): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 578cd396b..3bc0c083c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase): '<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>' ).encode('latin-1') ], - 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), - 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')], + 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), + 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], + 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1348,9 +1349,16 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { 'format': 'csv', 'indent': None, - 'fields': ['bar', 'foo'], + 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, + os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + 'format': 'jsonlines', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + 'batch_item_count': 0, + }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase): for expected_batch, got_batch in zip(expected, data[fmt]): self.assertEqual(expected_batch, got_batch) + @defer.inlineCallbacks + def test_batch_item_count_feeds_setting(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + + formats = { + 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), + '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + } + + settings = { + 'FEEDS': { + os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { + 'format': 'jsonlines', + 'indent': None, + 'fields': ['foo', 'bar'], + 'encoding': 'utf-8', + 'batch_item_count': 1, + }, + }, + } + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + for expected_batch, got_batch in zip(expected, data[fmt]): + self.assertEqual(expected_batch, got_batch) + @defer.inlineCallbacks def test_batch_path_differ(self): """ From 05c2587c6a32b84a94463f2b1187e49f94957aa2 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Sun, 28 Jun 2020 09:45:45 +0300 Subject: [PATCH 067/125] Docs update and tiny fixes --- docs/topics/feed-exports.rst | 1 + tests/test_feedexport.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b37e9a7d..3da56821e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,6 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` +* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 3bc0c083c..542cce70f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'csv', self._file_mark): { + os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { 'format': 'jsonlines', 'indent': None, 'fields': ['foo', 'bar'], @@ -1423,8 +1423,8 @@ class BatchDeliveriesTest(FeedExportTestBase): [testenv] setenv = AWS_SECRET_ACCESS_KEY = ABCD - AWS_ACCESS_KEY_ID = ABCD - S3_TEST_BUCKET_NAME = ABCD + AWS_ACCESS_KEY_ID = EFGH + S3_TEST_BUCKET_NAME = IJKL """ try: import boto3 From 5b88c522ac1b1a9ba1588573d90cf3bc01339282 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Tue, 30 Jun 2020 12:18:21 -0300 Subject: [PATCH 068/125] Simplify dataclass example in item loader docs --- docs/topics/loaders.rst | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index e921395d2..9c82bb4d9 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -88,29 +88,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods. -Given the way that item loaders store data internally, one approach -to overcome this is to define items using the :func:`~dataclasses.field` -function, with ``list`` as the ``default_factory`` argument:: +One approach to overcome this is to define items using the +:func:`~dataclasses.field` function, with a ``default`` argument:: from dataclasses import dataclass, field + from typing import Optional @dataclass class InventoryItem: - name: str = field(default_factory=list) - price: float = field(default_factory=list) - stock: int = field(default_factory=list) - -Note that in order to keep the example simple, the types do not match -completely. A more accurate but verbose definition would be:: - - from dataclasses import dataclass, field - from typing import List, Union - - @dataclass - class InventoryItem: - name: Union[str, List[str]] = field(default_factory=list) - price: Union[float, List[float]] = field(default_factory=list) - stock: Union[int, List[int]] = field(default_factory=list) + name: Optional[str] = field(default=None) + price: Optional[float] = field(default=None) + stock: Optional[int] = field(default=None) .. _topics-loaders-processors: From 7b1d3c35ea3bfde2ac7fc69a2a26bbcb94aec1bf Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Wed, 1 Jul 2020 11:54:39 +0300 Subject: [PATCH 069/125] Minor updates --- docs/topics/feed-exports.rst | 4 ++-- scrapy/extensions/feedexport.py | 34 +++++++++++++++++---------------- scrapy/utils/conf.py | 4 ++++ tests/test_feedexport.py | 23 ++++++---------------- tests/test_utils_conf.py | 4 ++++ 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 3da56821e..0b659f30e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu storing up to the specified number of items in each output file. When generating multiple output files, you must use at least one of the following -placeholders in :setting:`FEED_URI` to indicate how the different output file names are +placeholders in the feed URI to indicate how the different output file names are generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created @@ -457,7 +457,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 2312c994e..5908987a3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -242,7 +242,6 @@ class FeedExporter: self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') - self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured @@ -253,7 +252,7 @@ class FeedExporter: def open_spider(self, spider): for uri, feed in self.feeds.items(): - uri_params = self._get_uri_params(spider, feed['uri_params'], None) + uri_params = self._get_uri_params(spider, feed['uri_params']) self.slots.append(self._start_new_batch( batch_id=1, uri=uri % uri_params, @@ -299,7 +298,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified. + Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -331,14 +330,15 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] - for idx, slot in enumerate(self.slots): + for slot in self.slots: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one - if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \ - and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count', - self.storage_batch_item_count): + if ( + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) slots.append(self._start_new_batch( @@ -369,15 +369,17 @@ class FeedExporter: def _settings_are_valid(self, uri): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s - to distinguish different files of partial output + If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ - if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri: - return True - logger.error( - '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified' - ) - return False + for uri_template, values in self.feeds.items(): + if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + logger.error( + '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + ) + return False + return True def _storage_supported(self, uri): scheme = urlparse(uri).scheme @@ -404,7 +406,7 @@ class FeedExporter: def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider, uri_params, slot): + def _get_uri_params(self, spider, uri_params, slot=None): params = {} for k in dir(spider): params[k] = getattr(spider, k) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5921f82bf..0e02f0f28 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) + out.setdefault( + "batch_item_count", + out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) + ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 542cce70f..db14b20b9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, @@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase): ], 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'), 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')], - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], } settings = { @@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase): 'fields': ['foo', 'bar'], 'encoding': 'utf-8', }, - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', - 'indent': None, - 'fields': ['foo', 'bar'], - 'encoding': 'utf-8', - 'batch_item_count': 0, - }, }, 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, } @@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] - + items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] formats = { - 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'), - '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')], + 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), + '[{"foo": "FOO1"}]'.encode('utf-8')], } - settings = { 'FEEDS': { - os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): { - 'format': 'jsonlines', + os.path.join(self._random_temp_filename(), 'json', self._file_mark): { + 'format': 'json', 'indent': None, - 'fields': ['foo', 'bar'], 'encoding': 'utf-8', 'batch_item_count': 1, }, diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index e5d3ef582..95ec2b64a 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": (1, 2, 3, 4), + "batch_item_count": 2, }) def test_feed_complete_default_values_from_settings_non_empty(self): @@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, + "FEED_STORAGE_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "indent": 42, "store_empty": True, "uri_params": None, + "batch_item_count": 2, }) From af55d23167f9cec22f815bc9f9884b10a9a35f5b Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin <wrar@wrar.name> Date: Wed, 1 Jul 2020 19:46:54 +0500 Subject: [PATCH 070/125] Update the OpenSSL cipher list format link OpenSSL `ciphers(1)` is now almost empty: https://www.openssl.org/docs/manmaster/man1/ciphers.html Alternative would be linking to 1.1.1 docs specifically. --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5178f272f..8cc8806a5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a specific cipher that is not included in ``DEFAULT`` if a website requires it. -.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT .. setting:: DOWNLOADER_CLIENT_TLS_METHOD From 1e245046ed8ac3d9f89860501c2da95b69aaabf6 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 2 Jul 2020 12:38:08 +0300 Subject: [PATCH 071/125] Change setting name. Add leading zeroes to batch_id. Minor fixes. --- docs/topics/feed-exports.rst | 19 +++++++++-------- scrapy/extensions/feedexport.py | 24 +++++++++++++--------- scrapy/settings/default_settings.py | 2 +- scrapy/utils/conf.py | 5 +---- tests/test_feedexport.py | 32 ++++++++++++++--------------- tests/test_utils_conf.py | 4 ++-- 6 files changed, 45 insertions(+), 41 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0b659f30e..56efa80a7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` + * :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. currentmodule:: scrapy.extensions.feedexport @@ -272,7 +272,7 @@ as a fallback value if that key is not provided for a specific feed definition. * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_STORAGE_BATCH_ITEM_COUNT` +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` .. setting:: FEED_EXPORT_ENCODING @@ -432,9 +432,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl -.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT +.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT -FEED_STORAGE_BATCH_ITEM_COUNT +FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- Default: ``0`` @@ -448,16 +448,19 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch - (e.g. ``2`` for the second file) +* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. +By replacing ``x`` with an integer you set the number of leading zeroes to prevent +inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: + ``%(batch_id)01d`` for the second batch gets replaced by ``2`` + ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` For instance, if your settings include:: - FEED_STORAGE_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT=100 And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json + scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json The command line above can generate a directory tree like:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 5908987a3..adb6ea2e4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst import logging import os +import re import sys import warnings from datetime import datetime @@ -25,6 +26,7 @@ from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -245,7 +247,7 @@ class FeedExporter: for uri, feed in self.feeds.items(): if not self._storage_supported(uri): raise NotConfigured - if not self._settings_are_valid(uri): + if not self._settings_are_valid(): raise NotConfigured if not self._exporter_supported(feed['format']): raise NotConfigured @@ -298,7 +300,7 @@ class FeedExporter: def _start_new_batch(self, batch_id, uri, feed, spider, uri_template): """ Redirect the output data stream to a new file. - Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified + Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified :param batch_id: sequence number of current batch :param uri: uri of the new batch to start :param feed: dict with parameters of feed @@ -334,10 +336,10 @@ class FeedExporter: slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 - # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one + # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one if ( - self.feeds[slot.uri_template]['batch_item_count'] - and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] + self.feeds[slot.uri_template]['batch_item_count'] + and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count'] ): uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot) self._close_slot(slot, spider) @@ -367,16 +369,18 @@ class FeedExporter: return True logger.error("Unknown feed format: %(format)s", {'format': format}) - def _settings_are_valid(self, uri): + def _settings_are_valid(self): """ - If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain + If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']): + if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): logger.error( - '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template) + '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' + ''.format(uri_template) ) return False return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 810acd5a3..0016bbe1b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -146,7 +146,7 @@ FEED_STORAGES_BASE = { 's3': 'scrapy.extensions.feedexport.S3FeedStorage', 'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage', } -FEED_STORAGE_BATCH_ITEM_COUNT = 0 +FEED_EXPORT_BATCH_ITEM_COUNT = 0 FEED_EXPORTERS = {} FEED_EXPORTERS_BASE = { 'json': 'scrapy.exporters.JsonItemExporter', diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0e02f0f28..64f9c824b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -113,12 +113,9 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() + out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')) out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) - out.setdefault( - "batch_item_count", - out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')) - ) out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) if settings["FEED_EXPORT_INDENT"] is None: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index db14b20b9..d20b40e2f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1108,7 +1108,7 @@ class FeedExportTest(FeedExportTestBase): class BatchDeliveriesTest(FeedExportTestBase): __test__ = True - _file_mark = '_%(batch_time)s_#%(batch_id)s_' + _file_mark = '_%(batch_time)s_#%(batch_id)02d_' @defer.inlineCallbacks def run_and_export(self, spider_cls, settings): @@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['jl']: @@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') data = yield self.exported_data(items, settings) for batch in data['csv']: got_batch = csv.DictReader(to_unicode(batch).splitlines()) @@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) for batch in data['xml']: @@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) # XML @@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import pickle @@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase): os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'}, }, }) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') rows = [{k: v for k, v in row.items() if v} for row in rows] data = yield self.exported_data(items, settings) import marshal @@ -1259,18 +1259,18 @@ class BatchDeliveriesTest(FeedExportTestBase): {'foo': 'bar3', 'baz': 'quux3', 'egg': ''} ] settings = { - 'FEED_STORAGE_BATCH_ITEM_COUNT': 2 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 2 } header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } crawler = get_crawler(settings_dict=settings) self.assertRaises(NotConfigured, FeedExporter, crawler) @@ -1282,7 +1282,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'FEEDS': { os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt}, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1 + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1 } data = yield self.exported_no_data(settings) data = dict(data) @@ -1304,7 +1304,7 @@ class BatchDeliveriesTest(FeedExportTestBase): }, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_no_data(settings) data = dict(data) @@ -1352,7 +1352,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'encoding': 'utf-8', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) for fmt, expected in formats.items(): @@ -1398,7 +1398,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, } data = yield self.exported_data(items, settings) self.assertEqual(len(items) + 1, len(data['json'])) @@ -1440,7 +1440,7 @@ class BatchDeliveriesTest(FeedExportTestBase): 'format': 'json', }, }, - 'FEED_STORAGE_BATCH_ITEM_COUNT': 1, + 'FEED_EXPORT_BATCH_ITEM_COUNT': 1, }) items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), @@ -1458,7 +1458,7 @@ class BatchDeliveriesTest(FeedExportTestBase): s3 = boto3.resource('s3') my_bucket = s3.Bucket(s3_test_bucket_name) - batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT') + batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT') with MockServer() as s: runner = CrawlerRunner(Settings(settings)) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 95ec2b64a..f3ef36127 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -149,7 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, "FEED_URI_PARAMS": (1, 2, 3, 4), - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { @@ -171,7 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase): "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], "FEED_EXPORT_INDENT": 42, "FEED_STORE_EMPTY": True, - "FEED_STORAGE_BATCH_ITEM_COUNT": 2, + "FEED_EXPORT_BATCH_ITEM_COUNT": 2, }) new_feed = feed_complete_default_values_from_settings(feed, settings) self.assertEqual(new_feed, { From 3199048520ebe3798c14cfa7362612b51d156b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 2 Jul 2020 20:10:08 +0200 Subject: [PATCH 072/125] Complete Azure Pipelines CI setup --- azure-pipelines.yml | 18 ++++++------------ tests/CrawlerRunner/ip_address.py | 15 ++++++++++++++- tests/mockserver.py | 5 ++--- tests/test_commands.py | 5 +++++ tests/test_crawler.py | 13 +++++++++++++ tests/test_feedexport.py | 8 +++++++- tests/test_proxy_connect.py | 3 +++ tests/test_spiderloader/__init__.py | 23 +++++++++++++++-------- tests/test_utils_asyncio.py | 7 ++++++- tox.ini | 18 +++++++++++++++--- 10 files changed, 86 insertions(+), 29 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffc4d549b..710e42090 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,29 +1,23 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - - +variables: + TOXENV: py pool: - vmImage: 'windows-2019' + vmImage: 'windows-latest' strategy: matrix: Python35: python.version: '3.5' - TOXENV: py35 + TOXENV: windows-pinned Python36: python.version: '3.6' - TOXENV: py36 Python37: python.version: '3.7' - TOXENV: py37 - + Python38: + python.version: '3.8' steps: - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' displayName: 'Use Python $(python.version)' - - script: | pip install -U tox twine wheel codecov tox diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 826374cd4..ea75bc3c9 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,7 +1,9 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names.client import createResolver +from twisted.names import cache, hosts as hostsModule, resolve +from twisted.names.client import Resolver +from twisted.python.runtime import platform from scrapy import Spider, Request from scrapy.crawler import CrawlerRunner @@ -10,6 +12,17 @@ from scrapy.utils.log import configure_logging from tests.mockserver import MockServer, MockDNSServer +# https://stackoverflow.com/a/32784190 +def createResolver(servers=None, resolvconf=None, hosts=None): + if hosts is None: + hosts = (b'/etc/hosts' if platform.getType() == 'posix' + else r'c:\windows\hosts') + theResolver = Resolver(resolvconf, servers) + hostResolver = hostsModule.Resolver(hosts) + L = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(L) + + class LocalhostSpider(Spider): name = "localhost_spider" diff --git a/tests/mockserver.py b/tests/mockserver.py index df30feab6..1f40473ba 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -247,9 +247,8 @@ class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) - host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") - self.host = host - self.port = int(port) + self.host = '127.0.0.1' + self.port = int(self.proc.stdout.readline().strip().decode('ascii').split(":")[1]) return self def __exit__(self, exc_type, exc_value, traceback): diff --git a/tests/test_commands.py b/tests/test_commands.py index 24a341759..ee0e4511a 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import inspect import json import optparse import os +import platform import subprocess import sys import tempfile @@ -10,6 +11,7 @@ from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp from threading import Timer +from unittest import skipIf from twisted.trial import unittest @@ -319,6 +321,9 @@ class BadSpider(scrapy.Spider): self.assertIn("start_requests", log) self.assertIn("badspider.py", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_true(self): log = self.get_log(self.debug_log_spider, args=[ '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 78704fb2c..1a4cfe813 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,7 @@ import platform import subprocess import sys import warnings +from unittest import skipIf from pytest import raises, mark from testfixtures import LogCapture @@ -252,6 +253,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase): }) @defer.inlineCallbacks + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': @@ -293,11 +297,17 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) @@ -327,6 +337,9 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e38644214..f7b997560 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -455,9 +455,15 @@ class FeedExportTest(unittest.TestCase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def path_to_url(path): + return urljoin('file:', pathname2url(str(path))) + + def printf_escape(string): + return string.replace('%', '%%') + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', pathname2url(str(file_path))): feed + printf_escape(path_to_url(file_path)): feed for file_path, feed in FEEDS.items() } diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index eb4ecc91d..fc5658ae7 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,5 +1,6 @@ import json import os +import platform import re import sys from subprocess import Popen, PIPE @@ -59,6 +60,8 @@ def _wrong_credentials(proxy_url): @skipIf(sys.version_info < (3, 5, 4), "requires mitmproxy < 3.0.0, which these tests do not support") +@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), + "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): def setUp(self): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d922c6059..4929f1e3e 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -20,13 +20,20 @@ from scrapy.crawler import CrawlerRunner module_dir = os.path.dirname(os.path.abspath(__file__)) +def _copytree(source, target): + try: + shutil.copytree(source, target) + except shutil.Error: + pass + + class SpiderLoaderTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') self.tmpdir = tempfile.mkdtemp() self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) self.spider_loader = SpiderLoader.from_settings(settings) @@ -124,7 +131,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.tmpdir = self.mktemp() os.mkdir(self.tmpdir) self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') - shutil.copytree(orig_spiders_dir, self.spiders_dir) + _copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) @@ -134,8 +141,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_dupename_warning(self): # copy 1 spider module so as to have duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider3dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) @@ -156,10 +163,10 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name # This should issue 2 warning, 1 for each duplicate spider name - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) - shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), - os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx', 'spider2dupe.py')) with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 295323e4d..a2114bd18 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,4 +1,6 @@ -from unittest import TestCase +import platform +import sys +from unittest import skipIf, TestCase from pytest import mark @@ -12,6 +14,9 @@ class AsyncioTest(TestCase): # the result should depend only on the pytest --reactor argument self.assertEqual(is_asyncio_reactor_installed(), self.reactor_pytest == 'asyncio') + # https://twistedmatrix.com/trac/ticket/9766 + @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8), + "the asyncio reactor is broken on Windows when running Python ≥ 3.8") def test_install_asyncio_reactor(self): # this should do nothing install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/tox.ini b/tox.ini index 27d21ade2..f729327ca 100644 --- a/tox.ini +++ b/tox.ini @@ -63,14 +63,12 @@ basepython = pypy3 commands = py.test {posargs:--durations=10 docs scrapy tests} -[testenv:pinned] -basepython = python3 +[pinned] deps = -ctests/constraints.txt cryptography==2.0 cssselect==0.9.1 itemadapter==0.1.0 - lxml==3.5.0 parsel==1.5.0 Protego==0.1.15 PyDispatcher==2.0.5 @@ -85,6 +83,20 @@ deps = botocore==1.3.23 Pillow==3.4.2 +[testenv:pinned] +basepython = python3 +deps = + {[pinned]deps} + lxml==3.5.0 + +[testenv:windows-pinned] +basepython = python3 +deps = + {[pinned]deps} + # First lxml version that includes a Windows wheel for Python 3.5, so we do + # not need to build lxml from sources in a CI Windows job: + lxml==3.8.0 + [testenv:extra-deps] deps = {[testenv]deps} From eb937742566105f3525a9f76e4ae68cc18e9fd8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Fri, 3 Jul 2020 01:41:47 +0200 Subject: [PATCH 073/125] TrackrefTestCase.test_get_oldest: protect from lack of precision --- tests/test_utils_trackref.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 16e02f919..b8e8c3130 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,7 +1,10 @@ import unittest from io import StringIO +from time import sleep, time from unittest import mock +from twisted.trial.unittest import SkipTest + from scrapy.utils import trackref @@ -55,7 +58,18 @@ Foo 1 oldest: 0s ago\n\n''') def test_get_oldest(self): o1 = Foo() # NOQA + + o1_time = time() + o2 = Bar() # NOQA + + o3_time = time() + if o3_time <= o1_time: + sleep(0.01) + o3_time = time() + if o3_time <= o1_time: + raise SkipTest('time.time is not precise enough') + o3 = Foo() # NOQA self.assertIs(trackref.get_oldest('Foo'), o1) self.assertIs(trackref.get_oldest('Bar'), o2) From 6454d456d2bcab0828aba6d81d98f7393ab7e04d Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Fri, 3 Jul 2020 08:29:54 +0300 Subject: [PATCH 074/125] Make check of placeholder less strict --- docs/topics/feed-exports.rst | 11 ++++++----- scrapy/extensions/feedexport.py | 4 ++-- tests/test_feedexport.py | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 56efa80a7..0bb5f1733 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -448,11 +448,12 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)0xd`` - gets replaced by the sequence number of the batch. -By replacing ``x`` with an integer you set the number of leading zeroes to prevent -inappropriate sorting like this: [``'1'``, ``'10'``, ``'2'``]. Here are some examples: - ``%(batch_id)01d`` for the second batch gets replaced by ``2`` - ``%(batch_id)05d`` for the third batch gets replaced by ``00003`` +* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. + + Use :ref:`printf-style string formatting <python:old-string-formatting>` to + alter the number format. For example, to make the batch ID a 5-digit + number by introducing leading zeroes as needed, use ``%(batch_id)05d`` + (e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``). For instance, if your settings include:: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index adb6ea2e4..e15c1a09c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -375,9 +375,9 @@ class FeedExporter: %(batch_time)s or %(batch_id)s to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): - if values['batch_item_count'] and not re.findall(r'(%\(batch_time\)s|(%\(batch_id\)0\d*d))', uri_template): + if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id)0xd must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' + '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' ''.format(uri_template) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d20b40e2f..4e0b867a4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=Settings(settings)) def test_wrong_path(self): - """ If path is without %(batch_time)s and %(batch_id)0xd an exception must be raised """ + """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """ settings = { 'FEEDS': { self._random_temp_filename(): {'format': 'xml'}, From ec06cf79a6a7264ec3e32d7de8c4e305c2afa05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 6 Jul 2020 10:47:11 +0200 Subject: [PATCH 075/125] Update tests/CrawlerRunner/ip_address.py Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- tests/CrawlerRunner/ip_address.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index ea75bc3c9..b8254afdf 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -15,8 +15,7 @@ from tests.mockserver import MockServer, MockDNSServer # https://stackoverflow.com/a/32784190 def createResolver(servers=None, resolvconf=None, hosts=None): if hosts is None: - hosts = (b'/etc/hosts' if platform.getType() == 'posix' - else r'c:\windows\hosts') + hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) L = [hostResolver, cache.CacheResolver(), theResolver] From 17aec5944cab33b3cdcd497d2362cacbf7773e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 6 Jul 2020 10:47:25 +0200 Subject: [PATCH 076/125] Update tests/CrawlerRunner/ip_address.py Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- tests/CrawlerRunner/ip_address.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index b8254afdf..3f9738798 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -18,8 +18,8 @@ def createResolver(servers=None, resolvconf=None, hosts=None): hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts' theResolver = Resolver(resolvconf, servers) hostResolver = hostsModule.Resolver(hosts) - L = [hostResolver, cache.CacheResolver(), theResolver] - return resolve.ResolverChain(L) + chain = [hostResolver, cache.CacheResolver(), theResolver] + return resolve.ResolverChain(chain) class LocalhostSpider(Spider): From f1020e0e6af064ab31b812c25bda6b0f08827222 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Mon, 6 Jul 2020 15:40:53 +0300 Subject: [PATCH 077/125] Tiny changes --- scrapy/extensions/feedexport.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e15c1a09c..21177b1b0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -305,7 +305,7 @@ class FeedExporter: :param uri: uri of the new batch to start :param feed: dict with parameters of feed :param spider: user spider - :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri + :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri """ storage = self._get_storage(uri) file = storage.open(spider) @@ -372,13 +372,13 @@ class FeedExporter: def _settings_are_valid(self): """ If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain - %(batch_time)s or %(batch_id)s to distinguish different files of partial output + %(batch_time)s or %(batch_id)d to distinguish different files of partial output """ for uri_template, values in self.feeds.items(): if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template): logger.error( - '%(batch_time)s or %(batch_id) must be in uri({}) if FEED_EXPORT_BATCH_ITEM_COUNT setting ' - 'or FEEDS.batch_item_count is specified and greater than 0. For more info see:' + '%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT ' + 'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: ' 'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count' ''.format(uri_template) ) From 770a8127e8e76d95243c1d586b4bb6113a38870a Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Tue, 7 Jul 2020 15:23:29 +0530 Subject: [PATCH 078/125] Added basic `scrapy check` tests --- tests/test_command_check.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_command_check.py diff --git a/tests/test_command_check.py b/tests/test_command_check.py new file mode 100644 index 000000000..52005a4c5 --- /dev/null +++ b/tests/test_command_check.py @@ -0,0 +1,96 @@ +from os.path import join, abspath + +from tests.test_commands import CommandTest + + +class CheckCommandTest(CommandTest): + + command = 'check' + + def setUp(self): + super(CheckCommandTest, self).setUp() + self.spider_name = 'check_spider' + self.spider = abspath(join(self.proj_mod_path, 'spiders', 'checkspider.py')) + + def _write_contract(self, contracts, parse_def): + with open(self.spider, 'w') as file: + file.write(f""" +import scrapy + +class CheckSpider(scrapy.Spider): + name = '{self.spider_name}' + start_urls = ['http://example.com'] + + def parse(self, response, **cb_kwargs): + \"\"\" + @url http://www.amazon.com/s?field-keywords=selfish+gene + {contracts} + \"\"\" + {parse_def} + """) + + def _test_contract(self, contracts='', parse_def='pass'): + self._write_contract(contracts, parse_def) + p, out, err = self.proc('check') + self.assertIn('OK', err) + self.assertEqual(p.returncode, 0) + + def test_check_returns_requests_contract(self): + contracts = """ + @returns requests 1 + """ + parse_def = """ + yield scrapy.Request(url='http://next-url.com') + """ + self._test_contract(contracts, parse_def) + + def test_check_returns_items_contract(self): + contracts = """ + @returns items 1 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_cb_kwargs_contract(self): + contracts = """ + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_check_scrapes_contract(self): + contracts = """ + @scrapes key1 key2 + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + """ + self._test_contract(contracts, parse_def) + + def test_check_all_default_contracts(self): + contracts = """ + @returns items 1 + @returns requests 1 + @scrapes key1 key2 + @cb_kwargs {"arg1": "val1", "arg2": "val2"} + """ + parse_def = """ + yield {'key1': 'val1', 'key2': 'val2'} + yield scrapy.Request(url='http://next-url.com') + if len(cb_kwargs.items()) == 0: + raise Exception("Callback args not set") + """ + self._test_contract(contracts, parse_def) + + def test_SCRAPY_CHECK_set(self): + parse_def = """ + import os + if not os.environ.get('SCRAPY_CHECK'): + raise Exception('SCRAPY_CHECK not set') + """ + self._test_contract(parse_def=parse_def) From d014840672820b3970282f31a51b9ff24cd46bd3 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Tue, 7 Jul 2020 15:24:33 +0530 Subject: [PATCH 079/125] Ignore flake8 E501 for `scrapy check` tests` --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index bae68cd3a..97320a008 100644 --- a/pytest.ini +++ b/pytest.ini @@ -173,6 +173,7 @@ flake8-ignore = tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 + tests/test_command_check.py E501 tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 From 79b4dfc53e431bdad31925aaf16831a0dde536ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:07:04 +0200 Subject: [PATCH 080/125] Fix permission handling on project generation from template files --- scrapy/commands/startproject.py | 30 +--- .../project/module/__init__.py | 0 .../project/module/items.py.tmpl | 12 ++ .../project/module/middlewares.py.tmpl | 103 +++++++++++ .../project/module/pipelines.py.tmpl | 13 ++ .../project/module/settings.py.tmpl | 88 ++++++++++ .../project/module/spiders/__init__.py | 4 + .../read_only_templates/project/scrapy.cfg | 11 ++ .../read_only_templates/spiders/basic.tmpl | 10 ++ .../read_only_templates/spiders/crawl.tmpl | 20 +++ .../read_only_templates/spiders/csvfeed.tmpl | 20 +++ .../read_only_templates/spiders/xmlfeed.tmpl | 16 ++ tests/test_commands.py | 164 ++++++++++++++++++ 13 files changed, 467 insertions(+), 24 deletions(-) create mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py create mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl create mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py create mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg create mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl create mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl create mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl create mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 852281959..e702d7cdc 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,10 +1,10 @@ import re import os -import stat import string from importlib import import_module from os.path import join, exists, abspath from shutil import ignore_patterns, move, copy2, copystat +from stat import S_IWUSR as OWNER_WRITE_PERMISSION import scrapy from scrapy.commands import ScrapyCommand @@ -78,30 +78,12 @@ class Command(ScrapyCommand): self._copytree(srcname, dstname) else: copy2(srcname, dstname) + current_permissions = os.stat(dstname).st_mode + os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION) + copystat(src, dst) - self._set_rw_permissions(dst) - - def _set_rw_permissions(self, path): - """ - Sets permissions of a directory tree to +rw and +rwx for folders. - This is necessary if the start template files come without write - permissions. - """ - mode_rw = (stat.S_IRUSR - | stat.S_IWUSR - | stat.S_IRGRP - | stat.S_IROTH) - - mode_x = (stat.S_IXUSR - | stat.S_IXGRP - | stat.S_IXOTH) - - os.chmod(path, mode_rw | mode_x) - for root, dirs, files in os.walk(path): - for dir in dirs: - os.chmod(join(root, dir), mode_rw | mode_x) - for file in files: - os.chmod(join(root, file), mode_rw) + current_permissions = os.stat(dst).st_mode + os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION) def run(self, args, opts): if len(args) not in (1, 2): diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl new file mode 100644 index 000000000..88a18331c --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/items.py.tmpl @@ -0,0 +1,12 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class ${ProjectName}Item(scrapy.Item): + # define the fields for your item here like: + # name = scrapy.Field() + pass diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl new file mode 100644 index 000000000..bd09890fe --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl @@ -0,0 +1,103 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + + +class ${ProjectName}SpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class ${ProjectName}DownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl new file mode 100644 index 000000000..e845f43e9 --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl @@ -0,0 +1,13 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html + + +# useful for handling different item types with a single interface +from itemadapter import ItemAdapter + + +class ${ProjectName}Pipeline: + def process_item(self, item, spider): + return item diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl new file mode 100644 index 000000000..a414b5fde --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl @@ -0,0 +1,88 @@ +# Scrapy settings for $project_name project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = '$project_name' + +SPIDER_MODULES = ['$project_name.spiders'] +NEWSPIDER_MODULE = '$project_name.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = '$project_name (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# '$project_name.pipelines.${ProjectName}Pipeline': 300, +#} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py new file mode 100644 index 000000000..ebd689ac5 --- /dev/null +++ b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg new file mode 100644 index 000000000..1daeaa541 --- /dev/null +++ b/tests/sample_data/read_only_templates/project/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = ${project_name}.settings + +[deploy] +#url = http://localhost:6800/ +project = ${project_name} diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl new file mode 100644 index 000000000..e9112bc95 --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/basic.tmpl @@ -0,0 +1,10 @@ +import scrapy + + +class $classname(scrapy.Spider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/'] + + def parse(self, response): + pass diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl new file mode 100644 index 000000000..356496487 --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/crawl.tmpl @@ -0,0 +1,20 @@ +import scrapy +from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import CrawlSpider, Rule + + +class $classname(CrawlSpider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/'] + + rules = ( + Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), + ) + + def parse_item(self, response): + item = {} + #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() + #item['name'] = response.xpath('//div[@id="name"]').get() + #item['description'] = response.xpath('//div[@id="description"]').get() + return item diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl new file mode 100644 index 000000000..cbcbe9e2c --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl @@ -0,0 +1,20 @@ +from scrapy.spiders import CSVFeedSpider + + +class $classname(CSVFeedSpider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/feed.csv'] + # headers = ['id', 'name', 'description', 'image_link'] + # delimiter = '\t' + + # Do any adaptations you need here + #def adapt_response(self, response): + # return response + + def parse_row(self, response, row): + i = {} + #i['url'] = row['url'] + #i['name'] = row['name'] + #i['description'] = row['description'] + return i diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl new file mode 100644 index 000000000..5aa2aa8b0 --- /dev/null +++ b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl @@ -0,0 +1,16 @@ +from scrapy.spiders import XMLFeedSpider + + +class $classname(XMLFeedSpider): + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://$domain/feed.xml'] + iterator = 'iternodes' # you can change this; see the docs + itertag = 'item' # change it accordingly + + def parse_node(self, response, selector): + item = {} + #item['url'] = selector.select('url').get() + #item['name'] = selector.select('name').get() + #item['description'] = selector.select('description').get() + return item diff --git a/tests/test_commands.py b/tests/test_commands.py index 24a341759..8336c8759 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,7 +6,9 @@ import subprocess import sys import tempfile from contextlib import contextmanager +from itertools import chain from os.path import exists, join, abspath +from pathlib import Path from shutil import rmtree, copytree from tempfile import mkdtemp from threading import Timer @@ -15,6 +17,7 @@ from twisted.trial import unittest import scrapy from scrapy.commands import ScrapyCommand +from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv @@ -119,8 +122,34 @@ class StartprojectTest(ProjectTest): self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params')) +def get_permissions_dict(path, renamings=None, ignore=None): + renamings = renamings or tuple() + permissions_dict = { + '.': os.stat(path).st_mode, + } + for root, dirs, files in os.walk(path): + nodes = list(chain(dirs, files)) + if ignore: + ignored_names = ignore(root, nodes) + nodes = [node for node in nodes + if node not in ignored_names] + for node in nodes: + absolute_path = os.path.join(root, node) + relative_path = os.path.relpath(absolute_path, path) + for search_string, replacement in renamings: + relative_path = relative_path.replace( + search_string, + replacement + ) + permissions = os.stat(absolute_path).st_mode + permissions_dict[relative_path] = permissions + return permissions_dict + + class StartprojectTemplatesTest(ProjectTest): + maxDiff = None + def setUp(self): super(StartprojectTemplatesTest, self).setUp() self.tmpl = join(self.temp_path, 'templates') @@ -139,6 +168,141 @@ class StartprojectTemplatesTest(ProjectTest): self.assertIn(self.tmpl_proj, out) assert exists(join(self.proj_path, 'root_template')) + def test_startproject_permissions_from_writable(self): + """Check that generated files have the right permissions when the + template folder has the same permissions as in the project, i.e. + everything is writable.""" + scrapy_path = scrapy.__path__[0] + templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_name = 'startproject1' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + templates_dir, + renamings, + IGNORE, + ) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + + def test_startproject_permissions_from_read_only(self): + """Check that generated files have the right permissions when the + template folder has been made read-only, which is something that some + systems do. + + See https://github.com/scrapy/scrapy/pull/4604 + """ + scrapy_path = scrapy.__path__[0] + templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_name = 'startproject2' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + templates_dir, + renamings, + IGNORE, + ) + + tests_path = os.path.dirname(__file__) + read_only_templates_dir = os.path.join( + tests_path, 'sample_data', 'read_only_templates' + ) + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + '--set', + 'TEMPLATES_DIR={}'.format(read_only_templates_dir), + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = os.path.join(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + + def test_startproject_permissions_unchanged_in_destination(self): + """Check that pre-existing folders and files in the destination folder + do not see their permissions modified.""" + scrapy_path = scrapy.__path__[0] + templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_name = 'startproject3' + renamings = ( + ('module', project_name), + ('.tmpl', ''), + ) + expected_permissions = get_permissions_dict( + templates_dir, + renamings, + IGNORE, + ) + + destination = mkdtemp() + project_dir = os.path.join(destination, project_name) + + existing_nodes = { + oct(permissions)[2:] + extension: permissions + for extension in ('', '.d') + for permissions in ( + 0o444, 0o555, 0o644, 0o666, 0o755, 0o777, + ) + } + os.mkdir(project_dir) + project_dir_path = Path(project_dir) + for node, permissions in existing_nodes.items(): + path = project_dir_path / node + if node.endswith('.d'): + path.mkdir(mode=permissions) + else: + path.touch(mode=permissions) + expected_permissions[node] = path.stat().st_mode + + process = subprocess.Popen( + ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'startproject', + project_name, + '.', + ), + cwd=project_dir, + env=self.env, + ) + process.wait() + + actual_permissions = get_permissions_dict(project_dir) + + self.assertEqual(actual_permissions, expected_permissions) + class CommandTest(ProjectTest): From a3afff4a0e1b25903d1de5c5501846d1f1288d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:11:02 +0200 Subject: [PATCH 081/125] Fix style issue --- tests/test_commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 8336c8759..bd799817d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -131,8 +131,7 @@ def get_permissions_dict(path, renamings=None, ignore=None): nodes = list(chain(dirs, files)) if ignore: ignored_names = ignore(root, nodes) - nodes = [node for node in nodes - if node not in ignored_names] + nodes = [node for node in nodes if node not in ignored_names] for node in nodes: absolute_path = os.path.join(root, node) relative_path = os.path.relpath(absolute_path, path) From e1450799ce2dfa32e248cb0b4069668ee5e6f4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:11:37 +0200 Subject: [PATCH 082/125] Remove debug test case variable --- tests/test_commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index bd799817d..c25495d16 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -147,8 +147,6 @@ def get_permissions_dict(path, renamings=None, ignore=None): class StartprojectTemplatesTest(ProjectTest): - maxDiff = None - def setUp(self): super(StartprojectTemplatesTest, self).setUp() self.tmpl = join(self.temp_path, 'templates') From ca77ca1f751a614b0c9394d1e8c6d0b9cfcdd957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 14:44:03 +0200 Subject: [PATCH 083/125] Generate read-only files on the fly --- scrapy/commands/startproject.py | 13 ++- .../project/module/__init__.py | 0 .../project/module/items.py.tmpl | 12 -- .../project/module/middlewares.py.tmpl | 103 ------------------ .../project/module/pipelines.py.tmpl | 13 --- .../project/module/settings.py.tmpl | 88 --------------- .../project/module/spiders/__init__.py | 4 - .../read_only_templates/project/scrapy.cfg | 11 -- .../read_only_templates/spiders/basic.tmpl | 10 -- .../read_only_templates/spiders/crawl.tmpl | 20 ---- .../read_only_templates/spiders/csvfeed.tmpl | 20 ---- .../read_only_templates/spiders/xmlfeed.tmpl | 16 --- tests/test_commands.py | 31 ++++-- 13 files changed, 28 insertions(+), 313 deletions(-) delete mode 100644 tests/sample_data/read_only_templates/project/module/__init__.py delete mode 100644 tests/sample_data/read_only_templates/project/module/items.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/settings.py.tmpl delete mode 100644 tests/sample_data/read_only_templates/project/module/spiders/__init__.py delete mode 100644 tests/sample_data/read_only_templates/project/scrapy.cfg delete mode 100644 tests/sample_data/read_only_templates/spiders/basic.tmpl delete mode 100644 tests/sample_data/read_only_templates/spiders/crawl.tmpl delete mode 100644 tests/sample_data/read_only_templates/spiders/csvfeed.tmpl delete mode 100644 tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e702d7cdc..eccc2a3e1 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = ( ('${project_name}', 'middlewares.py.tmpl'), ) -IGNORE = ignore_patterns('*.pyc', '.svn') +IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn') + + +def _make_writable(path): + current_permissions = os.stat(path).st_mode + os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION) class Command(ScrapyCommand): @@ -78,12 +83,10 @@ class Command(ScrapyCommand): self._copytree(srcname, dstname) else: copy2(srcname, dstname) - current_permissions = os.stat(dstname).st_mode - os.chmod(dstname, current_permissions | OWNER_WRITE_PERMISSION) + _make_writable(dstname) copystat(src, dst) - current_permissions = os.stat(dst).st_mode - os.chmod(dst, current_permissions | OWNER_WRITE_PERMISSION) + _make_writable(dst) def run(self, args, opts): if len(args) not in (1, 2): diff --git a/tests/sample_data/read_only_templates/project/module/__init__.py b/tests/sample_data/read_only_templates/project/module/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/sample_data/read_only_templates/project/module/items.py.tmpl b/tests/sample_data/read_only_templates/project/module/items.py.tmpl deleted file mode 100644 index 88a18331c..000000000 --- a/tests/sample_data/read_only_templates/project/module/items.py.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -# Define here the models for your scraped items -# -# See documentation in: -# https://docs.scrapy.org/en/latest/topics/items.html - -import scrapy - - -class ${ProjectName}Item(scrapy.Item): - # define the fields for your item here like: - # name = scrapy.Field() - pass diff --git a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl b/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl deleted file mode 100644 index bd09890fe..000000000 --- a/tests/sample_data/read_only_templates/project/module/middlewares.py.tmpl +++ /dev/null @@ -1,103 +0,0 @@ -# Define here the models for your spider middleware -# -# See documentation in: -# https://docs.scrapy.org/en/latest/topics/spider-middleware.html - -from scrapy import signals - -# useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter - - -class ${ProjectName}SpiderMiddleware: - # Not all methods need to be defined. If a method is not defined, - # scrapy acts as if the spider middleware does not modify the - # passed objects. - - @classmethod - def from_crawler(cls, crawler): - # This method is used by Scrapy to create your spiders. - s = cls() - crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) - return s - - def process_spider_input(self, response, spider): - # Called for each response that goes through the spider - # middleware and into the spider. - - # Should return None or raise an exception. - return None - - def process_spider_output(self, response, result, spider): - # Called with the results returned from the Spider, after - # it has processed the response. - - # Must return an iterable of Request, or item objects. - for i in result: - yield i - - def process_spider_exception(self, response, exception, spider): - # Called when a spider or process_spider_input() method - # (from other spider middleware) raises an exception. - - # Should return either None or an iterable of Request or item objects. - pass - - def process_start_requests(self, start_requests, spider): - # Called with the start requests of the spider, and works - # similarly to the process_spider_output() method, except - # that it doesn’t have a response associated. - - # Must return only requests (not items). - for r in start_requests: - yield r - - def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) - - -class ${ProjectName}DownloaderMiddleware: - # Not all methods need to be defined. If a method is not defined, - # scrapy acts as if the downloader middleware does not modify the - # passed objects. - - @classmethod - def from_crawler(cls, crawler): - # This method is used by Scrapy to create your spiders. - s = cls() - crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) - return s - - def process_request(self, request, spider): - # Called for each request that goes through the downloader - # middleware. - - # Must either: - # - return None: continue processing this request - # - or return a Response object - # - or return a Request object - # - or raise IgnoreRequest: process_exception() methods of - # installed downloader middleware will be called - return None - - def process_response(self, request, response, spider): - # Called with the response returned from the downloader. - - # Must either; - # - return a Response object - # - return a Request object - # - or raise IgnoreRequest - return response - - def process_exception(self, request, exception, spider): - # Called when a download handler or a process_request() - # (from other downloader middleware) raises an exception. - - # Must either: - # - return None: continue processing this exception - # - return a Response object: stops process_exception() chain - # - return a Request object: stops process_exception() chain - pass - - def spider_opened(self, spider): - spider.logger.info('Spider opened: %s' % spider.name) diff --git a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl b/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl deleted file mode 100644 index e845f43e9..000000000 --- a/tests/sample_data/read_only_templates/project/module/pipelines.py.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -# Define your item pipelines here -# -# Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html - - -# useful for handling different item types with a single interface -from itemadapter import ItemAdapter - - -class ${ProjectName}Pipeline: - def process_item(self, item, spider): - return item diff --git a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl b/tests/sample_data/read_only_templates/project/module/settings.py.tmpl deleted file mode 100644 index a414b5fde..000000000 --- a/tests/sample_data/read_only_templates/project/module/settings.py.tmpl +++ /dev/null @@ -1,88 +0,0 @@ -# Scrapy settings for $project_name project -# -# For simplicity, this file contains only settings considered important or -# commonly used. You can find more settings consulting the documentation: -# -# https://docs.scrapy.org/en/latest/topics/settings.html -# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -# https://docs.scrapy.org/en/latest/topics/spider-middleware.html - -BOT_NAME = '$project_name' - -SPIDER_MODULES = ['$project_name.spiders'] -NEWSPIDER_MODULE = '$project_name.spiders' - - -# Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = '$project_name (+http://www.yourdomain.com)' - -# Obey robots.txt rules -ROBOTSTXT_OBEY = True - -# Configure maximum concurrent requests performed by Scrapy (default: 16) -#CONCURRENT_REQUESTS = 32 - -# Configure a delay for requests for the same website (default: 0) -# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay -# See also autothrottle settings and docs -#DOWNLOAD_DELAY = 3 -# The download delay setting will honor only one of: -#CONCURRENT_REQUESTS_PER_DOMAIN = 16 -#CONCURRENT_REQUESTS_PER_IP = 16 - -# Disable cookies (enabled by default) -#COOKIES_ENABLED = False - -# Disable Telnet Console (enabled by default) -#TELNETCONSOLE_ENABLED = False - -# Override the default request headers: -#DEFAULT_REQUEST_HEADERS = { -# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -# 'Accept-Language': 'en', -#} - -# Enable or disable spider middlewares -# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html -#SPIDER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, -#} - -# Enable or disable downloader middlewares -# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -#DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, -#} - -# Enable or disable extensions -# See https://docs.scrapy.org/en/latest/topics/extensions.html -#EXTENSIONS = { -# 'scrapy.extensions.telnet.TelnetConsole': None, -#} - -# Configure item pipelines -# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html -#ITEM_PIPELINES = { -# '$project_name.pipelines.${ProjectName}Pipeline': 300, -#} - -# Enable and configure the AutoThrottle extension (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/autothrottle.html -#AUTOTHROTTLE_ENABLED = True -# The initial download delay -#AUTOTHROTTLE_START_DELAY = 5 -# The maximum download delay to be set in case of high latencies -#AUTOTHROTTLE_MAX_DELAY = 60 -# The average number of requests Scrapy should be sending in parallel to -# each remote server -#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 -# Enable showing throttling stats for every response received: -#AUTOTHROTTLE_DEBUG = False - -# Enable and configure HTTP caching (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings -#HTTPCACHE_ENABLED = True -#HTTPCACHE_EXPIRATION_SECS = 0 -#HTTPCACHE_DIR = 'httpcache' -#HTTPCACHE_IGNORE_HTTP_CODES = [] -#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py b/tests/sample_data/read_only_templates/project/module/spiders/__init__.py deleted file mode 100644 index ebd689ac5..000000000 --- a/tests/sample_data/read_only_templates/project/module/spiders/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# This package will contain the spiders of your Scrapy project -# -# Please refer to the documentation for information on how to create and manage -# your spiders. diff --git a/tests/sample_data/read_only_templates/project/scrapy.cfg b/tests/sample_data/read_only_templates/project/scrapy.cfg deleted file mode 100644 index 1daeaa541..000000000 --- a/tests/sample_data/read_only_templates/project/scrapy.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Automatically created by: scrapy startproject -# -# For more information about the [deploy] section see: -# https://scrapyd.readthedocs.io/en/latest/deploy.html - -[settings] -default = ${project_name}.settings - -[deploy] -#url = http://localhost:6800/ -project = ${project_name} diff --git a/tests/sample_data/read_only_templates/spiders/basic.tmpl b/tests/sample_data/read_only_templates/spiders/basic.tmpl deleted file mode 100644 index e9112bc95..000000000 --- a/tests/sample_data/read_only_templates/spiders/basic.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -import scrapy - - -class $classname(scrapy.Spider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] - - def parse(self, response): - pass diff --git a/tests/sample_data/read_only_templates/spiders/crawl.tmpl b/tests/sample_data/read_only_templates/spiders/crawl.tmpl deleted file mode 100644 index 356496487..000000000 --- a/tests/sample_data/read_only_templates/spiders/crawl.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -import scrapy -from scrapy.linkextractors import LinkExtractor -from scrapy.spiders import CrawlSpider, Rule - - -class $classname(CrawlSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/'] - - rules = ( - Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), - ) - - def parse_item(self, response): - item = {} - #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() - #item['name'] = response.xpath('//div[@id="name"]').get() - #item['description'] = response.xpath('//div[@id="description"]').get() - return item diff --git a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl b/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl deleted file mode 100644 index cbcbe9e2c..000000000 --- a/tests/sample_data/read_only_templates/spiders/csvfeed.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -from scrapy.spiders import CSVFeedSpider - - -class $classname(CSVFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.csv'] - # headers = ['id', 'name', 'description', 'image_link'] - # delimiter = '\t' - - # Do any adaptations you need here - #def adapt_response(self, response): - # return response - - def parse_row(self, response, row): - i = {} - #i['url'] = row['url'] - #i['name'] = row['name'] - #i['description'] = row['description'] - return i diff --git a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl b/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl deleted file mode 100644 index 5aa2aa8b0..000000000 --- a/tests/sample_data/read_only_templates/spiders/xmlfeed.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -from scrapy.spiders import XMLFeedSpider - - -class $classname(XMLFeedSpider): - name = '$name' - allowed_domains = ['$domain'] - start_urls = ['http://$domain/feed.xml'] - iterator = 'iternodes' # you can change this; see the docs - itertag = 'item' # change it accordingly - - def parse_node(self, response, selector): - item = {} - #item['url'] = selector.select('url').get() - #item['name'] = selector.select('name').get() - #item['description'] = selector.select('description').get() - return item diff --git a/tests/test_commands.py b/tests/test_commands.py index c25495d16..f3fe45139 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,6 +2,7 @@ import inspect import json import optparse import os +from stat import S_IWRITE as ANYONE_WRITE_PERMISSION import subprocess import sys import tempfile @@ -17,7 +18,7 @@ from twisted.trial import unittest import scrapy from scrapy.commands import ScrapyCommand -from scrapy.commands.startproject import IGNORE +from scrapy.commands.startproject import IGNORE, _make_writable from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv @@ -170,14 +171,14 @@ class StartprojectTemplatesTest(ProjectTest): template folder has the same permissions as in the project, i.e. everything is writable.""" scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_template = os.path.join(scrapy_path, 'templates', 'project') project_name = 'startproject1' renamings = ( ('module', project_name), ('.tmpl', ''), ) expected_permissions = get_permissions_dict( - templates_dir, + project_template, renamings, IGNORE, ) @@ -209,22 +210,30 @@ class StartprojectTemplatesTest(ProjectTest): See https://github.com/scrapy/scrapy/pull/4604 """ scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates', 'project') + templates_dir = os.path.join(scrapy_path, 'templates') + project_template = os.path.join(templates_dir, 'project') project_name = 'startproject2' renamings = ( ('module', project_name), ('.tmpl', ''), ) expected_permissions = get_permissions_dict( - templates_dir, + project_template, renamings, IGNORE, ) - tests_path = os.path.dirname(__file__) - read_only_templates_dir = os.path.join( - tests_path, 'sample_data', 'read_only_templates' - ) + def _make_read_only(path): + current_permissions = os.stat(path).st_mode + os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION) + + read_only_templates_dir = str(Path(mkdtemp()) / 'templates') + copytree(templates_dir, read_only_templates_dir) + + for root, dirs, files in os.walk(read_only_templates_dir): + for node in chain(dirs, files): + _make_read_only(os.path.join(root, node)) + destination = mkdtemp() process = subprocess.Popen( ( @@ -250,14 +259,14 @@ class StartprojectTemplatesTest(ProjectTest): """Check that pre-existing folders and files in the destination folder do not see their permissions modified.""" scrapy_path = scrapy.__path__[0] - templates_dir = os.path.join(scrapy_path, 'templates', 'project') + project_template = os.path.join(scrapy_path, 'templates', 'project') project_name = 'startproject3' renamings = ( ('module', project_name), ('.tmpl', ''), ) expected_permissions = get_permissions_dict( - templates_dir, + project_template, renamings, IGNORE, ) From 7e386157033e69bccc56e3a34d5545daa0174d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Tue, 7 Jul 2020 15:30:19 +0200 Subject: [PATCH 084/125] Remove unused import --- tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index f3fe45139..002237824 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -18,7 +18,7 @@ from twisted.trial import unittest import scrapy from scrapy.commands import ScrapyCommand -from scrapy.commands.startproject import IGNORE, _make_writable +from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv From 3e98ed24b6e9189d7fc7b4209d24971068274ddb Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 8 Jul 2020 17:13:57 +0530 Subject: [PATCH 085/125] Convert f-string to .format() --- tests/test_command_check.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 52005a4c5..72acd817c 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -14,20 +14,20 @@ class CheckCommandTest(CommandTest): def _write_contract(self, contracts, parse_def): with open(self.spider, 'w') as file: - file.write(f""" + file.write(""" import scrapy class CheckSpider(scrapy.Spider): - name = '{self.spider_name}' + name = '{0}' start_urls = ['http://example.com'] def parse(self, response, **cb_kwargs): \"\"\" @url http://www.amazon.com/s?field-keywords=selfish+gene - {contracts} + {1} \"\"\" - {parse_def} - """) + {2} + """.format(self.spider_name, contracts, parse_def)) def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) From 75bff7b6d33bdc74c1a8eb0e43e4b484473c3062 Mon Sep 17 00:00:00 2001 From: ajaymittur28 <ajay.cs18@bmsce.ac.in> Date: Wed, 8 Jul 2020 19:48:42 +0530 Subject: [PATCH 086/125] Update url contract value --- tests/test_command_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 72acd817c..f27f526a3 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -23,7 +23,7 @@ class CheckSpider(scrapy.Spider): def parse(self, response, **cb_kwargs): \"\"\" - @url http://www.amazon.com/s?field-keywords=selfish+gene + @url http://example.com {1} \"\"\" {2} @@ -32,6 +32,7 @@ class CheckSpider(scrapy.Spider): def _test_contract(self, contracts='', parse_def='pass'): self._write_contract(contracts, parse_def) p, out, err = self.proc('check') + self.assertNotIn('F', out) self.assertIn('OK', err) self.assertEqual(p.returncode, 0) From 9e99be982a2916559219693e6665ec7e9319f0e9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 17 Jun 2020 15:52:57 -0300 Subject: [PATCH 087/125] Remove backslash --- scrapy/cmdline.py | 10 ++++++---- scrapy/commands/crawl.py | 6 ++++-- scrapy/commands/fetch.py | 6 ++++-- scrapy/commands/genspider.py | 7 ++++--- scrapy/commands/startproject.py | 7 ++++--- scrapy/commands/view.py | 3 +-- scrapy/core/engine.py | 8 +++++--- scrapy/downloadermiddlewares/redirect.py | 13 +++++++------ scrapy/downloadermiddlewares/retry.py | 6 ++++-- scrapy/extensions/memusage.py | 12 ++++++++---- scrapy/http/request/form.py | 3 +-- scrapy/http/response/text.py | 5 ++++- scrapy/link.py | 14 ++++++++++---- scrapy/settings/__init__.py | 3 +-- scrapy/utils/benchserver.py | 3 +-- scrapy/utils/conf.py | 12 +++++++----- scrapy/utils/curl.py | 3 +-- scrapy/utils/ossignal.py | 3 +-- scrapy/utils/response.py | 16 ++++++++++------ scrapy/utils/spider.py | 10 ++++++---- 20 files changed, 89 insertions(+), 61 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b189e016b..3e88536e4 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -19,10 +19,12 @@ def _iter_command_classes(module_name): # scrapy.utils.spider.iter_spider_classes for module in walk_modules(module_name): for obj in vars(module).values(): - if inspect.isclass(obj) and \ - issubclass(obj, ScrapyCommand) and \ - obj.__module__ == module.__name__ and \ - not obj == ScrapyCommand: + if ( + inspect.isclass(obj) + and issubclass(obj, ScrapyCommand) + and obj.__module__ == module.__name__ + and not obj == ScrapyCommand + ): yield obj diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index e1724c1e6..f205c40b0 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand): else: self.crawler_process.start() - if self.crawler_process.bootstrap_failed or \ - (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception): + if ( + self.crawler_process.bootstrap_failed + or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception + ): self.exitcode = 1 diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 063195f50..95f87e8c3 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -19,8 +19,10 @@ class Command(ScrapyCommand): return "Fetch a URL using the Scrapy downloader" def long_desc(self): - return "Fetch a URL using the Scrapy downloader and print its content " \ - "to stdout. You may want to use --nolog to disable logging" + return ( + "Fetch a URL using the Scrapy downloader and print its content" + " to stdout. You may want to use --nolog to disable logging" + ) def add_options(self, parser): ScrapyCommand.add_options(self, parser) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index abf3b7a5c..4c7548e9c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -121,6 +121,7 @@ class Command(ScrapyCommand): @property def templates_dir(self): - _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ - join(scrapy.__path__[0], 'templates') - return join(_templates_base_dir, 'spiders') + return join( + self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + 'spiders' + ) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 852281959..ae4a15b0f 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -137,6 +137,7 @@ class Command(ScrapyCommand): @property def templates_dir(self): - _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ - join(scrapy.__path__[0], 'templates') - return join(_templates_base_dir, 'project') + return join( + self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'), + 'project' + ) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 41e77ba3b..908bee966 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -8,8 +8,7 @@ class Command(fetch.Command): return "Open URL in browser, as seen by Scrapy" def long_desc(self): - return "Fetch a URL using the Scrapy downloader and show its " \ - "contents in a browser" + return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): super(Command, self).add_options(parser) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index de0da4b70..86a6abb23 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -141,10 +141,12 @@ class ExecutionEngine: def _needs_backout(self, spider): slot = self.slot - return not self.running \ - or slot.closing \ - or self.downloader.needs_backout() \ + return ( + not self.running + or slot.closing + or self.downloader.needs_backout() or self.scraper.slot.needs_backout() + ) def _next_request_from_scheduler(self, spider): slot = self.slot diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index b32afb8e4..366d60dcb 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -33,10 +33,8 @@ class BaseRedirectMiddleware: if ttl and redirects <= self.max_redirect_times: redirected.meta['redirect_times'] = redirects redirected.meta['redirect_ttl'] = ttl - 1 - redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \ - [request.url] - redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \ - [reason] + redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url] + redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason] redirected.dont_filter = request.dont_filter redirected.priority = request.priority + self.priority_adjust logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s", @@ -99,8 +97,11 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') def process_response(self, request, response, spider): - if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \ - not isinstance(response, HtmlResponse): + if ( + request.meta.get('dont_redirect', False) + or request.method == 'HEAD' + or not isinstance(response, HtmlResponse) + ): return response interval, url = get_meta_refresh(response, diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 6d11af5b2..67be8c282 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -60,8 +60,10 @@ class RetryMiddleware: return response def process_exception(self, request, exception, spider): - if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \ - and not request.meta.get('dont_retry', False): + if ( + isinstance(exception, self.EXCEPTIONS_TO_RETRY) + and not request.meta.get('dont_retry', False) + ): return self._retry(request, exception, spider) def _retry(self, request, reason, spider): diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index a0540bf8f..ab2e43e8c 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -81,8 +81,10 @@ class MemoryUsage: logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: - subj = "%s terminated: memory usage exceeded %dM at %s" % \ - (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + subj = ( + "%s terminated: memory usage exceeded %dM at %s" + % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/limit_notified', 1) @@ -102,8 +104,10 @@ class MemoryUsage: logger.warning("Memory usage reached %(memusage)dM", {'memusage': mem}, extra={'crawler': self.crawler}) if self.notify_mails: - subj = "%s warning: memory usage reached %dM at %s" % \ - (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + subj = ( + "%s warning: memory usage reached %dM at %s" + % (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) + ) self._send_report(self.notify_mails, subj) self.crawler.stats.set_value('memusage/warning_notified', 1) self.warned = True diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index cd4e3373f..0e6ceef0b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -205,8 +205,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = u'.//*' + \ - u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) + xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index b43fe5c19..0f300c8da 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -62,8 +62,11 @@ class TextResponse(Response): return self._declared_encoding() or self._body_inferred_encoding() def _declared_encoding(self): - return self._encoding or self._headers_encoding() \ + return ( + self._encoding + or self._headers_encoding() or self._body_declared_encoding() + ) def body_as_unicode(self): """Return body as unicode""" diff --git a/scrapy/link.py b/scrapy/link.py index 7cb0765cc..1ef50b113 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -21,12 +21,18 @@ class Link: self.nofollow = nofollow def __eq__(self, other): - return self.url == other.url and self.text == other.text and \ - self.fragment == other.fragment and self.nofollow == other.nofollow + return ( + self.url == other.url + and self.text == other.text + and self.fragment == other.fragment + and self.nofollow == other.nofollow + ) def __hash__(self): return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow) def __repr__(self): - return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \ - (self.url, self.text, self.fragment, self.nofollow) + return ( + 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' + % (self.url, self.text, self.fragment, self.nofollow) + ) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b9a13c018..ff8317cd1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -52,8 +52,7 @@ class SettingsAttribute: self.priority = priority def __str__(self): - return "<SettingsAttribute value={self.value!r} " \ - "priority={self.priority}>".format(self=self) + return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self) __repr__ = __str__ diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 9d8d64612..f595a1acb 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -28,8 +28,7 @@ class Root(Resource): def _getarg(request, name, default=None, type=str): - return type(request.args[name][0]) \ - if name in request.args else default + return type(request.args[name][0]) if name in request.args else default if __name__ == '__main__': diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5921f82bf..728bb5f1b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -101,11 +101,13 @@ def get_config(use_closest=True): def get_sources(use_closest=True): - xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \ - os.path.expanduser('~/.config') - sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg', - xdg_config_home + '/scrapy.cfg', - os.path.expanduser('~/.scrapy.cfg')] + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config') + sources = [ + '/etc/scrapy.cfg', + r'c:\scrapy\scrapy.cfg', + xdg_config_home + '/scrapy.cfg', + os.path.expanduser('~/.scrapy.cfg'), + ] if use_closest: sources.append(closest_scrapy_cfg()) return sources diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 67b22dbc5..aa681522f 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -9,8 +9,7 @@ from w3lib.http import basic_auth_header class CurlParser(argparse.ArgumentParser): def error(self, message): - error_msg = \ - 'There was an error parsing the curl command: {}'.format(message) + error_msg = 'There was an error parsing the curl command: {}'.format(message) raise ValueError(error_msg) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index 45c9cef0c..cf867f3f8 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True): from twisted.internet import reactor reactor._handleSignals() signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \ - override_sigint: + if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, 'SIGBREAK'): diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index edbc0db25..c29b619ce 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -47,13 +47,17 @@ def response_httprepr(response): is provided only for reference, since it's not the exact stream of bytes that was received (that's not exposed by Twisted). """ - s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \ - to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n" + values = [ + b"HTTP/1.1 ", + to_bytes(str(response.status)), + b" ", + to_bytes(http.RESPONSES.get(response.status, b'')), + b"\r\n", + ] if response.headers: - s += response.headers.to_string() + b"\r\n" - s += b"\r\n" - s += response.body - return s + values.extend([response.headers.to_string(), b"\r\n"]) + values.extend([b"\r\n", response.body]) + return b"".join(values) def open_in_browser(response, _openfunc=webbrowser.open): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 7e7a50c88..f3a9a67a3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -34,10 +34,12 @@ def iter_spider_classes(module): from scrapy.spiders import Spider for obj in vars(module).values(): - if inspect.isclass(obj) and \ - issubclass(obj, Spider) and \ - obj.__module__ == module.__name__ and \ - getattr(obj, 'name', None): + if ( + inspect.isclass(obj) + and issubclass(obj, Spider) + and obj.__module__ == module.__name__ + and getattr(obj, 'name', None) + ): yield obj From 9aea1f096171d38348b1403302c6c40eeef7f0a6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Thu, 9 Jul 2020 11:04:46 -0300 Subject: [PATCH 088/125] Remove backslash (tests) --- ...st_downloadermiddleware_httpcompression.py | 3 +- tests/test_downloadermiddleware_redirect.py | 18 ++++------- tests/test_selector.py | 3 +- tests/test_settings/__init__.py | 3 +- tests/test_spider.py | 10 ++++-- tests/test_spidermiddleware_referer.py | 32 +++++++++++++------ tests/test_utils_curl.py | 6 ++-- tests/test_utils_defer.py | 9 ++++-- tests/test_utils_iterators.py | 30 +++++++++-------- tests/test_utils_request.py | 8 +++-- tests/test_utils_response.py | 3 +- tests/test_utils_sitemap.py | 3 +- tests/test_utils_url.py | 3 +- tests/test_webclient.py | 5 ++- 14 files changed, 75 insertions(+), 61 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 87304d76c..a806f55ce 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -5,8 +5,7 @@ from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ - ACCEPTED_ENCODINGS +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from tests import tests_datadir diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index c46b1bb87..919dbed23 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -77,12 +77,9 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, url2) self.assertEqual(req2.method, 'GET') - assert 'Content-Type' not in req2.headers, \ - "Content-Type header must not be present in redirected request" - assert 'Content-Length' not in req2.headers, \ - "Content-Length header must not be present in redirected request" - assert not req2.body, \ - "Redirected body must be empty, not '%s'" % req2.body + assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" + assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" + assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body # response without Location header but with status code is 3XX should be ignored del rsp.headers['Location'] @@ -244,12 +241,9 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, 'http://example.org/newpage') self.assertEqual(req2.method, 'GET') - assert 'Content-Type' not in req2.headers, \ - "Content-Type header must not be present in redirected request" - assert 'Content-Length' not in req2.headers, \ - "Content-Length header must not be present in redirected request" - assert not req2.body, \ - "Redirected body must be empty, not '%s'" % req2.body + assert 'Content-Type' not in req2.headers, "Content-Type header must not be present in redirected request" + assert 'Content-Length' not in req2.headers, "Content-Length header must not be present in redirected request" + assert not req2.body, "Redirected body must be empty, not '%s'" % req2.body def test_max_redirect_times(self): self.mw.max_redirect_times = 1 diff --git a/tests/test_selector.py b/tests/test_selector.py index bcf653444..00e663c11 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -88,8 +88,7 @@ class SelectorTestCase(unittest.TestCase): """Check that classes are using slots and are weak-referenceable""" x = Selector(text='') weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ + assert not hasattr(x, '__dict__'), "%s does not use __slots__" % x.__class__.__name__ def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 2da6aa4b5..6e56a28f5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -86,8 +86,7 @@ class BaseSettingsTest(unittest.TestCase): def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute('value', 10) - with mock.patch.object(attr, '__setattr__') as mock_setattr, \ - mock.patch.object(attr, 'set') as mock_set: + with mock.patch.object(attr, '__setattr__') as mock_setattr, mock.patch.object(attr, 'set') as mock_set: self.settings.attributes = {'TEST_OPTION': attr} diff --git a/tests/test_spider.py b/tests/test_spider.py index 805d70459..bd9238810 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -11,8 +11,14 @@ from scrapy import signals from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse from scrapy.spiders.init import InitSpider -from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \ - CSVFeedSpider, SitemapSpider +from scrapy.spiders import ( + CSVFeedSpider, + CrawlSpider, + Rule, + SitemapSpider, + Spider, + XMLFeedSpider, +) from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 067118cf0..5141f47af 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,16 +6,28 @@ from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.downloadermiddlewares.redirect import RedirectMiddleware -from scrapy.spidermiddlewares.referer import RefererMiddleware, \ - POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ - POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ - POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ - POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ - DefaultReferrerPolicy, \ - NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ - OriginWhenCrossOriginPolicy, OriginPolicy, \ - StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ - SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy +from scrapy.spidermiddlewares.referer import ( + DefaultReferrerPolicy, + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, + POLICY_NO_REFERRER, + POLICY_NO_REFERRER_WHEN_DOWNGRADE, + POLICY_ORIGIN, + POLICY_ORIGIN_WHEN_CROSS_ORIGIN, + POLICY_SAME_ORIGIN, + POLICY_SCRAPY_DEFAULT, + POLICY_STRICT_ORIGIN, + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + POLICY_UNSAFE_URL, + RefererMiddleware, + ReferrerPolicy, + SameOriginPolicy, + StrictOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, +) class TestRefererMiddleware(TestCase): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 299a51efe..6b05c8771 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -29,8 +29,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self._test_command(curl_command, expected_result) def test_get_basic_auth(self): - curl_command = 'curl "https://api.test.com/" -u ' \ - '"some_username:some_password"' + curl_command = 'curl "https://api.test.com/" -u "some_username:some_password"' expected_result = { "method": "GET", "url": "https://api.test.com/", @@ -212,8 +211,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): with warnings.catch_warnings(): # avoid warning when executing tests warnings.simplefilter('ignore') curl_command = 'curl --bar --baz http://www.example.com' - expected_result = \ - {"method": "GET", "url": "http://www.example.com"} + expected_result = {"method": "GET", "url": "http://www.example.com"} self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 2d4b88121..8c84331b9 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -2,8 +2,13 @@ from twisted.trial import unittest from twisted.internet import reactor, defer from twisted.python.failure import Failure -from scrapy.utils.defer import mustbe_deferred, process_chain, \ - process_chain_both, process_parallel, iter_errback +from scrapy.utils.defer import ( + iter_errback, + mustbe_deferred, + process_chain, + process_chain_both, + process_parallel, +) class MustbeDeferredTest(unittest.TestCase): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 8344c6701..3ebe3ac24 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -15,18 +15,20 @@ class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) def test_xmliter(self): - body = b"""<?xml version="1.0" encoding="UTF-8"?>\ + body = b""" + <?xml version="1.0" encoding="UTF-8"?> <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:noNamespaceSchemaLocation="someschmea.xsd">\ - <product id="001">\ - <type>Type 1</type>\ - <name>Name 1</name>\ - </product>\ - <product id="002">\ - <type>Type 2</type>\ - <name>Name 2</name>\ - </product>\ - </products>""" + xsi:noNamespaceSchemaLocation="someschmea.xsd"> + <product id="001"> + <type>Type 1</type> + <name>Name 1</name> + </product> + <product id="002"> + <type>Type 2</type> + <name>Name 2</name> + </product> + </products> + """ response = XmlResponse(url="http://example.com", body=body) attrs = [] @@ -115,7 +117,7 @@ class XmliterTestCase(unittest.TestCase): [[u'one'], [u'two']]) def test_xmliter_namespaces(self): - body = b"""\ + body = b""" <?xml version="1.0" encoding="UTF-8"?> <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0"> <channel> @@ -185,7 +187,7 @@ class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): - body = b"""\ + body = b""" <?xml version="1.0" encoding="UTF-8"?> <rss version="2.0" xmlns="http://base.google.com/ns/1.0"> <channel> @@ -214,7 +216,7 @@ class LxmlXmliterTestCase(XmliterTestCase): self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg']) def test_xmliter_namespaces_prefix(self): - body = b"""\ + body = b""" <?xml version="1.0" encoding="UTF-8"?> <root> <h:table xmlns:h="http://www.w3.org/TR/html4/"> diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 4cd4b7010..7e0049b1d 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,7 +1,11 @@ import unittest from scrapy.http import Request -from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ - request_authenticate, request_httprepr +from scrapy.utils.request import ( + _fingerprint_cache, + request_authenticate, + request_fingerprint, + request_httprepr, +) class UtilsRequestTest(unittest.TestCase): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 6ebf290c0..d6f4c0bb5 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -37,8 +37,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody) return True response = HtmlResponse(url, body=body) - assert open_in_browser(response, _openfunc=browser_open), \ - "Browser not called" + assert open_in_browser(response, _openfunc=browser_open), "Browser not called" resp = Response(url, body=body) self.assertRaises(TypeError, open_in_browser, resp, debug=True) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index bfbf9abb3..23eb261b7 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -156,8 +156,7 @@ Disallow: /forum/active/ def test_sitemap_blanklines(self): """Assert we can deal with starting blank lines before <xml> tag""" - s = Sitemap(b"""\ - + s = Sitemap(b""" <?xml version="1.0" encoding="UTF-8"?> <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 09a6d6c70..a194a0998 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -207,8 +207,7 @@ def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) assert url.startswith(args[1]), \ - 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % ( - args[0], url, args[1]) + 'Wrong scheme guessed: for `%s` got `%s`, expected `%s...`' % (args[0], url, args[1]) return do_expected diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 188e54602..c1c5945c2 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -356,9 +356,8 @@ class WebClientTestCase(unittest.TestCase): """ Test that non-standart body encoding matches Content-Encoding header """ body = b'\xd0\x81\xd1\x8e\xd0\xaf' - return getPage( - self.getURL('encoding'), body=body, response_transform=lambda r: r)\ - .addCallback(self._check_Encoding, body) + dfd = getPage(self.getURL('encoding'), body=body, response_transform=lambda r: r) + return dfd.addCallback(self._check_Encoding, body) def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b'Content-Encoding']) From cbe4dc57f3f65ecb851941dcfae0bc18c6c8582a Mon Sep 17 00:00:00 2001 From: Ajay Mittur <ajay.cs18@bmsce.ac.in> Date: Fri, 10 Jul 2020 18:22:43 +0530 Subject: [PATCH 089/125] Update pytest.ini --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 92c5bcb75..ca8191f42 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,4 +39,5 @@ flake8-ignore = scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 - tests/test_loader.py E741 \ No newline at end of file + tests/test_loader.py E741 + From a6a5fa91da8944943e2c9d8f34f09662be17b781 Mon Sep 17 00:00:00 2001 From: Artur Shellunts <shellunts.artur@gmail.com> Date: Fri, 10 Jul 2020 23:10:49 +0200 Subject: [PATCH 090/125] Remove deprecated class HtmlParserLinkExtractor Issue #4356 --- scrapy/linkextractors/htmlparser.py | 91 ----------------------------- 1 file changed, 91 deletions(-) delete mode 100644 scrapy/linkextractors/htmlparser.py diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py deleted file mode 100644 index 0425d4340..000000000 --- a/scrapy/linkextractors/htmlparser.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -HTMLParser-based link extractor -""" -import warnings -from html.parser import HTMLParser -from urllib.parse import urljoin - -from w3lib.url import safe_url_string -from w3lib.html import strip_html5_whitespace - -from scrapy.link import Link -from scrapy.utils.python import unique as unique_list -from scrapy.exceptions import ScrapyDeprecationWarning - - -class HtmlParserLinkExtractor(HTMLParser): - - def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True): - HTMLParser.__init__(self) - - warnings.warn( - "HtmlParserLinkExtractor is deprecated and will be removed in " - "future releases. Please use scrapy.linkextractors.LinkExtractor", - ScrapyDeprecationWarning, stacklevel=2, - ) - - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v - self.unique = unique - self.strip = strip - - def _extract_links(self, response_text, response_url, response_encoding): - self.reset() - self.feed(response_text) - self.close() - - links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links - - ret = [] - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url - for link in links: - if isinstance(link.url, str): - link.url = link.url.encode(response_encoding) - try: - link.url = urljoin(base_url, link.url) - except ValueError: - continue - link.url = safe_url_string(link.url, response_encoding) - link.text = link.text.decode(response_encoding) - ret.append(link) - - return ret - - def extract_links(self, response): - # wrapper needed to allow to work directly with text - return self._extract_links(response.body, response.url, response.encoding) - - def reset(self): - HTMLParser.reset(self) - - self.base_url = None - self.current_link = None - self.links = [] - - def handle_starttag(self, tag, attrs): - if tag == 'base': - self.base_url = dict(attrs).get('href') - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - if self.strip: - value = strip_html5_whitespace(value) - url = self.process_attr(value) - link = Link(url=url) - self.links.append(link) - self.current_link = link - - def handle_endtag(self, tag): - if self.scan_tag(tag): - self.current_link = None - - def handle_data(self, data): - if self.current_link: - self.current_link.text = self.current_link.text + data - - def matches(self, url): - """This extractor matches with any url, since - it doesn't contain any patterns""" - return True From 3f7e8635f479f4de2ab1c3d518010730a5f6f0a6 Mon Sep 17 00:00:00 2001 From: Aditya Kumar <k.aditya00@gmail.com> Date: Sat, 11 Jul 2020 12:18:24 +0530 Subject: [PATCH 091/125] Allow the parse command to write data to a file (#4377) --- docs/topics/commands.rst | 4 +++- scrapy/commands/__init__.py | 2 +- scrapy/commands/parse.py | 24 ++++++++---------------- tests/test_command_parse.py | 23 ++++++++++++++++++++++- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a0dcba90d..4fce51abc 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -468,7 +468,7 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response -* ``--meta`` or ``-m``: additional request meta that will be passed to the callback +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' * ``--cbkwargs``: additional keyword arguments that will be passed to the callback. @@ -491,6 +491,8 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +* ``--output`` or ``-o``: dump scraped items to a file + .. skip: start Usage example:: diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index ab850dcb3..57ce4e522 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -108,7 +108,7 @@ class ScrapyCommand: class BaseRunSpiderCommand(ScrapyCommand): """ - Common class used to share functionality between the crawl and runspider commands + Common class used to share functionality between the crawl, parse and runspider commands """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 8b7fa8b58..abc8ba9ff 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -4,18 +4,16 @@ import logging from itemadapter import is_item, ItemAdapter from w3lib.url import is_url -from scrapy.commands import ScrapyCommand +from scrapy.commands import BaseRunSpiderCommand from scrapy.http import Request from scrapy.utils import display -from scrapy.utils.conf import arglist_to_dict from scrapy.utils.spider import iterate_spider_output, spidercls_for_request from scrapy.exceptions import UsageError logger = logging.getLogger(__name__) -class Command(ScrapyCommand): - +class Command(BaseRunSpiderCommand): requires_project = True spider = None @@ -31,11 +29,9 @@ class Command(ScrapyCommand): return "Parse URL (using its spider) and print the results" def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + BaseRunSpiderCommand.add_options(self, parser) parser.add_option("--spider", dest="spider", default=None, help="use this spider without looking for one") - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") parser.add_option("--pipelines", action="store_true", help="process items through pipelines") parser.add_option("--nolinks", dest="nolinks", action="store_true", @@ -200,12 +196,15 @@ class Command(ScrapyCommand): self.add_items(depth, items) self.add_requests(depth, requests) + scraped_data = items if opts.output else [] if depth < opts.depth: for req in requests: req.meta['_depth'] = depth + 1 req.meta['_callback'] = req.callback req.callback = callback - return requests + scraped_data += requests + + return scraped_data # update request meta if any extra meta was passed through the --meta/-m opts. if opts.meta: @@ -221,18 +220,11 @@ class Command(ScrapyCommand): return request def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) + BaseRunSpiderCommand.process_options(self, args, opts) - self.process_spider_arguments(opts) self.process_request_meta(opts) self.process_request_cb_kwargs(opts) - def process_spider_arguments(self, opts): - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - def process_request_meta(self, opts): if opts.meta: try: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index a09dcf072..5754a5478 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,5 +1,5 @@ import os -from os.path import join, abspath +from os.path import join, abspath, isfile, exists from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest @@ -218,3 +218,24 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) + + @defer.inlineCallbacks + def test_output_flag(self): + """Checks if a file was created successfully having + correct format containing correct data in it. + """ + file_name = 'data.json' + file_path = join(self.proj_path, file_name) + yield self.execute([ + '--spider', self.spider_name, + '-c', 'parse', + '-o', file_name, + self.url('/html') + ]) + + self.assertTrue(exists(file_path)) + self.assertTrue(isfile(file_path)) + + content = '[\n{},\n{"foo": "bar"}\n]' + with open(file_path, 'r') as f: + self.assertEqual(f.read(), content) From d54c4496ee57785f3d6f882e2d128bb64b6b262c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 13 Jul 2020 14:36:33 +0200 Subject: [PATCH 092/125] Refactor guess_scheme --- scrapy/utils/url.py | 60 ++++++++++++++++++++++++++++++----------- tests/test_utils_url.py | 33 +++++++++++++++++++++-- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 60e2b50eb..1e431d3bf 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -83,26 +83,54 @@ def add_http_if_no_scheme(url): return url +def _is_posix_path(string): + return bool( + re.match( + r''' + ^ # start with... + ( + \. # ...a single dot, + ( + \. | [^/\.]+ # optionally followed by + )? # either a second dot or some characters + | + ~ # $HOME + )? # optional match of ".", ".." or ".blabla" + / # at least one "/" for a file path, + . # and something after the "/" + ''', + string, + flags=re.VERBOSE, + ) + ) + + +def _is_windows_path(string): + return bool( + re.match( + r''' + ^ + ( + [a-z]:\\ + | \\\\ + ) + ''', + string, + flags=re.IGNORECASE | re.VERBOSE, + ) + ) + + +def _is_path(string): + return _is_posix_path(string) or _is_windows_path(string) + + def guess_scheme(url): """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - # POSIX path - if re.match(r'''^ # start with... - ( - \. # ...a single dot, - ( - \. | [^/\.]+ # optionally followed by - )? # either a second dot or some characters - )? # optional match of ".", ".." or ".blabla" - / # at least one "/" for a file path, - . # and something after the "/" - ''', url, flags=re.VERBOSE): + if _is_path(url): return any_to_uri(url) - # Windows drive-letter path - elif re.match(r'''^[a-z]:\\''', url, flags=re.IGNORECASE): - return any_to_uri(url) - else: - return add_http_if_no_scheme(url) + return add_http_if_no_scheme(url) def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 09a6d6c70..6a5254d54 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,8 +1,14 @@ import unittest from scrapy.spiders import Spider -from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, strip_url) +from scrapy.utils.url import ( + add_http_if_no_scheme, + guess_scheme, + _is_path, + strip_url, + url_is_from_any_domain, + url_is_from_spider, +) __doctests__ = ['scrapy.utils.url'] @@ -434,5 +440,28 @@ class StripUrl(unittest.TestCase): self.assertEqual(strip_url(i, origin_only=True), o) +class IsPathTestCase(unittest.TestCase): + + def test_path(self): + for input_value, output_value in ( + # https://en.wikipedia.org/wiki/Path_(computing)#Representations_of_paths_by_operating_system_and_shell + # Unix-like OS, Microsoft Windows / cmd.exe + ("/home/user/docs/Letter.txt", True), + ("./inthisdir", True), + ("../../greatgrandparent", True), + ("~/.rcinfo", True), + (r"C:\user\docs\Letter.txt", True), + ("/user/docs/Letter.txt", True), + (r"C:\Letter.txt", True), + (r"\\Server01\user\docs\Letter.txt", True), + (r"\\?\UNC\Server01\user\docs\Letter.txt", True), + (r"\\?\C:\user\docs\Letter.txt", True), + (r"C:\user\docs\somefile.ext:alternate_stream_name", True), + + (r"https://example.com", False), + ): + self.assertEqual(_is_path(input_value), output_value, input_value) + + if __name__ == "__main__": unittest.main() From 53c323b19d81784e6c376ce8b9602de24d8e3037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 13 Jul 2020 15:29:30 +0200 Subject: [PATCH 093/125] =?UTF-8?q?=5Fis=5Fpath=20=E2=86=92=20=5Fis=5Ffile?= =?UTF-8?q?system=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/url.py | 4 ++-- tests/test_utils_url.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1e431d3bf..b23ddb459 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -121,14 +121,14 @@ def _is_windows_path(string): ) -def _is_path(string): +def _is_filesystem_path(string): return _is_posix_path(string) or _is_windows_path(string) def guess_scheme(url): """Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise.""" - if _is_path(url): + if _is_filesystem_path(url): return any_to_uri(url) return add_http_if_no_scheme(url) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 6a5254d54..3a143ba2f 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,7 +4,7 @@ from scrapy.spiders import Spider from scrapy.utils.url import ( add_http_if_no_scheme, guess_scheme, - _is_path, + _is_filesystem_path, strip_url, url_is_from_any_domain, url_is_from_spider, @@ -460,7 +460,7 @@ class IsPathTestCase(unittest.TestCase): (r"https://example.com", False), ): - self.assertEqual(_is_path(input_value), output_value, input_value) + self.assertEqual(_is_filesystem_path(input_value), output_value, input_value) if __name__ == "__main__": From ed5247ca4cbef98d1acf499c888ded74e444ef48 Mon Sep 17 00:00:00 2001 From: Artur Shellunts <shellunts.artur@gmail.com> Date: Tue, 14 Jul 2020 18:06:11 +0200 Subject: [PATCH 094/125] Remove htmlparser.py from tests/ignore.txt --- tests/ignores.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ignores.txt b/tests/ignores.txt index 45cf6fb92..f6e0d6fbe 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,6 +1,5 @@ scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py -scrapy/linkextractors/htmlparser.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py From 38496a00b7d2bcfa9d435551409e2c44007d168d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20C=C3=A9sar=20Batista?= <jcb.1611@gmail.com> Date: Wed, 15 Jul 2020 07:08:36 -0300 Subject: [PATCH 095/125] Use the itemlaoders library (#4516) --- docs/conf.py | 13 + docs/topics/loaders.rst | 418 +----------------- scrapy/loader/__init__.py | 269 +++--------- scrapy/loader/common.py | 19 +- scrapy/loader/processors.py | 99 +---- scrapy/utils/misc.py | 6 + setup.cfg | 3 + setup.py | 1 + tests/requirements-py3.txt | 1 - tests/test_loader.py | 531 +---------------------- tests/test_loader_deprecated.py | 720 ++++++++++++++++++++++++++++++++ 11 files changed, 854 insertions(+), 1226 deletions(-) create mode 100644 tests/test_loader_deprecated.py diff --git a/docs/conf.py b/docs/conf.py index 86734fae7..427c79481 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,6 +284,7 @@ intersphinx_mapping = { 'attrs': ('https://www.attrs.org/en/stable/', None), 'coverage': ('https://coverage.readthedocs.io/en/stable', None), 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), + 'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None), 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), @@ -305,3 +306,15 @@ hoverxref_role_types = { "ref": "tooltip", } hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] + + +def setup(app): + app.connect('autodoc-skip-member', maybe_skip_member) + + +def maybe_skip_member(app, what, name, obj, skip, options): + if not skip: + # autodocs was generating a text "alias of" for the following members + # https://github.com/sphinx-doc/sphinx/issues/4422 + return name in {'default_item_class', 'default_selector_class'} + return skip diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 9c82bb4d9..d0eeb4097 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism for extending and overriding different field parsing rules, either by spider, or by source format (HTML, XML, etc) without becoming a nightmare to maintain. +.. note:: Item Loaders are an extension of the itemloaders_ library that make it + easier to work with Scrapy by adding support for + :ref:`responses <topics-request-response>`. + Using Item Loaders to populate items ==================================== @@ -173,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. -Last, but not least, Scrapy comes with some :ref:`commonly used processors -<topics-loaders-available-processors>` built-in for convenience. +Last, but not least, itemloaders_ comes with some :ref:`commonly used +processors <itemloaders:built-in-processors>` built-in for convenience. Declaring Item Loaders @@ -182,8 +186,8 @@ Declaring Item Loaders Item Loaders are declared using a class definition syntax. Here is an example:: + from itemloaders.processors import TakeFirst, MapCompose, Join from scrapy.loader import ItemLoader - from scrapy.loader.processors import TakeFirst, MapCompose, Join class ProductLoader(ItemLoader): @@ -214,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>` metadata. Here is an example:: import scrapy - from scrapy.loader.processors import Join, MapCompose, TakeFirst + from itemloaders.processors import Join, MapCompose, TakeFirst from w3lib.html import remove_tags def filter_price(value): @@ -295,250 +299,9 @@ There are several ways to modify Item Loader context values: ItemLoader objects ================== -.. class:: ItemLoader([item, selector, response], **kwargs) - - Return a new Item Loader for populating the given :ref:`item object - <topics-items>`. If no item object is given, one is instantiated - automatically using the class in :attr:`default_item_class`. - - When instantiated with a ``selector`` or a ``response`` parameters - the :class:`ItemLoader` class provides convenient mechanisms for extracting - data from web pages using :ref:`selectors <topics-selectors>`. - - :param item: The item instance to populate using subsequent calls to - :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, - or :meth:`~ItemLoader.add_value`. - :type item: :ref:`item object <topics-items>` - - :param selector: The selector to extract data from, when using the - :meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath` - (resp. :meth:`replace_css`) method. - :type selector: :class:`~scrapy.selector.Selector` object - - :param response: The response used to construct the selector using the - :attr:`default_selector_class`, unless the selector argument is given, - in which case this argument is ignored. - :type response: :class:`~scrapy.http.Response` object - - The item, selector, response and the remaining keyword arguments are - assigned to the Loader context (accessible through the :attr:`context` attribute). - - :class:`ItemLoader` instances have the following methods: - - .. method:: get_value(value, *processors, **kwargs) - - Process the given ``value`` by the given ``processors`` and keyword - arguments. - - Available keyword arguments: - - :param re: a regular expression to use for extracting data from the - given value using :meth:`~scrapy.utils.misc.extract_regex` method, - applied before processors - :type re: str or compiled regex - - Examples: - - >>> from scrapy.loader.processors import TakeFirst - >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') - 'FOO` - - .. method:: add_value(field_name, value, *processors, **kwargs) - - Process and then add the given ``value`` for the given field. - - The value is first passed through :meth:`get_value` by giving the - ``processors`` and ``kwargs``, and then passed through the - :ref:`field input processor <topics-loaders-processors>` and its result - appended to the data collected for that field. If the field already - contains collected data, the new data is added. - - The given ``field_name`` can be ``None``, in which case values for - multiple fields may be added. And the processed value should be a dict - with field_name mapped to values. - - Examples:: - - loader.add_value('name', u'Color TV') - loader.add_value('colours', [u'white', u'blue']) - loader.add_value('length', u'100') - loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)') - loader.add_value(None, {'name': u'foo', 'sex': u'male'}) - - .. method:: replace_value(field_name, value, *processors, **kwargs) - - Similar to :meth:`add_value` but replaces the collected data with the - new value instead of adding it. - .. method:: get_xpath(xpath, *processors, **kwargs) - - Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`ItemLoader`. - - :param xpath: the XPath to extract data from - :type xpath: str - - :param re: a regular expression to use for extracting data from the - selected XPath region - :type re: str or compiled regex - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.get_xpath('//p[@class="product-name"]') - # HTML snippet: <p id="price">the price is $1200</p> - loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)') - - .. method:: add_xpath(field_name, xpath, *processors, **kwargs) - - Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`ItemLoader`. - - See :meth:`get_xpath` for ``kwargs``. - - :param xpath: the XPath to extract data from - :type xpath: str - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.add_xpath('name', '//p[@class="product-name"]') - # HTML snippet: <p id="price">the price is $1200</p> - loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - - .. method:: replace_xpath(field_name, xpath, *processors, **kwargs) - - Similar to :meth:`add_xpath` but replaces collected data instead of - adding it. - - .. method:: get_css(css, *processors, **kwargs) - - Similar to :meth:`ItemLoader.get_value` but receives a CSS selector - instead of a value, which is used to extract a list of unicode strings - from the selector associated with this :class:`ItemLoader`. - - :param css: the CSS selector to extract data from - :type css: str - - :param re: a regular expression to use for extracting data from the - selected CSS region - :type re: str or compiled regex - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.get_css('p.product-name') - # HTML snippet: <p id="price">the price is $1200</p> - loader.get_css('p#price', TakeFirst(), re='the price is (.*)') - - .. method:: add_css(field_name, css, *processors, **kwargs) - - Similar to :meth:`ItemLoader.add_value` but receives a CSS selector - instead of a value, which is used to extract a list of unicode strings - from the selector associated with this :class:`ItemLoader`. - - See :meth:`get_css` for ``kwargs``. - - :param css: the CSS selector to extract data from - :type css: str - - Examples:: - - # HTML snippet: <p class="product-name">Color TV</p> - loader.add_css('name', 'p.product-name') - # HTML snippet: <p id="price">the price is $1200</p> - loader.add_css('price', 'p#price', re='the price is (.*)') - - .. method:: replace_css(field_name, css, *processors, **kwargs) - - Similar to :meth:`add_css` but replaces collected data instead of - adding it. - - .. method:: load_item() - - Populate the item with the data collected so far, and return it. The - data collected is first passed through the :ref:`output processors - <topics-loaders-processors>` to get the final value to assign to each - item field. - - .. method:: nested_xpath(xpath) - - Create a nested loader with an xpath selector. - The supplied selector is applied relative to selector associated - with this :class:`ItemLoader`. The nested loader shares the :ref:`item - object <topics-items>` with the parent :class:`ItemLoader` so calls to - :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will - behave as expected. - - .. method:: nested_css(css) - - Create a nested loader with a css selector. - The supplied selector is applied relative to selector associated - with this :class:`ItemLoader`. The nested loader shares the :ref:`item - object <topics-items>` with the parent :class:`ItemLoader` so calls to - :meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will - behave as expected. - - .. method:: get_collected_values(field_name) - - Return the collected values for the given field. - - .. method:: get_output_value(field_name) - - Return the collected values parsed using the output processor, for the - given field. This method doesn't populate or modify the item at all. - - .. method:: get_input_processor(field_name) - - Return the input processor for the given field. - - .. method:: get_output_processor(field_name) - - Return the output processor for the given field. - - :class:`ItemLoader` instances have the following attributes: - - .. attribute:: item - - The :ref:`item object <topics-items>` being parsed by this Item Loader. - This is mostly used as a property so when attempting to override this - value, you may want to check out :attr:`default_item_class` first. - - .. attribute:: context - - The currently active :ref:`Context <topics-loaders-context>` of this - Item Loader. - - .. attribute:: default_item_class - - An :ref:`item object <topics-items>` class or factory, used to - instantiate items when not given in the ``__init__`` method. - - .. attribute:: default_input_processor - - The default input processor to use for those fields which don't specify - one. - - .. attribute:: default_output_processor - - The default output processor to use for those fields which don't specify - one. - - .. attribute:: default_selector_class - - The class used to construct the :attr:`selector` of this - :class:`ItemLoader`, if only a response is given in the ``__init__`` method. - If a selector is given in the ``__init__`` method this attribute is ignored. - This attribute is sometimes overridden in subclasses. - - .. attribute:: selector - - The :class:`~scrapy.selector.Selector` object to extract data from. - It's either the selector given in the ``__init__`` method or one created from - the response given in the ``__init__`` method using the - :attr:`default_selector_class`. This attribute is meant to be - read-only. +.. autoclass:: scrapy.loader.ItemLoader + :members: + :inherited-members: .. _topics-loaders-nested: @@ -609,7 +372,7 @@ those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default Product Item Loader (``ProductLoader``):: - from scrapy.loader.processors import MapCompose + from itemloaders.processors import MapCompose from myproject.ItemLoaders import ProductLoader def strip_dashes(x): @@ -622,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - from scrapy.loader.processors import MapCompose + from itemloaders.processors import MapCompose from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata @@ -642,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific organization of your Loaders collection - that's up to you and your project's needs. -.. _topics-loaders-available-processors: - -Available built-in processors -============================= - -.. module:: scrapy.loader.processors - :synopsis: A collection of processors to use with Item Loaders - -Even though you can use any callable function as input and output processors, -Scrapy provides some commonly used processors, which are described below. Some -of them, like the :class:`MapCompose` (which is typically used as input -processor) compose the output of several functions executed in order, to -produce the final parsed value. - -Here is a list of all built-in processors: - -.. class:: Identity - - The simplest processor, which doesn't do anything. It returns the original - values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it - accept Loader contexts. - - Example: - - >>> from scrapy.loader.processors import Identity - >>> proc = Identity() - >>> proc(['one', 'two', 'three']) - ['one', 'two', 'three'] - -.. class:: TakeFirst - - Returns the first non-null/non-empty value from the values received, - so it's typically used as an output processor to single-valued fields. - It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts. - - Example: - - >>> from scrapy.loader.processors import TakeFirst - >>> proc = TakeFirst() - >>> proc(['', 'one', 'two', 'three']) - 'one' - -.. class:: Join(separator=u' ') - - Returns the values joined with the separator given in the ``__init__`` method, which - defaults to ``u' '``. It doesn't accept Loader contexts. - - When using the default separator, this processor is equivalent to the - function: ``u' '.join`` - - Examples: - - >>> from scrapy.loader.processors import Join - >>> proc = Join() - >>> proc(['one', 'two', 'three']) - 'one two three' - >>> proc = Join('<br>') - >>> proc(['one', 'two', 'three']) - 'one<br>two<br>three' - -.. class:: Compose(*functions, **default_loader_context) - - A processor which is constructed from the composition of the given - functions. This means that each input value of this processor is passed to - the first function, and the result of that function is passed to the second - function, and so on, until the last function returns the output value of - this processor. - - By default, stop process on ``None`` value. This behaviour can be changed by - passing keyword argument ``stop_on_none=False``. - - Example: - - >>> from scrapy.loader.processors import Compose - >>> proc = Compose(lambda v: v[0], str.upper) - >>> proc(['hello', 'world']) - 'HELLO' - - Each function can optionally receive a ``loader_context`` parameter. For - those which do, this processor will pass the currently active :ref:`Loader - context <topics-loaders-context>` through that parameter. - - The keyword arguments passed in the ``__init__`` method are used as the default - Loader context values passed to each function call. However, the final - Loader context values passed to functions are overridden with the currently - active Loader context accessible through the :meth:`ItemLoader.context` - attribute. - -.. class:: MapCompose(*functions, **default_loader_context) - - A processor which is constructed from the composition of the given - functions, similar to the :class:`Compose` processor. The difference with - this processor is the way internal results are passed among functions, - which is as follows: - - The input value of this processor is *iterated* and the first function is - applied to each element. The results of these function calls (one for each element) - are concatenated to construct a new iterable, which is then used to apply the - second function, and so on, until the last function is applied to each - value of the list of values collected so far. The output values of the last - function are concatenated together to produce the output of this processor. - - Each particular function can return a value or a list of values, which is - flattened with the list of values returned by the same function applied to - the other input values. The functions can also return ``None`` in which - case the output of that function is ignored for further processing over the - chain. - - This processor provides a convenient way to compose functions that only - work with single values (instead of iterables). For this reason the - :class:`MapCompose` processor is typically used as input processor, since - data is often extracted using the - :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors - <topics-selectors>`, which returns a list of unicode strings. - - The example below should clarify how it works: - - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.loader.processors import MapCompose - >>> proc = MapCompose(filter_world, str.upper) - >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] - - As with the Compose processor, functions can receive Loader contexts, and - ``__init__`` method keyword arguments are used as default context values. See - :class:`Compose` processor for more info. - -.. class:: SelectJmes(json_path) - - Queries the value using the json path provided to the ``__init__`` method and returns the output. - Requires jmespath (https://github.com/jmespath/jmespath.py) to run. - This processor takes only one input at a time. - - Example: - - >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose - >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries - >>> proc({'foo': 'bar'}) - 'bar' - >>> proc({'foo': {'bar': 'baz'}}) - {'bar': 'baz'} - - Working with Json: - - >>> import json - >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo")) - >>> proc_single_json_str('{"foo": "bar"}') - 'bar' - >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo'))) - >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]') - ['bar'] +.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/ +.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 18f57945f..014951a8e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -3,217 +3,86 @@ Item Loader See documentation in docs/topics/loaders.rst """ -from collections import defaultdict -from contextlib import suppress - -from itemadapter import ItemAdapter +import itemloaders from scrapy.item import Item -from scrapy.loader.common import wrap_loader_context -from scrapy.loader.processors import Identity from scrapy.selector import Selector -from scrapy.utils.misc import arg_to_iter, extract_regex -from scrapy.utils.python import flatten -def unbound_method(method): +class ItemLoader(itemloaders.ItemLoader): """ - Allow to use single-argument functions as input or output processors - (no need to define an unused first 'self' argument) + A user-friendly abstraction to populate an :ref:`item <topics-items>` with data + by applying :ref:`field processors <topics-loaders-processors>` to scraped data. + When instantiated with a ``selector`` or a ``response`` it supports + data extraction from web pages using :ref:`selectors <topics-selectors>`. + + :param item: The item instance to populate using subsequent calls to + :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`, + or :meth:`~ItemLoader.add_value`. + :type item: scrapy.item.Item + + :param selector: The selector to extract data from, when using the + :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or + :meth:`replace_css` method. + :type selector: :class:`~scrapy.selector.Selector` object + + :param response: The response used to construct the selector using the + :attr:`default_selector_class`, unless the selector argument is given, + in which case this argument is ignored. + :type response: :class:`~scrapy.http.Response` object + + If no item is given, one is instantiated automatically using the class in + :attr:`default_item_class`. + + The item, selector, response and remaining keyword arguments are + assigned to the Loader context (accessible through the :attr:`context` attribute). + + .. attribute:: item + + The item object being parsed by this Item Loader. + This is mostly used as a property so, when attempting to override this + value, you may want to check out :attr:`default_item_class` first. + + .. attribute:: context + + The currently active :ref:`Context <loaders-context>` of this Item Loader. + + .. attribute:: default_item_class + + An :ref:`item <topics-items>` class (or factory), used to instantiate + items when not given in the ``__init__`` method. + + .. attribute:: default_input_processor + + The default input processor to use for those fields which don't specify + one. + + .. attribute:: default_output_processor + + The default output processor to use for those fields which don't specify + one. + + .. attribute:: default_selector_class + + The class used to construct the :attr:`selector` of this + :class:`ItemLoader`, if only a response is given in the ``__init__`` method. + If a selector is given in the ``__init__`` method this attribute is ignored. + This attribute is sometimes overridden in subclasses. + + .. attribute:: selector + + The :class:`~scrapy.selector.Selector` object to extract data from. + It's either the selector given in the ``__init__`` method or one created from + the response given in the ``__init__`` method using the + :attr:`default_selector_class`. This attribute is meant to be + read-only. """ - with suppress(AttributeError): - if '.' not in method.__qualname__: - return method.__func__ - return method - - -class ItemLoader: default_item_class = Item - default_input_processor = Identity() - default_output_processor = Identity() default_selector_class = Selector def __init__(self, item=None, selector=None, response=None, parent=None, **context): if selector is None and response is not None: selector = self.default_selector_class(response) - self.selector = selector - context.update(selector=selector, response=response) - if item is None: - item = self.default_item_class() - self.context = context - self.parent = parent - self._local_item = context['item'] = item - self._local_values = defaultdict(list) - # values from initial item - for field_name, value in ItemAdapter(item).items(): - self._values[field_name] += arg_to_iter(value) - - @property - def _values(self): - if self.parent is not None: - return self.parent._values - else: - return self._local_values - - @property - def item(self): - if self.parent is not None: - return self.parent.item - else: - return self._local_item - - def nested_xpath(self, xpath, **context): - selector = self.selector.xpath(xpath) - context.update(selector=selector) - subloader = self.__class__( - item=self.item, parent=self, **context - ) - return subloader - - def nested_css(self, css, **context): - selector = self.selector.css(css) - context.update(selector=selector) - subloader = self.__class__( - item=self.item, parent=self, **context - ) - return subloader - - def add_value(self, field_name, value, *processors, **kw): - value = self.get_value(value, *processors, **kw) - if value is None: - return - if not field_name: - for k, v in value.items(): - self._add_value(k, v) - else: - self._add_value(field_name, value) - - def replace_value(self, field_name, value, *processors, **kw): - value = self.get_value(value, *processors, **kw) - if value is None: - return - if not field_name: - for k, v in value.items(): - self._replace_value(k, v) - else: - self._replace_value(field_name, value) - - def _add_value(self, field_name, value): - value = arg_to_iter(value) - processed_value = self._process_input_value(field_name, value) - if processed_value: - self._values[field_name] += arg_to_iter(processed_value) - - def _replace_value(self, field_name, value): - self._values.pop(field_name, None) - self._add_value(field_name, value) - - def get_value(self, value, *processors, **kw): - regex = kw.get('re', None) - if regex: - value = arg_to_iter(value) - value = flatten(extract_regex(regex, x) for x in value) - - for proc in processors: - if value is None: - break - _proc = proc - proc = wrap_loader_context(proc, self.context) - try: - value = proc(value) - except Exception as e: - raise ValueError("Error with processor %s value=%r error='%s: %s'" % - (_proc.__class__.__name__, value, - type(e).__name__, str(e))) - return value - - def load_item(self): - adapter = ItemAdapter(self.item) - for field_name in tuple(self._values): - value = self.get_output_value(field_name) - if value is not None: - adapter[field_name] = value - return adapter.item - - def get_output_value(self, field_name): - proc = self.get_output_processor(field_name) - proc = wrap_loader_context(proc, self.context) - try: - return proc(self._values[field_name]) - except Exception as e: - raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % - (field_name, self._values[field_name], type(e).__name__, str(e))) - - def get_collected_values(self, field_name): - return self._values[field_name] - - def get_input_processor(self, field_name): - proc = getattr(self, '%s_in' % field_name, None) - if not proc: - proc = self._get_item_field_attr(field_name, 'input_processor', - self.default_input_processor) - return unbound_method(proc) - - def get_output_processor(self, field_name): - proc = getattr(self, '%s_out' % field_name, None) - if not proc: - proc = self._get_item_field_attr(field_name, 'output_processor', - self.default_output_processor) - return unbound_method(proc) - - def _process_input_value(self, field_name, value): - proc = self.get_input_processor(field_name) - _proc = proc - proc = wrap_loader_context(proc, self.context) - try: - return proc(value) - except Exception as e: - raise ValueError( - "Error with input processor %s: field=%r value=%r " - "error='%s: %s'" % (_proc.__class__.__name__, field_name, - value, type(e).__name__, str(e))) - - def _get_item_field_attr(self, field_name, key, default=None): - field_meta = ItemAdapter(self.item).get_field_meta(field_name) - return field_meta.get(key, default) - - def _check_selector_method(self): - if self.selector is None: - raise RuntimeError("To use XPath or CSS selectors, " - "%s must be instantiated with a selector " - "or a response" % self.__class__.__name__) - - def add_xpath(self, field_name, xpath, *processors, **kw): - values = self._get_xpathvalues(xpath, **kw) - self.add_value(field_name, values, *processors, **kw) - - def replace_xpath(self, field_name, xpath, *processors, **kw): - values = self._get_xpathvalues(xpath, **kw) - self.replace_value(field_name, values, *processors, **kw) - - def get_xpath(self, xpath, *processors, **kw): - values = self._get_xpathvalues(xpath, **kw) - return self.get_value(values, *processors, **kw) - - def _get_xpathvalues(self, xpaths, **kw): - self._check_selector_method() - xpaths = arg_to_iter(xpaths) - return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths) - - def add_css(self, field_name, css, *processors, **kw): - values = self._get_cssvalues(css, **kw) - self.add_value(field_name, values, *processors, **kw) - - def replace_css(self, field_name, css, *processors, **kw): - values = self._get_cssvalues(css, **kw) - self.replace_value(field_name, values, *processors, **kw) - - def get_css(self, css, *processors, **kw): - values = self._get_cssvalues(css, **kw) - return self.get_value(values, *processors, **kw) - - def _get_cssvalues(self, csss, **kw): - self._check_selector_method() - csss = arg_to_iter(csss) - return flatten(self.selector.css(css).getall() for css in csss) + context.update(response=response) + super().__init__(item=item, selector=selector, parent=parent, **context) diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py index 42f8de636..3b8a6ee94 100644 --- a/scrapy/loader/common.py +++ b/scrapy/loader/common.py @@ -1,14 +1,21 @@ """Common functions used in Item Loaders code""" -from functools import partial -from scrapy.utils.python import get_func_args +import warnings + +from itemloaders import common + +from scrapy.utils.deprecate import ScrapyDeprecationWarning def wrap_loader_context(function, context): """Wrap functions that receive loader_context to contain the context "pre-loaded" and expose a interface that receives only one argument """ - if 'loader_context' in get_func_args(function): - return partial(function, loader_context=context) - else: - return function + warnings.warn( + "scrapy.loader.common.wrap_loader_context has moved to a new library." + "Please update your reference to itemloaders.common.wrap_loader_context", + ScrapyDeprecationWarning, + stacklevel=2 + ) + + return common.wrap_loader_context(function, context) diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index a7be65609..51fbd19eb 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ -from collections import ChainMap +from itemloaders import processors -from scrapy.utils.misc import arg_to_iter -from scrapy.loader.common import wrap_loader_context +from scrapy.utils.deprecate import create_deprecated_class -class MapCompose: +MapCompose = create_deprecated_class('MapCompose', processors.MapCompose) - def __init__(self, *functions, **default_loader_context): - self.functions = functions - self.default_loader_context = default_loader_context +Compose = create_deprecated_class('Compose', processors.Compose) - def __call__(self, value, loader_context=None): - values = arg_to_iter(value) - if loader_context: - context = ChainMap(loader_context, self.default_loader_context) - else: - context = self.default_loader_context - wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] - for func in wrapped_funcs: - next_values = [] - for v in values: - try: - next_values += arg_to_iter(func(v)) - except Exception as e: - raise ValueError("Error in MapCompose with " - "%s value=%r error='%s: %s'" % - (str(func), value, type(e).__name__, - str(e))) - values = next_values - return values +TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst) +Identity = create_deprecated_class('Identity', processors.Identity) -class Compose: +SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes) - def __init__(self, *functions, **default_loader_context): - self.functions = functions - self.stop_on_none = default_loader_context.get('stop_on_none', True) - self.default_loader_context = default_loader_context - - def __call__(self, value, loader_context=None): - if loader_context: - context = ChainMap(loader_context, self.default_loader_context) - else: - context = self.default_loader_context - wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] - for func in wrapped_funcs: - if value is None and self.stop_on_none: - break - try: - value = func(value) - except Exception as e: - raise ValueError("Error in Compose with " - "%s value=%r error='%s: %s'" % - (str(func), value, type(e).__name__, str(e))) - return value - - -class TakeFirst: - - def __call__(self, values): - for value in values: - if value is not None and value != '': - return value - - -class Identity: - - def __call__(self, values): - return values - - -class SelectJmes: - """ - Query the input string for the jmespath (given at instantiation), - and return the answer - Requires : jmespath(https://github.com/jmespath/jmespath) - Note: SelectJmes accepts only one input element at a time. - """ - def __init__(self, json_path): - self.json_path = json_path - import jmespath - self.compiled_path = jmespath.compile(self.json_path) - - def __call__(self, value): - """Query value for the jmespath query and return answer - :param value: a data structure (dict, list) to extract from - :return: Element extracted according to jmespath query - """ - return self.compiled_path.search(value) - - -class Join: - - def __init__(self, separator=u' '): - self.separator = separator - - def __call__(self, values): - return self.separator.join(values) +Join = create_deprecated_class('Join', processors.Join) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8e5fde246..d6966be8e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -15,6 +15,7 @@ from w3lib.html import replace_entities from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode from scrapy.item import _BaseItem +from scrapy.utils.deprecate import ScrapyDeprecationWarning _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes @@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'): * if the regex contains multiple numbered groups, all those will be returned (flattened) * if the regex doesn't contain any group the entire regex matching is returned """ + warnings.warn( + "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.", + ScrapyDeprecationWarning, + stacklevel=2 + ) if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) diff --git a/setup.cfg b/setup.cfg index a9138c1c0..46a3d13fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -118,6 +118,9 @@ ignore_errors = True [mypy-tests.test_loader] ignore_errors = True +[mypy-tests.test_loader_deprecated] +ignore_errors = True + [mypy-tests.test_pipeline_crawl] ignore_errors = True diff --git a/setup.py b/setup.py index 5a99fd1bf..f8d9b491b 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ setup( 'Twisted>=17.9.0', 'cryptography>=2.0', 'cssselect>=0.9.1', + 'itemloaders>=1.0.1', 'lxml>=3.5.0', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index dacb86e56..0551b1e95 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,6 @@ # Tests requirements attrs dataclasses; python_version == '3.6' -jmespath mitmproxy; python_version >= '3.6' mitmproxy<4.0.0; python_version < '3.6' # https://github.com/pytest-dev/pytest-twisted/issues/93 diff --git a/tests/test_loader.py b/tests/test_loader.py index 8a9c6fca9..581183625 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,14 +1,12 @@ -from functools import partial import unittest import attr from itemadapter import ItemAdapter +from itemloaders.processors import Compose, Identity, MapCompose, TakeFirst from scrapy.http import HtmlResponse from scrapy.item import Item, Field from scrapy.loader import ItemLoader -from scrapy.loader.processors import (Compose, Identity, Join, - MapCompose, SelectJmes, TakeFirst) from scrapy.selector import Selector @@ -69,6 +67,10 @@ def processor_with_args(value, other=None, loader_context=None): class BasicItemLoaderTest(unittest.TestCase): + def test_add_value_on_unknown_field(self): + il = TestItemLoader() + self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) + def test_load_item_using_default_loader(self): i = TestItem() i['summary'] = u'lala' @@ -85,391 +87,6 @@ class BasicItemLoaderTest(unittest.TestCase): item = il.load_item() self.assertEqual(item['name'], [u'Marta']) - def test_load_item_ignore_none_field_values(self): - def validate_sku(value): - # Let's assume a SKU is only digits. - if value.isdigit(): - return value - - class MyLoader(ItemLoader): - name_out = Compose(lambda vs: vs[0]) # take first which allows empty values - price_out = Compose(TakeFirst(), float) - sku_out = Compose(TakeFirst(), validate_sku) - - valid_fragment = u'SKU: 1234' - invalid_fragment = u'SKU: not available' - sku_re = 'SKU: (.+)' - - il = MyLoader(item={}) - # Should not return "sku: None". - il.add_value('sku', [invalid_fragment], re=sku_re) - # Should not ignore empty values. - il.add_value('name', u'') - il.add_value('price', [u'0']) - self.assertEqual(il.load_item(), { - 'name': u'', - 'price': 0.0, - }) - - il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], u'1234') - - def test_self_referencing_loader(self): - class MyLoader(ItemLoader): - url_out = TakeFirst() - - def img_url_out(self, values): - return (self.get_output_value('url') or '') + values[0] - - il = MyLoader(item={}) - il.add_value('url', 'http://example.com/') - il.add_value('img_url', '1234.png') - self.assertEqual(il.load_item(), { - 'url': 'http://example.com/', - 'img_url': 'http://example.com/1234.png', - }) - - il = MyLoader(item={}) - il.add_value('img_url', '1234.png') - self.assertEqual(il.load_item(), { - 'img_url': '1234.png', - }) - - def test_add_value(self): - il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.add_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) - - # test add object value - il.add_value('summary', {'key': 1}) - self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) - - il.add_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) - - def test_add_zero(self): - il = NameItemLoader() - il.add_value('name', 0) - self.assertEqual(il.get_collected_values('name'), [0]) - - def test_replace_value(self): - il = TestItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.replace_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Pepe']) - - il.replace_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Jim']) - - def test_get_value(self): - il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) - self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) - self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) - - il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') - self.assertEqual([u'foo'], il.get_collected_values('name')) - il.replace_value('name', u'name:bar', re=u'name:(.*)$') - self.assertEqual([u'bar'], il.get_collected_values('name')) - - def test_iter_on_input_processor_input(self): - class NameFirstItemLoader(NameItemLoader): - name_in = TakeFirst() - - il = NameFirstItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) - il = NameFirstItemLoader() - il.add_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) - - il = NameFirstItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) - il = NameFirstItemLoader() - il.replace_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) - - il = NameFirstItemLoader() - il.add_value('name', u'marta') - il.add_value('name', [u'jose', u'pedro']) - self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) - - def test_map_compose_filter(self): - def filter_world(x): - return None if x == 'world' else x - - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) - - def test_map_compose_filter_multil(self): - class TestItemLoader(NameItemLoader): - name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Mart']) - item = il.load_item() - self.assertEqual(item['name'], [u'Mart']) - - def test_default_input_processor(self): - il = DefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) - - def test_inherited_default_input_processor(self): - class InheritDefaultedItemLoader(DefaultedItemLoader): - pass - - il = InheritDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) - - def test_input_processor_inheritance(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(lambda v: v.lower()) - - il = ChildItemLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) - - class ChildChildItemLoader(ChildItemLoader): - url_in = MapCompose(lambda v: v.upper()) - summary_in = MapCompose(lambda v: v) - - il = ChildChildItemLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) - - def test_empty_map_compose(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose() - - il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) - - def test_identity_input_processor(self): - class IdentityDefaultedItemLoader(DefaultedItemLoader): - name_in = Identity() - - il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) - - def test_extend_custom_input_processors(self): - class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, str.swapcase) - - il = ChildItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mARTA']) - - def test_extend_default_input_processors(self): - class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) - - il = ChildDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'MART']) - - def test_output_processor_using_function(self): - il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - class TakeFirstItemLoader(TestItemLoader): - name_out = u" ".join - - il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') - - def test_output_processor_error(self): - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = MapCompose(float) - - il = TestItemLoader() - il.add_value('name', [u'$10']) - try: - float(u'$10') - except Exception as e: - expected_exc_str = str(e) - - exc = None - try: - il.load_item() - except Exception as e: - exc = e - assert isinstance(exc, ValueError) - s = str(exc) - assert 'name' in s, s - assert '$10' in s, s - assert 'ValueError' in s, s - assert expected_exc_str in s, s - - def test_output_processor_using_classes(self): - il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - class TakeFirstItemLoader(TestItemLoader): - name_out = Join() - - il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') - - class TakeFirstItemLoader(TestItemLoader): - name_out = Join("<br>") - - il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta') - - def test_default_output_processor(self): - il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - class LalaItemLoader(TestItemLoader): - default_output_processor = Identity() - - il = LalaItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) - - def test_loader_context_on_declaration(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key=u'val') - - il = ChildItemLoader() - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['val']) - - def test_loader_context_on_instantiation(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader(key=u'val') - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['val']) - - def test_loader_context_on_assign(self): - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args) - - il = ChildItemLoader() - il.context['key'] = u'val' - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['val']) - - def test_item_passed_to_input_processor_functions(self): - def processor(value, loader_context): - return loader_context['item']['name'] - - class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor) - - it = TestItem(name='marta') - il = ChildItemLoader(item=it) - il.add_value('url', u'text') - self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', u'text2') - self.assertEqual(il.get_output_value('url'), ['marta']) - - def test_add_value_on_unknown_field(self): - il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) - - def test_compose_processor(self): - class TestItemLoader(NameItemLoader): - name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) - - il = TestItemLoader() - il.add_value('name', [u'marta', u'other']) - self.assertEqual(il.get_output_value('name'), u'Mart') - item = il.load_item() - self.assertEqual(item['name'], u'Mart') - - def test_partial_processor(self): - def join(values, sep=None, loader_context=None, ignored=None): - if sep is not None: - return sep.join(values) - elif loader_context and 'sep' in loader_context: - return loader_context['sep'].join(values) - else: - return ''.join(values) - - class TestItemLoader(NameItemLoader): - name_out = Compose(partial(join, sep='+')) - url_out = Compose(partial(join, loader_context={'sep': '.'})) - summary_out = Compose(partial(join, ignored='foo')) - - il = TestItemLoader() - il.add_value('name', [u'rabbit', u'hole']) - il.add_value('url', [u'rabbit', u'hole']) - il.add_value('summary', [u'rabbit', u'hole']) - item = il.load_item() - self.assertEqual(item['name'], u'rabbit+hole') - self.assertEqual(item['url'], u'rabbit.hole') - self.assertEqual(item['summary'], u'rabbithole') - - def test_error_input_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_in = MapCompose(float) - - il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other']) - - def test_error_output_processor(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - name_out = Compose(Join(), float) - - il = TestItemLoader() - il.add_value('name', u'marta') - with self.assertRaises(ValueError): - il.load_item() - - def test_error_processor_as_argument(self): - class TestItem(Item): - name = Field() - - class TestItemLoader(ItemLoader): - default_item_class = TestItem - - il = TestItemLoader() - self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other'], Compose(float)) - class InitializationTestMixin: @@ -587,41 +204,6 @@ class BaseNoInputReprocessingLoader(ItemLoader): title_out = TakeFirst() -class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): - default_item_class = dict - - -class NoInputReprocessingFromDictTest(unittest.TestCase): - """ - Loaders initialized from loaded items must not reprocess fields (dict instances) - """ - def test_avoid_reprocessing_with_initial_values_single(self): - il = NoInputReprocessingDictLoader(item=dict(title='foo')) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='foo')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) - - def test_avoid_reprocessing_with_initial_values_list(self): - il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar'])) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='foo')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) - - def test_avoid_reprocessing_without_initial_values_single(self): - il = NoInputReprocessingDictLoader() - il.add_value('title', 'foo') - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='FOO')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) - - def test_avoid_reprocessing_without_initial_values_list(self): - il = NoInputReprocessingDictLoader() - il.add_value('title', ['foo', 'bar']) - il_loaded = il.load_item() - self.assertEqual(il_loaded, dict(title='FOO')) - self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) - - class NoInputReprocessingItem(Item): title = Field() @@ -661,25 +243,6 @@ class NoInputReprocessingFromItemTest(unittest.TestCase): self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'}) -class TestOutputProcessorDict(unittest.TestCase): - def test_output_processor(self): - - class TempDict(dict): - def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) - self.setdefault('temp', 0.3) - - class TempLoader(ItemLoader): - default_item_class = TempDict - default_input_processor = Identity() - default_output_processor = Compose(TakeFirst()) - - loader = TempLoader() - item = loader.load_item() - self.assertIsInstance(item, TempDict) - self.assertEqual(dict(item), {'temp': 0.3}) - - class TestOutputProcessorItem(unittest.TestCase): def test_output_processor(self): @@ -701,49 +264,6 @@ class TestOutputProcessorItem(unittest.TestCase): self.assertEqual(dict(item), {'temp': 0.3}) -class ProcessorsTest(unittest.TestCase): - - def test_take_first(self): - proc = TakeFirst() - self.assertEqual(proc([None, '', 'hello', 'world']), 'hello') - self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0) - - def test_identity(self): - proc = Identity() - self.assertEqual(proc([None, '', 'hello', 'world']), - [None, '', 'hello', 'world']) - - def test_join(self): - proc = Join() - self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), u' hello world') - self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertIsInstance(proc(['hello', 'world']), str) - - def test_compose(self): - proc = Compose(lambda v: v[0], str.upper) - self.assertEqual(proc(['hello', 'world']), 'HELLO') - proc = Compose(str.upper) - self.assertEqual(proc(None), None) - proc = Compose(str.upper, stop_on_none=False) - self.assertRaises(ValueError, proc, None) - proc = Compose(str.upper, lambda x: x + 1) - self.assertRaises(ValueError, proc, 'hello') - - def test_mapcompose(self): - def filter_world(x): - return None if x == 'world' else x - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), - [u'HELLO', u'THIS', u'IS', u'SCRAPY']) - proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, str.upper) - self.assertRaises(ValueError, proc, [1]) - proc = MapCompose(filter_world, lambda x: x + 1) - self.assertRaises(ValueError, proc, 'hello') - - class SelectortemLoaderTest(unittest.TestCase): response = HtmlResponse(url="", encoding='utf-8', body=b""" <html> @@ -921,6 +441,7 @@ class SubselectorLoaderTest(unittest.TestCase): def test_nested_xpath(self): l = NestedItemLoader(response=self.response) + nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') @@ -998,31 +519,6 @@ class SubselectorLoaderTest(unittest.TestCase): self.assertEqual(item['image'], [u'/images/logo.png']) -class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } - - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) - - # Functions as processors def function_processor_strip(iterable): @@ -1044,12 +540,6 @@ class FunctionProcessorItemLoader(ItemLoader): default_item_class = FunctionProcessorItem -class FunctionProcessorDictLoader(ItemLoader): - default_item_class = dict - foo_in = function_processor_strip - foo_out = function_processor_upper - - class FunctionProcessorTestCase(unittest.TestCase): def test_processor_defined_in_item(self): @@ -1061,15 +551,6 @@ class FunctionProcessorTestCase(unittest.TestCase): {'foo': ['BAR', 'ASDF', 'QWERTY']} ) - def test_processor_defined_in_item_loader(self): - lo = FunctionProcessorDictLoader() - lo.add_value('foo', ' bar ') - lo.add_value('foo', [' asdf ', ' qwerty ']) - self.assertEqual( - dict(lo.load_item()), - {'foo': ['BAR', 'ASDF', 'QWERTY']} - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py new file mode 100644 index 000000000..d0a59e8cd --- /dev/null +++ b/tests/test_loader_deprecated.py @@ -0,0 +1,720 @@ +""" +These tests are kept as references from the ones that were ported to a itemloaders library. +Once we remove the references from scrapy, we can remove these tests. +""" + +import unittest +import warnings +from functools import partial + +from itemloaders.processors import (Compose, Identity, Join, + MapCompose, SelectJmes, TakeFirst) + +from scrapy.item import Item, Field +from scrapy.loader import ItemLoader +from scrapy.loader.common import wrap_loader_context +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.misc import extract_regex + + +# test items +class NameItem(Item): + name = Field() + + +class TestItem(NameItem): + url = Field() + summary = Field() + + +# test item loaders +class NameItemLoader(ItemLoader): + default_item_class = TestItem + + +class TestItemLoader(NameItemLoader): + name_in = MapCompose(lambda v: v.title()) + + +class DefaultedItemLoader(NameItemLoader): + default_input_processor = MapCompose(lambda v: v[:-1]) + + +# test processors +def processor_with_args(value, other=None, loader_context=None): + if 'key' in loader_context: + return loader_context['key'] + return value + + +class BasicItemLoaderTest(unittest.TestCase): + + def test_load_item_using_default_loader(self): + i = TestItem() + i['summary'] = u'lala' + il = ItemLoader(item=i) + il.add_value('name', u'marta') + item = il.load_item() + assert item is i + self.assertEqual(item['summary'], [u'lala']) + self.assertEqual(item['name'], [u'marta']) + + def test_load_item_using_custom_loader(self): + il = TestItemLoader() + il.add_value('name', u'marta') + item = il.load_item() + self.assertEqual(item['name'], [u'Marta']) + + def test_load_item_ignore_none_field_values(self): + def validate_sku(value): + # Let's assume a SKU is only digits. + if value.isdigit(): + return value + + class MyLoader(ItemLoader): + name_out = Compose(lambda vs: vs[0]) # take first which allows empty values + price_out = Compose(TakeFirst(), float) + sku_out = Compose(TakeFirst(), validate_sku) + + valid_fragment = u'SKU: 1234' + invalid_fragment = u'SKU: not available' + sku_re = 'SKU: (.+)' + + il = MyLoader(item={}) + # Should not return "sku: None". + il.add_value('sku', [invalid_fragment], re=sku_re) + # Should not ignore empty values. + il.add_value('name', u'') + il.add_value('price', [u'0']) + self.assertEqual(il.load_item(), { + 'name': u'', + 'price': 0.0, + }) + + il.replace_value('sku', [valid_fragment], re=sku_re) + self.assertEqual(il.load_item()['sku'], u'1234') + + def test_self_referencing_loader(self): + class MyLoader(ItemLoader): + url_out = TakeFirst() + + def img_url_out(self, values): + return (self.get_output_value('url') or '') + values[0] + + il = MyLoader(item={}) + il.add_value('url', 'http://example.com/') + il.add_value('img_url', '1234.png') + self.assertEqual(il.load_item(), { + 'url': 'http://example.com/', + 'img_url': 'http://example.com/1234.png', + }) + + il = MyLoader(item={}) + il.add_value('img_url', '1234.png') + self.assertEqual(il.load_item(), { + 'img_url': '1234.png', + }) + + def test_add_value(self): + il = TestItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'Marta']) + self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('name', u'pepe') + self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) + self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) + + # test add object value + il.add_value('summary', {'key': 1}) + self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) + + il.add_value(None, u'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) + + def test_add_zero(self): + il = NameItemLoader() + il.add_value('name', 0) + self.assertEqual(il.get_collected_values('name'), [0]) + + def test_replace_value(self): + il = TestItemLoader() + il.replace_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'Marta']) + self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.replace_value('name', u'pepe') + self.assertEqual(il.get_collected_values('name'), [u'Pepe']) + self.assertEqual(il.get_output_value('name'), [u'Pepe']) + + il.replace_value(None, u'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), [u'Jim']) + + def test_get_value(self): + il = NameItemLoader() + self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) + self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) + self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) + + il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') + self.assertEqual([u'foo'], il.get_collected_values('name')) + il.replace_value('name', u'name:bar', re=u'name:(.*)$') + self.assertEqual([u'bar'], il.get_collected_values('name')) + + def test_iter_on_input_processor_input(self): + class NameFirstItemLoader(NameItemLoader): + name_in = TakeFirst() + + il = NameFirstItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'marta']) + il = NameFirstItemLoader() + il.add_value('name', [u'marta', u'jose']) + self.assertEqual(il.get_collected_values('name'), [u'marta']) + + il = NameFirstItemLoader() + il.replace_value('name', u'marta') + self.assertEqual(il.get_collected_values('name'), [u'marta']) + il = NameFirstItemLoader() + il.replace_value('name', [u'marta', u'jose']) + self.assertEqual(il.get_collected_values('name'), [u'marta']) + + il = NameFirstItemLoader() + il.add_value('name', u'marta') + il.add_value('name', [u'jose', u'pedro']) + self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) + + def test_map_compose_filter(self): + def filter_world(x): + return None if x == 'world' else x + + proc = MapCompose(filter_world, str.upper) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) + + def test_map_compose_filter_multil(self): + class TestItemLoader(NameItemLoader): + name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) + + il = TestItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'Mart']) + item = il.load_item() + self.assertEqual(item['name'], [u'Mart']) + + def test_default_input_processor(self): + il = DefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'mart']) + + def test_inherited_default_input_processor(self): + class InheritDefaultedItemLoader(DefaultedItemLoader): + pass + + il = InheritDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'mart']) + + def test_input_processor_inheritance(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(lambda v: v.lower()) + + il = ChildItemLoader() + il.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'Marta']) + + class ChildChildItemLoader(ChildItemLoader): + url_in = MapCompose(lambda v: v.upper()) + summary_in = MapCompose(lambda v: v) + + il = ChildChildItemLoader() + il.add_value('url', u'http://scrapy.org') + self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'Marta']) + + def test_empty_map_compose(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): + name_in = MapCompose() + + il = IdentityDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'marta']) + + def test_identity_input_processor(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): + name_in = Identity() + + il = IdentityDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'marta']) + + def test_extend_custom_input_processors(self): + class ChildItemLoader(TestItemLoader): + name_in = MapCompose(TestItemLoader.name_in, str.swapcase) + + il = ChildItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'mARTA']) + + def test_extend_default_input_processors(self): + class ChildDefaultedItemLoader(DefaultedItemLoader): + name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) + + il = ChildDefaultedItemLoader() + il.add_value('name', u'marta') + self.assertEqual(il.get_output_value('name'), [u'MART']) + + def test_output_processor_using_function(self): + il = TestItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemLoader(TestItemLoader): + name_out = u" ".join + + il = TakeFirstItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), u'Mar Ta') + + def test_output_processor_error(self): + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_out = MapCompose(float) + + il = TestItemLoader() + il.add_value('name', [u'$10']) + try: + float(u'$10') + except Exception as e: + expected_exc_str = str(e) + + exc = None + try: + il.load_item() + except Exception as e: + exc = e + assert isinstance(exc, ValueError) + s = str(exc) + assert 'name' in s, s + assert '$10' in s, s + assert 'ValueError' in s, s + assert expected_exc_str in s, s + + def test_output_processor_using_classes(self): + il = TestItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemLoader(TestItemLoader): + name_out = Join() + + il = TakeFirstItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), u'Mar Ta') + + class TakeFirstItemLoader(TestItemLoader): + name_out = Join("<br>") + + il = TakeFirstItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta') + + def test_default_output_processor(self): + il = TestItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + class LalaItemLoader(TestItemLoader): + default_output_processor = Identity() + + il = LalaItemLoader() + il.add_value('name', [u'mar', u'ta']) + self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + + def test_loader_context_on_declaration(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor_with_args, key=u'val') + + il = ChildItemLoader() + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['val']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['val']) + + def test_loader_context_on_instantiation(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor_with_args) + + il = ChildItemLoader(key=u'val') + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['val']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['val']) + + def test_loader_context_on_assign(self): + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor_with_args) + + il = ChildItemLoader() + il.context['key'] = u'val' + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['val']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['val']) + + def test_item_passed_to_input_processor_functions(self): + def processor(value, loader_context): + return loader_context['item']['name'] + + class ChildItemLoader(TestItemLoader): + url_in = MapCompose(processor) + + it = TestItem(name='marta') + il = ChildItemLoader(item=it) + il.add_value('url', u'text') + self.assertEqual(il.get_output_value('url'), ['marta']) + il.replace_value('url', u'text2') + self.assertEqual(il.get_output_value('url'), ['marta']) + + def test_compose_processor(self): + class TestItemLoader(NameItemLoader): + name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) + + il = TestItemLoader() + il.add_value('name', [u'marta', u'other']) + self.assertEqual(il.get_output_value('name'), u'Mart') + item = il.load_item() + self.assertEqual(item['name'], u'Mart') + + def test_partial_processor(self): + def join(values, sep=None, loader_context=None, ignored=None): + if sep is not None: + return sep.join(values) + elif loader_context and 'sep' in loader_context: + return loader_context['sep'].join(values) + else: + return ''.join(values) + + class TestItemLoader(NameItemLoader): + name_out = Compose(partial(join, sep='+')) + url_out = Compose(partial(join, loader_context={'sep': '.'})) + summary_out = Compose(partial(join, ignored='foo')) + + il = TestItemLoader() + il.add_value('name', [u'rabbit', u'hole']) + il.add_value('url', [u'rabbit', u'hole']) + il.add_value('summary', [u'rabbit', u'hole']) + item = il.load_item() + self.assertEqual(item['name'], u'rabbit+hole') + self.assertEqual(item['url'], u'rabbit.hole') + self.assertEqual(item['summary'], u'rabbithole') + + def test_error_input_processor(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_in = MapCompose(float) + + il = TestItemLoader() + self.assertRaises(ValueError, il.add_value, 'name', + [u'marta', u'other']) + + def test_error_output_processor(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + name_out = Compose(Join(), float) + + il = TestItemLoader() + il.add_value('name', u'marta') + with self.assertRaises(ValueError): + il.load_item() + + def test_error_processor_as_argument(self): + class TestItem(Item): + name = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + + il = TestItemLoader() + self.assertRaises(ValueError, il.add_value, 'name', + [u'marta', u'other'], Compose(float)) + + +class InitializationFromDictTest(unittest.TestCase): + + item_class = dict + + def test_keep_single_value(self): + """Loaded item should contain values from the initial item""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo']}) + + def test_keep_list(self): + """Loaded item should contain values from the initial item""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + + def test_add_value_singlevalue_singlevalue(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + il.add_value('name', 'bar') + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']}) + + def test_add_value_singlevalue_list(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + il.add_value('name', ['item', 'loader']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']}) + + def test_add_value_list_singlevalue(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + il.add_value('name', 'qwerty') + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']}) + + def test_add_value_list_list(self): + """Values added after initialization should be appended""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + il.add_value('name', ['item', 'loader']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']}) + + def test_get_output_value_singlevalue(self): + """Getting output value must not remove value from item""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + self.assertEqual(il.get_output_value('name'), ['foo']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(loaded_item, dict({'name': ['foo']})) + + def test_get_output_value_list(self): + """Getting output value must not remove value from item""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + self.assertEqual(il.get_output_value('name'), ['foo', 'bar']) + loaded_item = il.load_item() + self.assertIsInstance(loaded_item, self.item_class) + self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']})) + + def test_values_single(self): + """Values from initial item must be added to loader._values""" + input_item = self.item_class(name='foo') + il = ItemLoader(item=input_item) + self.assertEqual(il._values.get('name'), ['foo']) + + def test_values_list(self): + """Values from initial item must be added to loader._values""" + input_item = self.item_class(name=['foo', 'bar']) + il = ItemLoader(item=input_item) + self.assertEqual(il._values.get('name'), ['foo', 'bar']) + + +class BaseNoInputReprocessingLoader(ItemLoader): + title_in = MapCompose(str.upper) + title_out = TakeFirst() + + +class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader): + default_item_class = dict + + +class NoInputReprocessingFromDictTest(unittest.TestCase): + """ + Loaders initialized from loaded items must not reprocess fields (dict instances) + """ + def test_avoid_reprocessing_with_initial_values_single(self): + il = NoInputReprocessingDictLoader(item=dict(title='foo')) + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='foo')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) + + def test_avoid_reprocessing_with_initial_values_list(self): + il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar'])) + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='foo')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo')) + + def test_avoid_reprocessing_without_initial_values_single(self): + il = NoInputReprocessingDictLoader() + il.add_value('title', 'foo') + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='FOO')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) + + def test_avoid_reprocessing_without_initial_values_list(self): + il = NoInputReprocessingDictLoader() + il.add_value('title', ['foo', 'bar']) + il_loaded = il.load_item() + self.assertEqual(il_loaded, dict(title='FOO')) + self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO')) + + +class TestOutputProcessorDict(unittest.TestCase): + def test_output_processor(self): + + class TempDict(dict): + def __init__(self, *args, **kwargs): + super(TempDict, self).__init__(self, *args, **kwargs) + self.setdefault('temp', 0.3) + + class TempLoader(ItemLoader): + default_item_class = TempDict + default_input_processor = Identity() + default_output_processor = Compose(TakeFirst()) + + loader = TempLoader() + item = loader.load_item() + self.assertIsInstance(item, TempDict) + self.assertEqual(dict(item), {'temp': 0.3}) + + +class ProcessorsTest(unittest.TestCase): + + def test_take_first(self): + proc = TakeFirst() + self.assertEqual(proc([None, '', 'hello', 'world']), 'hello') + self.assertEqual(proc([None, '', 0, 'hello', 'world']), 0) + + def test_identity(self): + proc = Identity() + self.assertEqual(proc([None, '', 'hello', 'world']), + [None, '', 'hello', 'world']) + + def test_join(self): + proc = Join() + self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) + self.assertEqual(proc(['', 'hello', 'world']), u' hello world') + self.assertEqual(proc(['hello', 'world']), u'hello world') + self.assertIsInstance(proc(['hello', 'world']), str) + + def test_compose(self): + proc = Compose(lambda v: v[0], str.upper) + self.assertEqual(proc(['hello', 'world']), 'HELLO') + proc = Compose(str.upper) + self.assertEqual(proc(None), None) + proc = Compose(str.upper, stop_on_none=False) + self.assertRaises(ValueError, proc, None) + proc = Compose(str.upper, lambda x: x + 1) + self.assertRaises(ValueError, proc, 'hello') + + def test_mapcompose(self): + def filter_world(x): + return None if x == 'world' else x + proc = MapCompose(filter_world, str.upper) + self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), + [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + proc = MapCompose(filter_world, str.upper) + self.assertEqual(proc(None), []) + proc = MapCompose(filter_world, str.upper) + self.assertRaises(ValueError, proc, [1]) + proc = MapCompose(filter_world, lambda x: x + 1) + self.assertRaises(ValueError, proc, 'hello') + + +class SelectJmesTestCase(unittest.TestCase): + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } + + def test_output(self): + for tl in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[tl] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(tl, test, expected) + ) + + +# Functions as processors + +def function_processor_strip(iterable): + return [x.strip() for x in iterable] + + +def function_processor_upper(iterable): + return [x.upper() for x in iterable] + + +class FunctionProcessorItem(Item): + foo = Field( + input_processor=function_processor_strip, + output_processor=function_processor_upper, + ) + + +class FunctionProcessorDictLoader(ItemLoader): + default_item_class = dict + foo_in = function_processor_strip + foo_out = function_processor_upper + + +class FunctionProcessorTestCase(unittest.TestCase): + + def test_processor_defined_in_item_loader(self): + lo = FunctionProcessorDictLoader() + lo.add_value('foo', ' bar ') + lo.add_value('foo', [' asdf ', ' qwerty ']) + self.assertEqual( + dict(lo.load_item()), + {'foo': ['BAR', 'ASDF', 'QWERTY']} + ) + + +class DeprecatedUtilityFunctionsTestCase(unittest.TestCase): + + def test_deprecated_wrap_loader_context(self): + def function(*args): + return None + + with warnings.catch_warnings(record=True) as w: + wrap_loader_context(function, context=dict()) + + assert len(w) == 1 + assert issubclass(w[0].category, ScrapyDeprecationWarning) + + def test_deprecated_extract_regex(self): + with warnings.catch_warnings(record=True) as w: + extract_regex(r'\w+', 'this is a test') + + assert len(w) == 1 + assert issubclass(w[0].category, ScrapyDeprecationWarning) + + +if __name__ == "__main__": + unittest.main() From 8bdcdb0a76e3780681dcfbbd4a0eee62e2bb05b1 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Jul 2020 09:13:54 +0300 Subject: [PATCH 096/125] Add quotes to example in docs --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 0bb5f1733..7e91b365d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -461,7 +461,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o dirname/%(batch_id)d-filename%(batch_time)s.json + scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json' The command line above can generate a directory tree like:: From 0e0d1ad64323033017c4a01893b5337a980cb5ec Mon Sep 17 00:00:00 2001 From: Marc <noviluni@gmail.com> Date: Thu, 16 Jul 2020 14:19:46 +0200 Subject: [PATCH 097/125] remove python 2 reminiscence in cookies --- scrapy/http/cookies.py | 3 --- tests/test_http_cookies.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 3e810992c..0c97e6999 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -186,9 +186,6 @@ class WrappedResponse: def info(self): return self - # python3 cookiejars calls get_all def get_all(self, name, default=None): return [to_unicode(v, errors='replace') for v in self.response.headers.getlist(name)] - # python2 cookiejars calls getheaders - getheaders = get_all diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 45ddb42ba..540e27907 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -64,9 +64,6 @@ class WrappedResponseTest(TestCase): def test_info(self): self.assertIs(self.wrapped.info(), self.wrapped) - def test_getheaders(self): - self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) - def test_get_all(self): # get_all result must be native string self.assertEqual(self.wrapped.get_all('content-type'), ['text/html']) From b97a39fda0b72d8a4dcd631599e5d9bf530a5bee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com> Date: Thu, 16 Jul 2020 17:38:22 +0200 Subject: [PATCH 098/125] deprecate retry_on_eintr (#4683) --- scrapy/utils/python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index afa8a8135..9204977cf 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -285,6 +285,7 @@ class WeakKeyCache: return self._weakdict[key] +@deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" while True: From 41263f61c6de8048023ba4c80e062f56b21e5a19 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo <johngolt33@gmail.com> Date: Thu, 16 Jul 2020 18:41:45 +0300 Subject: [PATCH 099/125] Change single quotes to double in example in docs --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7e91b365d..fdc6e7cba 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -461,7 +461,7 @@ For instance, if your settings include:: And your :command:`crawl` command line is:: - scrapy crawl spidername -o 'dirname/%(batch_id)d-filename%(batch_time)s.json' + scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json" The command line above can generate a directory tree like:: From d29bec60d795b13ecb6e5978cb9e4d8fbd298b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 16 Jul 2020 23:19:24 +0200 Subject: [PATCH 100/125] Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest) (#4504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest) * Log a detailed error message to discover why MockServer is not working * Go for all lines! * Disable tests based on mitmproxy while running on PyPy * Fix test_get_func_args for PyPy 3.6+ * Make testPayloadDefaultCiphers work regardless of OpenSSL default ciphers * Crossing fingers… * Rename: testPayloadDefaultCiphers → testPayloadDisabledCipher * Test the PyPy version currently documented as the minimum required version * Fix the PYPY_VERSION tag * Update the documentation about supported PyPy versions * Also test the latest 3.5 Python version with PyPy * Fix the PYPY_VERSION value for the latest 3.5 version * Use pinned dependencies for asyncio and PyPy tests against oldest supported Python versions * Fix PyPy installation for the pypy3-pinned Tox environment * Try installing Cython * Maybe PyPy requires lxml 3.6.0? * install.rst: minor clarification * lxml 4.0.0 is required on PyPy * Require setuptools 18.5+ * Revert "Require setuptools 18.5+" This reverts commit 017ec33ac2d237523cdd53be9be8169dd540759e. * Maintain lxml as a dependency if setuptools < 18.5 is used --- .travis.yml | 16 +++++++++----- docs/faq.rst | 14 ------------ docs/intro/install.rst | 12 +++++++--- setup.py | 44 +++++++++++++++++++++++-------------- tests/test_proxy_connect.py | 2 ++ tests/test_utils_python.py | 6 ++++- tests/test_webclient.py | 6 +++-- tox.ini | 31 +++++++++++++++++--------- 8 files changed, 80 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index b403ac54c..db720b918 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,19 +18,25 @@ matrix: - env: TOXENV=typing python: 3.8 - - env: TOXENV=pypy3 - env: TOXENV=pinned python: 3.5.2 - - env: TOXENV=asyncio + - env: TOXENV=asyncio-pinned python: 3.5.2 # We use additional code to support 3.5.3 and earlier + - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0 + - env: TOXENV=py python: 3.5 - env: TOXENV=asyncio python: 3.5 # We use specific code to support >= 3.5.4, < 3.6 + - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0 + - env: TOXENV=py python: 3.6 + - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1 + - env: TOXENV=py python: 3.7 + - env: TOXENV=py PYPI_RELEASE_JOB=true python: 3.8 dist: bionic @@ -42,9 +48,9 @@ matrix: dist: bionic install: - | - if [ "$TOXENV" = "pypy3" ]; then - export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" - wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + if [[ ! -z "$PYPY_VERSION" ]]; then + export PYPY_VERSION="pypy$PYPY_VERSION-linux64" + wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2" tar -jxf ${PYPY_VERSION}.tar.bz2 virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" diff --git a/docs/faq.rst b/docs/faq.rst index d5ea3cb87..ea2c8216f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars .. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use -.. _faq-python-versions: - -What Python versions does Scrapy support? ------------------------------------------ - -Scrapy is supported under Python 3.5.2+ -under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). -Python 3 support was added in Scrapy 1.1. -PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. -Python 2 support was dropped in Scrapy 2.0. - -.. note:: - For Python 3 support on Windows, it is recommended to use - Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`. Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index fb64d443c..6d65ae2ee 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -4,12 +4,18 @@ Installation guide ================== +.. _faq-python-versions: + +Supported Python versions +========================= + +Scrapy requires Python 3.5.2+, either the CPython implementation (default) or +the PyPy 5.9+ implementation (see :ref:`python:implementations`). + + Installing Scrapy ================= -Scrapy runs on Python 3.5.2 or above under CPython (default Python -implementation) and PyPy (starting with PyPy 5.9). - If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows and macOS. diff --git a/setup.py b/setup.py index f8d9b491b..58090f7a2 100644 --- a/setup.py +++ b/setup.py @@ -18,12 +18,39 @@ def has_environment_marker_platform_impl_support(): return parse_version(setuptools_version) >= parse_version('18.5') +install_requires = [ + 'Twisted>=17.9.0', + 'cryptography>=2.0', + 'cssselect>=0.9.1', + 'itemloaders>=1.0.1', + 'lxml>=3.5.0', + 'parsel>=1.5.0', + 'PyDispatcher>=2.0.5', + 'pyOpenSSL>=16.2.0', + 'queuelib>=1.4.2', + 'service_identity>=16.0.0', + 'w3lib>=1.17.0', + 'zope.interface>=4.1.3', + 'protego>=0.1.15', + 'itemadapter>=0.1.0', +] extras_require = {} if has_environment_marker_platform_impl_support(): + extras_require[':platform_python_implementation == "CPython"'] = [ + 'lxml>=3.5.0', + ] extras_require[':platform_python_implementation == "PyPy"'] = [ + # Earlier lxml versions are affected by + # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no, + # which was fixed in Cython 0.26, released on 2017-06-19, and used to + # generate the C headers of lxml release tarballs published since then, the + # first of which was: + 'lxml>=4.0.0', 'PyPyDispatcher>=2.1.0', ] +else: + install_requires.append('lxml>=3.5.0') setup( @@ -67,21 +94,6 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], python_requires='>=3.5.2', - install_requires=[ - 'Twisted>=17.9.0', - 'cryptography>=2.0', - 'cssselect>=0.9.1', - 'itemloaders>=1.0.1', - 'lxml>=3.5.0', - 'parsel>=1.5.0', - 'PyDispatcher>=2.0.5', - 'pyOpenSSL>=16.2.0', - 'queuelib>=1.4.2', - 'service_identity>=16.0.0', - 'w3lib>=1.17.0', - 'zope.interface>=4.1.3', - 'protego>=0.1.15', - 'itemadapter>=0.1.0', - ], + install_requires=install_requires, extras_require=extras_require, ) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index fc5658ae7..a56e3c39a 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -60,6 +60,8 @@ def _wrong_credentials(proxy_url): @skipIf(sys.version_info < (3, 5, 4), "requires mitmproxy < 3.0.0, which these tests do not support") +@skipIf("pypy" in sys.executable, + "mitmproxy does not support PyPy") @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7), "mitmproxy does not support Windows when running Python < 3.7") class ProxyConnectTestCase(TestCase): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 65e6ba876..ebce3c079 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -4,6 +4,7 @@ import operator import platform import unittest from itertools import count +from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( @@ -214,9 +215,12 @@ class UtilsPythonTestCase(unittest.TestCase): else: self.assertEqual( get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ['obj']) + if version_info < (3, 6): + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) + else: + self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable']) def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index c1c5945c2..ee64d455c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -413,7 +413,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): self.getURL("payload"), body=s, contextFactory=client_context_factory ).addCallback(self.assertEqual, to_bytes(s)) - def testPayloadDefaultCiphers(self): + def testPayloadDisabledCipher(self): s = "0123456789" * 10 - d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory()) + settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) + client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) + d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory) return self.assertFailure(d, OpenSSL.SSL.Error) diff --git a/tox.ini b/tox.ini index 5d79739bb..4557c63e3 100644 --- a/tox.ini +++ b/tox.ini @@ -58,11 +58,6 @@ deps = commands = pylint conftest.py docs extras scrapy setup.py tests -[testenv:pypy3] -basepython = pypy3 -commands = - py.test {posargs:--durations=10 docs scrapy tests} - [pinned] deps = -ctests/constraints.txt @@ -85,7 +80,6 @@ deps = Pillow==3.4.2 [testenv:pinned] -basepython = python3 deps = {[pinned]deps} lxml==3.5.0 @@ -104,6 +98,27 @@ deps = reppy robotexclusionrulesparser +[testenv:asyncio] +commands = + {[testenv]commands} --reactor=asyncio + +[testenv:asyncio-pinned] +commands = {[testenv:asyncio]commands} +deps = {[testenv:pinned]deps} + +[testenv:pypy3] +basepython = pypy3 +commands = + py.test {posargs:--durations=10 docs scrapy tests} + +[testenv:pypy3-pinned] +basepython = {[testenv:pypy3]basepython} +commands = {[testenv:pypy3]commands} +deps = + {[pinned]deps} + lxml==4.0.0 + PyPyDispatcher==2.1.0 + [docs] changedir = docs deps = @@ -135,7 +150,3 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:asyncio] -commands = - {[testenv]commands} --reactor=asyncio From 62a4ede5e995f83abd5a90f7dd6ac242f2f3870d Mon Sep 17 00:00:00 2001 From: Artur Shellunts <shellunts.artur@gmail.com> Date: Fri, 17 Jul 2020 12:40:04 +0200 Subject: [PATCH 101/125] Remove deprecated classes BaseSgmlLinkExtractor, RegexLinkExtractor and SgmlLinkExtractor (#4356) --- scrapy/linkextractors/regex.py | 41 ----- scrapy/linkextractors/sgml.py | 151 ------------------ tests/ignores.txt | 2 - .../link_extractor/linkextractor.html | 2 +- .../link_extractor/linkextractor_latin1.html | 2 +- 5 files changed, 2 insertions(+), 196 deletions(-) delete mode 100644 scrapy/linkextractors/regex.py delete mode 100644 scrapy/linkextractors/sgml.py diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py deleted file mode 100644 index 3f2557248..000000000 --- a/scrapy/linkextractors/regex.py +++ /dev/null @@ -1,41 +0,0 @@ -import re -from urllib.parse import urljoin - -from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url - -from scrapy.link import Link -from scrapy.linkextractors.sgml import SgmlLinkExtractor - - -linkre = re.compile( - "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>", - re.DOTALL | re.IGNORECASE) - - -def clean_link(link_text): - """Remove leading and trailing whitespace and punctuation""" - return link_text.strip("\t\r\n '\"\x0c") - - -class RegexLinkExtractor(SgmlLinkExtractor): - """High performant link extractor""" - - def _extract_links(self, response_text, response_url, response_encoding, base_url=None): - def clean_text(text): - return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip() - - def clean_url(url): - clean_url = '' - try: - clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding)))) - except ValueError: - pass - return clean_url - - if base_url is None: - base_url = get_base_url(response_text, response_url, response_encoding) - - links_text = linkre.findall(response_text) - return [Link(clean_url(url).encode(response_encoding), - clean_text(text)) - for url, _, text in links_text] diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py deleted file mode 100644 index 2ba6bca45..000000000 --- a/scrapy/linkextractors/sgml.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -SGMLParser-based Link extractors -""" -import warnings -from urllib.parse import urljoin -from sgmllib import SGMLParser - -from w3lib.url import safe_url_string, canonicalize_url -from w3lib.html import strip_html5_whitespace - -from scrapy.link import Link -from scrapy.linkextractors import FilteringLinkExtractor -from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_unicode -from scrapy.utils.response import get_base_url -from scrapy.exceptions import ScrapyDeprecationWarning - - -class BaseSgmlLinkExtractor(SGMLParser): - - def __init__(self, tag="a", attr="href", unique=False, process_value=None, - strip=True, canonicalized=False): - warnings.warn( - "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.linkextractors.LinkExtractor", - ScrapyDeprecationWarning, stacklevel=2, - ) - SGMLParser.__init__(self) - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_value = (lambda v: v) if process_value is None else process_value - self.current_link = None - self.unique = unique - self.strip = strip - if canonicalized: - self.link_key = lambda link: link.url - else: - self.link_key = lambda link: canonicalize_url(link.url, - keep_fragments=True) - - def _extract_links(self, response_text, response_url, response_encoding, base_url=None): - """ Do the real extraction work """ - self.reset() - self.feed(response_text) - self.close() - - ret = [] - if base_url is None: - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url - for link in self.links: - if isinstance(link.url, str): - link.url = link.url.encode(response_encoding) - try: - link.url = urljoin(base_url, link.url) - except ValueError: - continue - link.url = safe_url_string(link.url, response_encoding) - link.text = to_unicode(link.text, response_encoding, errors='replace').strip() - ret.append(link) - - return ret - - def _process_links(self, links): - """ Normalize and filter extracted links - - The subclass should override it if necessary - """ - return unique_list(links, key=self.link_key) if self.unique else links - - def extract_links(self, response): - # wrapper needed to allow to work directly with text - links = self._extract_links(response.body, response.url, response.encoding) - links = self._process_links(links) - return links - - def reset(self): - SGMLParser.reset(self) - self.links = [] - self.base_url = None - self.current_link = None - - def unknown_starttag(self, tag, attrs): - if tag == 'base': - self.base_url = dict(attrs).get('href') - if self.scan_tag(tag): - for attr, value in attrs: - if self.scan_attr(attr): - if self.strip and value is not None: - value = strip_html5_whitespace(value) - url = self.process_value(value) - if url is not None: - link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel'))) - self.links.append(link) - self.current_link = link - - def unknown_endtag(self, tag): - if self.scan_tag(tag): - self.current_link = None - - def handle_data(self, data): - if self.current_link: - self.current_link.text = self.current_link.text + data - - def matches(self, url): - """This extractor matches with any url, since - it doesn't contain any patterns""" - return True - - -class SgmlLinkExtractor(FilteringLinkExtractor): - - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, - process_value=None, deny_extensions=None, restrict_css=(), - strip=True, restrict_text=()): - warnings.warn( - "SgmlLinkExtractor is deprecated and will be removed in future releases. " - "Please use scrapy.linkextractors.LinkExtractor", - ScrapyDeprecationWarning, stacklevel=2, - ) - - tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs - - with warnings.catch_warnings(): - warnings.simplefilter('ignore', ScrapyDeprecationWarning) - lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip, - canonicalized=canonicalize) - - super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions, - restrict_text=restrict_text) - - def extract_links(self, response): - base_url = None - if self.restrict_xpaths: - base_url = get_base_url(response) - body = u''.join(f - for x in self.restrict_xpaths - for f in response.xpath(x).getall() - ).encode(response.encoding, errors='xmlcharrefreplace') - else: - body = response.body - - links = self._extract_links(body, response.url, response.encoding, base_url) - links = self._process_links(links) - return links diff --git a/tests/ignores.txt b/tests/ignores.txt index f6e0d6fbe..222288841 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,5 +1,3 @@ -scrapy/linkextractors/sgml.py -scrapy/linkextractors/regex.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html index 7d5db368a..2307ea865 100644 --- a/tests/sample_data/link_extractor/linkextractor.html +++ b/tests/sample_data/link_extractor/linkextractor.html @@ -1,7 +1,7 @@ <html> <head> <base href='http://example.com' /> -<title>Sample page with links for testing RegexLinkExtractor +Sample page with links for testing LinkExtractor
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html index fc31d7e5d..e7eee18de 100644 --- a/tests/sample_data/link_extractor/linkextractor_latin1.html +++ b/tests/sample_data/link_extractor/linkextractor_latin1.html @@ -2,7 +2,7 @@ - Sample page with links for testing RegexLinkExtractor + Sample page with links for testing LinkExtractor
From 86f7ac2f2b5d58e0b2588fa2aa4c777a8decf299 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Fri, 17 Jul 2020 17:48:25 +0300 Subject: [PATCH 102/125] Try to fix error at Windows --- tests/test_feedexport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 129b7fc0b..cc124624d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1204,6 +1204,8 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) + if not os.path.exists(str(dir_name)): + continue for file in sorted(os.listdir(dir_name)): with open(os.path.join(dir_name, file), 'rb') as f: data = f.read() From 3e0492741d93b05c464457b3b128a2b0d24c994b Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Sun, 19 Jul 2020 00:10:29 +0300 Subject: [PATCH 103/125] Another try to fix test errors on Windows --- tests/test_feedexport.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cc124624d..c49b2e92f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1190,9 +1190,14 @@ class BatchDeliveriesTest(FeedExportTestBase): def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ + def build_url(path): + if path[0] != '/': + path = '/' + path + return urljoin('file:', path) + FEEDS = settings.get('FEEDS') or {} settings['FEEDS'] = { - urljoin('file:', file_path): feed + build_url(file_path): feed for file_path, feed in FEEDS.items() } content = defaultdict(list) @@ -1204,8 +1209,6 @@ class BatchDeliveriesTest(FeedExportTestBase): for path, feed in FEEDS.items(): dir_name = os.path.dirname(path) - if not os.path.exists(str(dir_name)): - continue for file in sorted(os.listdir(dir_name)): with open(os.path.join(dir_name, file), 'rb') as f: data = f.read() From de297a3a167a6be3ac1ed94a891effefc12a6d00 Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Mon, 20 Jul 2020 17:53:38 +0530 Subject: [PATCH 104/125] enable ANSI color (instead of ANSI color codes) in the Windows terminal #4393 (#4403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * changed ie. -> i.e.(spelling error) on lines 667, 763 (issue scrapy#4332) * updated all text files for issue #4332 (ie. -> i.e.) * Apply ie. → i.e. in source comments * ie → e.g. * modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393) * modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393) * enabled virtual terminal processing (pr #4403) * check for specific windows 10 version (pr #4403) * fixing flake-8 test (pr #4403) * added error handling for terminal info (pr #4403) * corrected stderr (pr #4403) * changed orientation, removed unwanted spaces (pr #4403) * no need for style variable (pr #4403) * fixing trailing whitespaces * commenting windows check * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * small fixes * Shifting _color_support_info() function * enabled virtual terminal processing (pr #4403) * check for specific windows 10 version (pr #4403) * fixing flake-8 test (pr #4403) * added error handling for terminal info (pr #4403) * corrected stderr (pr #4403) * changed orientation, removed unwanted spaces (pr #4403) * no need for style variable (pr #4403) * fixing trailing whitespaces * commenting windows check * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * Update scrapy/utils/display.py Co-Authored-By: Adrián Chaves * small fixes * Shifting _color_support_info() function * error handling * error handlingy * raise ValueError * added in-built function for version comparison * recommit changes * changed check -> parse * version comparison -> parse_version * added scrapy/utils/display.py in pytest.ini * Trigger * Add simple test for scrapy.utils.display._colorize * Flake8: E501 for tests/test_utils_display.py * assertEquals -> assertEqual * Normal formatter for all platforms * separate test for windows * all curses under try block * added global TestStr * more test added * small fix * covering exceptions * windows test failing * Refactor output color handling * Fix pprint test * fix flake8 Co-authored-by: Adrián Chaves Co-authored-by: Eugenio Lacuesta --- scrapy/utils/display.py | 28 +++++++++++-- tests/test_utils_display.py | 78 +++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/test_utils_display.py diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 9735220ef..f4d17224b 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -2,20 +2,42 @@ pprint and pformat wrappers with colorization support """ +import ctypes +import platform import sys +from distutils.version import LooseVersion as parse_version from pprint import pformat as pformat_ +def _enable_windows_terminal_processing(): + # https://stackoverflow.com/a/36760881 + kernel32 = ctypes.windll.kernel32 + return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)) + + +def _tty_supports_color(): + if sys.platform != "win32": + return True + + if parse_version(platform.version()) < parse_version("10.0.14393"): + return True + + # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal + # processing is enabled. + return _enable_windows_terminal_processing() + + def _colorize(text, colorize=True): - if not colorize or not sys.stdout.isatty(): + if not colorize or not sys.stdout.isatty() or not _tty_supports_color(): return text try: from pygments import highlight + except ImportError: + return text + else: from pygments.formatters import TerminalFormatter from pygments.lexers import PythonLexer return highlight(text, PythonLexer(), TerminalFormatter()) - except ImportError: - return text def pformat(obj, *args, **kwargs): diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py new file mode 100644 index 000000000..9ec8311d9 --- /dev/null +++ b/tests/test_utils_display.py @@ -0,0 +1,78 @@ +from io import StringIO + +from unittest import mock, TestCase + +from scrapy.utils.display import pformat, pprint + + +class TestDisplay(TestCase): + object = {'a': 1} + colorized_string = ( + "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'" + "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n" + ) + plain_string = "{'a': 1}" + + @mock.patch('sys.platform', 'linux') + @mock.patch("sys.stdout.isatty") + def test_pformat(self, isatty): + isatty.return_value = True + self.assertEqual(pformat(self.object), self.colorized_string) + + @mock.patch("sys.stdout.isatty") + def test_pformat_dont_colorize(self, isatty): + isatty.return_value = True + self.assertEqual(pformat(self.object, colorize=False), self.plain_string) + + def test_pformat_not_tty(self): + self.assertEqual(pformat(self.object), self.plain_string) + + @mock.patch('sys.platform', 'win32') + @mock.patch('platform.version') + @mock.patch("sys.stdout.isatty") + def test_pformat_old_windows(self, isatty, version): + isatty.return_value = True + version.return_value = '10.0.14392' + self.assertEqual(pformat(self.object), self.colorized_string) + + @mock.patch('sys.platform', 'win32') + @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') + @mock.patch('platform.version') + @mock.patch("sys.stdout.isatty") + def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = '10.0.14393' + terminal_processing.return_value = False + self.assertEqual(pformat(self.object), self.plain_string) + + @mock.patch('sys.platform', 'win32') + @mock.patch('scrapy.utils.display._enable_windows_terminal_processing') + @mock.patch('platform.version') + @mock.patch("sys.stdout.isatty") + def test_pformat_windows(self, isatty, version, terminal_processing): + isatty.return_value = True + version.return_value = '10.0.14393' + terminal_processing.return_value = True + self.assertEqual(pformat(self.object), self.colorized_string) + + @mock.patch('sys.platform', 'linux') + @mock.patch("sys.stdout.isatty") + def test_pformat_no_pygments(self, isatty): + isatty.return_value = True + + import builtins + real_import = builtins.__import__ + + def mock_import(name, globals, locals, fromlist, level): + if 'pygments' in name: + raise ImportError + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = mock_import + self.assertEqual(pformat(self.object), self.plain_string) + builtins.__import__ = real_import + + def test_pprint(self): + with mock.patch('sys.stdout', new=StringIO()) as mock_out: + pprint(self.object) + self.assertEqual(mock_out.getvalue(), "{'a': 1}\n") From 8fae3d5bb768ebb3f9674cc38172226f404fb297 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 22 Jul 2020 16:08:35 -0300 Subject: [PATCH 105/125] Remove monkeypatches module from mypy section in setup.cfg --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 46a3d13fc..f8e7c0c91 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,9 +13,6 @@ follow_imports = skip [mypy-scrapy] ignore_errors = True -[mypy-scrapy._monkeypatches] -ignore_errors = True - [mypy-scrapy.commands] ignore_errors = True From a6c1d79b7cc3bc2c408eab356bbbf99a0536f110 Mon Sep 17 00:00:00 2001 From: BroodingKangaroo Date: Tue, 28 Jul 2020 11:53:05 +0300 Subject: [PATCH 106/125] pep8 tiny changes --- docs/topics/feed-exports.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 2c9774b55..dd4eb3c61 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -453,6 +453,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter FEED_EXPORT_BATCH_ITEM_COUNT ----------------------------- + Default: ``0`` If assigned an integer number higher than ``0``, Scrapy generates multiple output files @@ -474,7 +475,7 @@ generated: For instance, if your settings include:: - FEED_EXPORT_BATCH_ITEM_COUNT=100 + FEED_EXPORT_BATCH_ITEM_COUNT = 100 And your :command:`crawl` command line is:: From 52658539370c442e63102a3208781335953cdf53 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 28 Jul 2020 06:15:14 -0300 Subject: [PATCH 107/125] Use ItemAdapter.field_names when writing header in CsvItemExporter (#4668) --- scrapy/exporters.py | 8 +- tests/test_exporters.py | 203 ++++++++++++++++++++++++++++------------ 2 files changed, 146 insertions(+), 65 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 712572673..0aba1c904 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter): def _write_headers_and_set_fields_to_export(self, item): if self.include_headers_line: if not self.fields_to_export: - if isinstance(item, dict): - # for dicts try using fields of the first item - self.fields_to_export = list(item.keys()) - else: - # use fields declared in Item - self.fields_to_export = list(item.fields.keys()) + # use declared field names, or keys if the item is a dict + self.fields_to_export = ItemAdapter(item).field_names() row = list(self._build_row(self.fields_to_export)) self.csv_writer.writerow(row) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index b27380309..25da54a65 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -8,6 +8,7 @@ from io import BytesIO from datetime import datetime import lxml.etree +from itemadapter import ItemAdapter from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -23,10 +24,37 @@ class TestItem(Item): age = Field() +def custom_serializer(value): + return str(int(value) + 2) + + +class CustomFieldItem(Item): + name = Field() + age = Field(serializer=custom_serializer) + + +try: + from dataclasses import make_dataclass, field +except ImportError: + TestDataClass = None + CustomFieldDataclass = None +else: + TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)]) + CustomFieldDataclass = make_dataclass( + "CustomFieldDataclass", + [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))] + ) + + class BaseItemExporterTest(unittest.TestCase): + item_class = TestItem + custom_field_item_class = CustomFieldItem + def setUp(self): - self.i = TestItem(name=u'John\xa3', age=u'22') + if self.item_class is None: + raise unittest.SkipTest("item class is None") + self.i = self.item_class(name=u'John\xa3', age=u'22') self.output = BytesIO() self.ie = self._get_exporter() @@ -39,7 +67,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): exported_dict[k] = to_unicode(v) - self.assertEqual(self.i, exported_dict) + self.assertEqual(self.i, self.item_class(**exported_dict)) def _get_nonstring_types_item(self): return { @@ -63,13 +91,14 @@ class BaseItemExporterTest(unittest.TestCase): self.assertItemExportWorks(self.i) def test_export_dict_item(self): - self.assertItemExportWorks(dict(self.i)) + self.assertItemExportWorks(ItemAdapter(self.i).asdict()) def test_serialize_field(self): - res = self.ie.serialize_field(self.i.fields['name'], 'name', self.i['name']) + a = ItemAdapter(self.i) + res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) self.assertEqual(res, u'John\xa3') - res = self.ie.serialize_field(self.i.fields['age'], 'age', self.i['age']) + res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) self.assertEqual(res, u'22') def test_fields_to_export(self): @@ -82,18 +111,16 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): - def custom_serializer(value): - return str(int(value) + 2) - - class CustomFieldItem(Item): - name = Field() - age = Field(serializer=custom_serializer) - - i = CustomFieldItem(name=u'John\xa3', age=u'22') - + i = self.custom_field_item_class(name=u'John\xa3', age=u'22') + a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') + + +class BaseItemExporterDataclassTest(BaseItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass class PythonItemExporterTest(BaseItemExporterTest): @@ -105,9 +132,9 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i3 = self.item_class(name=u'Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) @@ -119,9 +146,9 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = TestItem(name=u'Joseph', age='22') - i2 = TestItem(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i1 = self.item_class(name=u'Joseph', age='22') + i2 = self.item_class(name=u'Maria', age=[i1]) + i3 = self.item_class(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( @@ -132,9 +159,9 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=[i1]) - i3 = TestItem(name=u'Jesus', age=[i2]) + i3 = self.item_class(name=u'Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( @@ -146,7 +173,7 @@ class PythonItemExporterTest(BaseItemExporterTest): def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = TestItem(name=u'John\xa3', age=u'22') + value = self.item_class(name=u'John\xa3', age=u'22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -157,6 +184,11 @@ class PythonItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class PythonItemExporterDataclassTest(PythonItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PprintItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -166,6 +198,11 @@ class PprintItemExporterTest(BaseItemExporterTest): self._assert_expected_item(eval(self.output.getvalue())) +class PprintItemExporterDataclassTest(PprintItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class PickleItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -175,8 +212,8 @@ class PickleItemExporterTest(BaseItemExporterTest): self._assert_expected_item(pickle.loads(self.output.getvalue())) def test_export_multiple_items(self): - i1 = TestItem(name='hello', age='world') - i2 = TestItem(name='bye', age='world') + i1 = self.item_class(name='hello', age='world') + i2 = self.item_class(name='bye', age='world') f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() @@ -184,8 +221,8 @@ class PickleItemExporterTest(BaseItemExporterTest): ie.export_item(i2) ie.finish_exporting() f.seek(0) - self.assertEqual(pickle.load(f), i1) - self.assertEqual(pickle.load(f), i2) + self.assertEqual(self.item_class(**pickle.load(f)), i1) + self.assertEqual(self.item_class(**pickle.load(f)), i2) def test_nonstring_types_item(self): item = self._get_nonstring_types_item() @@ -197,6 +234,11 @@ class PickleItemExporterTest(BaseItemExporterTest): self.assertEqual(pickle.loads(fp.getvalue()), item) +class PickleItemExporterDataclassTest(PickleItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class MarshalItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -219,6 +261,11 @@ class MarshalItemExporterTest(BaseItemExporterTest): self.assertEqual(marshal.load(fp), item) +class MarshalItemExporterDataclassTest(MarshalItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) @@ -245,18 +292,18 @@ class CsvItemExporterTest(BaseItemExporterTest): def test_header_export_all(self): self.assertExportResult( item=self.i, - fields_to_export=self.i.fields.keys(), + fields_to_export=ItemAdapter(self.i).field_names(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_all_dict(self): self.assertExportResult( - item=dict(self.i), + item=ItemAdapter(self.i).asdict(), expected=b'age,name\r\n22,John\xc2\xa3\r\n', ) def test_header_export_single_field(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, fields_to_export=['age'], @@ -264,7 +311,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ) def test_header_export_two_items(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: output = BytesIO() ie = CsvItemExporter(output) ie.start_exporting() @@ -275,7 +322,7 @@ class CsvItemExporterTest(BaseItemExporterTest): b'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') def test_header_no_header_line(self): - for item in [self.i, dict(self.i)]: + for item in [self.i, ItemAdapter(self.i).asdict()]: self.assertExportResult( item=item, include_headers_line=False, @@ -309,6 +356,11 @@ class CsvItemExporterTest(BaseItemExporterTest): ) +class CsvItemExporterDataclassTest(CsvItemExporterTest): + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class XmlItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -318,8 +370,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def xmltuple(elem): children = list(elem.iterchildren()) if children: - return [(child.tag, sorted(xmltuple(child))) - for child in children] + return [(child.tag, sorted(xmltuple(child))) for child in children] else: return [(elem.tag, [(elem.text, ())])] @@ -345,17 +396,21 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - TestItem(name=[u'John\xa3', u'Doe']), - ( - b'\n' - b'John\xc2\xa3Doe' - ) + self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]), + b"""\n + + + John\xc2\xa3Doe + 123 + + + """ ) def test_nested_item(self): - i1 = TestItem(name=u'foo\xa3hoo', age='22') + i1 = dict(name=u'foo\xa3hoo', age='22') i2 = dict(name=u'bar', age=i1) - i3 = TestItem(name=u'buz', age=i2) + i3 = self.item_class(name=u'buz', age=i2) self.assertExportResult( i3, @@ -376,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = TestItem(name=u'foo') + i1 = dict(name=u'foo') i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = TestItem(name=u'buz', age=[i1, i2]) + i3 = self.item_class(name=u'buz', age=[i1, i2]) self.assertExportResult( i3, @@ -412,6 +467,12 @@ class XmlItemExporterTest(BaseItemExporterTest): ) +class XmlItemExporterDataclassTest(XmlItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonLinesItemExporterTest(BaseItemExporterTest): _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} @@ -421,12 +482,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, dict(self.i)) + self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph', age='22') + i1 = self.item_class(name=u'Joseph', age='22') i2 = dict(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i3 = self.item_class(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -449,6 +510,12 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, item) +class JsonLinesItemExporterDataclassTest(JsonLinesItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + class JsonItemExporterTest(JsonLinesItemExporterTest): _expected_nested = [JsonLinesItemExporterTest._expected_nested] @@ -458,7 +525,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): def _check_output(self): exported = json.loads(to_unicode(self.output.getvalue().strip())) - self.assertEqual(exported, [dict(self.i)]) + self.assertEqual(exported, [ItemAdapter(self.i).asdict()]) def assertTwoItemsExported(self, item): self.ie.start_exporting() @@ -466,28 +533,28 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.ie.export_item(item) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - self.assertEqual(exported, [dict(item), dict(item)]) + self.assertEqual(exported, [ItemAdapter(item).asdict(), ItemAdapter(item).asdict()]) def test_two_items(self): self.assertTwoItemsExported(self.i) def test_two_dict_items(self): - self.assertTwoItemsExported(dict(self.i)) + self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = TestItem(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) - i3 = TestItem(name=u'Jesus', age=i2) + i1 = self.item_class(name=u'Joseph\xa3', age='22') + i2 = self.item_class(name=u'Maria', age=i1) + i3 = self.item_class(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}} + expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): i1 = dict(name=u'Joseph\xa3', age='22') - i2 = TestItem(name=u'Maria', age=i1) + i2 = self.item_class(name=u'Maria', age=i1) i3 = dict(name=u'Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) @@ -506,7 +573,19 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertEqual(exported, [item]) -class CustomItemExporterTest(unittest.TestCase): +class JsonItemExporterDataclassTest(JsonItemExporterTest): + + item_class = TestDataClass + custom_field_item_class = CustomFieldDataclass + + +class CustomExporterItemTest(unittest.TestCase): + + item_class = TestItem + + def setUp(self): + if self.item_class is None: + raise unittest.SkipTest("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): @@ -516,16 +595,22 @@ class CustomItemExporterTest(unittest.TestCase): else: return super(CustomItemExporter, self).serialize_field(field, name, value) - i = TestItem(name=u'John', age='22') + i = self.item_class(name=u'John', age='22') + a = ItemAdapter(i) ie = CustomItemExporter() - self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John') - self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '23') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') + self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') i2 = {'name': u'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') +class CustomExporterDataclassTest(CustomExporterItemTest): + + item_class = TestDataClass + + if __name__ == '__main__': unittest.main() From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 10:16:18 +0530 Subject: [PATCH 108/125] Code cleanup scrapy.utils.python.WeakKeyCache #4684 --- scrapy/utils/python.py | 12 ------------ tests/test_utils_python.py | 18 +----------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9204977cf..7a393925e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..b23ae2e52 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,7 +9,7 @@ from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) @@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - wk = WeakKeyCache(lambda k: next(_values)) - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) def test_get_func_args(self): def f1(a, b, c): From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 18:05:33 +0530 Subject: [PATCH 109/125] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- tests/test_utils_python.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b23ae2e52..5a53d89e4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,9 +1,7 @@ import functools -import gc import operator import platform import unittest -from itertools import count from sys import version_info from warnings import catch_warnings @@ -12,7 +10,6 @@ from scrapy.utils.python import ( get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) - __doctests__ = ['scrapy.utils.python'] @@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_get_func_args(self): def f1(a, b, c): pass From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Thu, 30 Jul 2020 12:25:21 +0530 Subject: [PATCH 111/125] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- scrapy/utils/python.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7a393925e..c8f921ff3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024): In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing the start position of the match, and the ending (regarding the entire text). """ + def _chunk_iter(): offset = len(text) while True: @@ -158,6 +159,7 @@ def memoizemethod_noargs(method): if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] + return new_method @@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes): return True +@deprecated +class WeakKeyCache: + + def __init__(self, default_factory): + self.default_factory = default_factory + self._weakdict = weakref.WeakKeyDictionary() + + def __getitem__(self, key): + if key not in self._weakdict: + self._weakdict[key] = self.default_factory(key) + return self._weakdict[key] + + @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" From 890b2138a605af2bfbf340a0d48d9d83c4cda53b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 13:39:30 +0200 Subject: [PATCH 112/125] Remove the u prefix from strings --- docs/_ext/scrapydocs.py | 2 +- docs/topics/loaders.rst | 6 +- docs/topics/selectors.rst | 4 +- docs/utils/linkfix.py | 2 +- scrapy/http/request/form.py | 6 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/logformatter.py | 2 +- tests/test_cmdline/__init__.py | 2 +- tests/test_downloader_handlers.py | 6 +- tests/test_downloadermiddleware_cookies.py | 16 +- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_redirect.py | 6 +- tests/test_downloadermiddleware_robotstxt.py | 4 +- tests/test_exporters.py | 84 +++---- tests/test_feedexport.py | 8 +- tests/test_http_headers.py | 6 +- tests/test_http_request.py | 90 ++++---- tests/test_http_response.py | 72 +++--- tests/test_item.py | 42 ++-- tests/test_linkextractors.py | 180 +++++++-------- tests/test_loader.py | 124 +++++----- tests/test_loader_deprecated.py | 226 +++++++++---------- tests/test_logformatter.py | 12 +- tests/test_mail.py | 8 +- tests/test_responsetypes.py | 10 +- tests/test_robotstxt_interface.py | 8 +- tests/test_selector.py | 32 +-- tests/test_spider.py | 12 +- tests/test_utils_iterators.py | 90 ++++---- tests/test_utils_python.py | 16 +- tests/test_utils_reqser.py | 2 +- tests/test_utils_template.py | 4 +- 32 files changed, 545 insertions(+), 545 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 192123473..640660943 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -17,7 +17,7 @@ class SettingsListDirective(Directive): def is_setting_index(node): if node.tagname == 'index': # index entries for setting directives look like: - # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')] + # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] entry_type, info, refid = node['entries'][0][:3] return entry_type == 'pair' and info.endswith('; setting') return False diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index d0eeb4097..29d9c5805 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -237,10 +237,10 @@ metadata. Here is an example:: >>> from scrapy.loader import ItemLoader >>> il = ItemLoader(item=Product()) ->>> il.add_value('name', [u'Welcome to my', u'website']) ->>> il.add_value('price', [u'€', u'1000']) +>>> il.add_value('name', ['Welcome to my', 'website']) +>>> il.add_value('price', ['€', '1000']) >>> il.load_item() -{'name': u'Welcome to my website', 'price': u'1000'} +{'name': 'Welcome to my website', 'price': '1000'} The precedence order, for both input and output processors, is as follows: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index bb46ea80f..5014df6ac 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit: >>> from scrapy import Selector ->>> doc = u""" +>>> doc = """ ...
...
    ...
  • first item
    • @@ -765,7 +765,7 @@ extracting text elements for example. Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops:: - >>> doc = u""" + >>> doc = """ ...
    ... Kenmore White 17" Microwave ... Kenmore 17" Microwave diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 9acfc3b23..95a3f17d5 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -23,7 +23,7 @@ def main(): _contents = None # A regex that matches standard linkcheck output lines - line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') + line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 0e6ceef0b..a260798ac 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={ "re": "http://exslt.org/regular-expressions"}) - values = [(k, u'' if v is None else v) + values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -168,7 +168,7 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or u'').strip() for o in selected_options] + v = [(o.get('value') or o.text or '').strip() for o in selected_options] return n, v @@ -205,7 +205,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such - xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) + xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1615d44d7..8b9f961ee 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -76,7 +76,7 @@ class LxmlParserLinkExtractor: url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) - link = Link(url, _collect_string_content(el) or u'', + link = Link(url, _collect_string_content(el) or '', nofollow=rel_has_nofollow(el.get('rel'))) links.append(link) return self._deduplicate_if_needed(links) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 219145f13..0f9e6f1cb 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -44,7 +44,7 @@ class LogFormatter: def dropped(self, item, exception, response, spider): return { 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", 'args': { 'exception': exception, 'item': item, diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index da99a6be8..591075a98 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase): 'EXTENSIONS=' + json.dumps(EXTENSIONS)) # XXX: There's gotta be a smarter way to do this... self.assertNotIn("...", settingsstr) - for char in ("'", "<", ">", 'u"'): + for char in ("'", "<", ">"): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 51deb20f4..57d4cdd6b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "iso-8859-7") @@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(response.body, b'\xbe\xd3\xbe') self.assertEqual(response.encoding, "iso-8859-7") @@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase): def test_mediatype_parameters(self): def _test(response): - self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.text, '\u038e\u03a3\u038e') self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) self.assertEqual(response.encoding, "utf-8") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 9ccc2110b..010577415 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase): def test_request_cookies_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')}) + req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')}) + req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', cookies={'a': u'á'}) + # 3) String + req3 = Request('http://example.org', cookies={'a': 'á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') def test_request_headers_cookie_encoding(self): # 1) UTF8-encoded bytes - req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')}) + req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')}) assert self.mw.process_request(req1, self.spider) is None self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') # 2) Non UTF8-encoded bytes - req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')}) + req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')}) assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') - # 3) Unicode string - req3 = Request('http://example.org', headers={'Cookie': u'a=á'}) + # 3) String + req3 = Request('http://example.org', headers={'Cookie': 'a=á'}) assert self.mw.process_request(req3, self.spider) is None self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 9841d7a76..351631eb8 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase): def test_proxy_auth_encoding(self): # utf-8 encoding - os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128' mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None @@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') @@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase): self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding - req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 919dbed23..131332131 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_latin1_location(self): req = Request('http://scrapytest.org/first') - latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 + latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' @@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding + utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' @@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.mw = MetaRefreshMiddleware.from_crawler(crawler) def _body(self, interval=5, url='http://example.org/newpage'): - html = u"""""" + html = """""" return html.format(interval, url).encode('utf-8') def test_priority_adjust(self): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index b9452a0e7..f9936baba 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = u""" + ROBOTS = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -56,7 +56,7 @@ Disallow: /some/randome/page.html self.assertIgnored(Request('http://site.local/admin/main'), middleware), self.assertIgnored(Request('http://site.local/static/'), middleware), self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware), - self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware) + self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware) ], fireOnOneErrback=True) def test_robotstxt_ready_parser(self): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 25da54a65..660c99ce1 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase): def setUp(self): if self.item_class is None: raise unittest.SkipTest("item class is None") - self.i = self.item_class(name=u'John\xa3', age=u'22') + self.i = self.item_class(name='John\xa3', age='22') self.output = BytesIO() self.ie = self._get_exporter() @@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase): def test_serialize_field(self): a = ItemAdapter(self.i) res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name']) - self.assertEqual(res, u'John\xa3') + self.assertEqual(res, 'John\xa3') res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age']) - self.assertEqual(res, u'22') + self.assertEqual(res, '22') def test_fields_to_export(self): ie = self._get_exporter(fields_to_export=['name']) - self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')]) + self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')]) ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] assert isinstance(name, str) - self.assertEqual(name, u'John\xa3') + self.assertEqual(name, 'John\xa3') def test_field_custom_serializer(self): - i = self.custom_field_item_class(name=u'John\xa3', age=u'22') + i = self.custom_field_item_class(name='John\xa3', age='22') a = ItemAdapter(i) ie = self._get_exporter() - self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3') + self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3') self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24') @@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest): PythonItemExporter(invalid_option='something') def test_nested_item(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual(type(exported), dict) self.assertEqual( exported, - {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'} + {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'} ) self.assertEqual(type(exported['age']), dict) self.assertEqual(type(exported['age']['age']), dict) def test_export_list(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = self.item_class(name=u'Maria', age=[i1]) - i3 = self.item_class(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = self.item_class(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_item_dict_list(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=[i1]) - i3 = self.item_class(name=u'Jesus', age=[i2]) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=[i1]) + i3 = self.item_class(name='Jesus', age=[i2]) ie = self._get_exporter() exported = ie.export_item(i3) self.assertEqual( exported, - {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'} + {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'} ) self.assertEqual(type(exported['age'][0]), dict) self.assertEqual(type(exported['age'][0]['age'][0]), dict) def test_export_binary(self): exporter = PythonItemExporter(binary=True) - value = self.item_class(name=u'John\xa3', age=u'22') + value = self.item_class(name='John\xa3', age='22') expected = {b'name': b'John\xc2\xa3', b'age': b'22'} self.assertEqual(expected, exporter.export_item(value)) @@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest): return self.assertEqual(split_csv(first), split_csv(second), msg=msg) def _check_output(self): - self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n') + self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n') def assertExportResult(self, item, expected, **kwargs): fp = BytesIO() @@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest): def test_multivalued_fields(self): self.assertExportResult( - self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]), + self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]), b"""\n @@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_item(self): - i1 = dict(name=u'foo\xa3hoo', age='22') - i2 = dict(name=u'bar', age=i1) - i3 = self.item_class(name=u'buz', age=i2) + i1 = dict(name='foo\xa3hoo', age='22') + i2 = dict(name='bar', age=i1) + i3 = self.item_class(name='buz', age=i2) self.assertExportResult( i3, @@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest): ) def test_nested_list_item(self): - i1 = dict(name=u'foo') - i2 = dict(name=u'bar', v2={"egg": ["spam"]}) - i3 = self.item_class(name=u'buz', age=[i1, i2]) + i1 = dict(name='foo') + i2 = dict(name='bar', v2={"egg": ["spam"]}) + i3 = self.item_class(name='buz', age=[i1, i2]) self.assertExportResult( i3, @@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} + _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}} def _get_exporter(self, **kwargs): return JsonLinesItemExporter(self.output, **kwargs) @@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest): self.assertEqual(exported, ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name=u'Joseph', age='22') - i2 = dict(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph', age='22') + i2 = dict(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() @@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest): self.assertTwoItemsExported(ItemAdapter(self.i).asdict()) def test_nested_item(self): - i1 = self.item_class(name=u'Joseph\xa3', age='22') - i2 = self.item_class(name=u'Maria', age=i1) - i3 = self.item_class(name=u'Jesus', age=i2) + i1 = self.item_class(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = self.item_class(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}} self.assertEqual(exported, [expected]) def test_nested_dict_item(self): - i1 = dict(name=u'Joseph\xa3', age='22') - i2 = self.item_class(name=u'Maria', age=i1) - i3 = dict(name=u'Jesus', age=i2) + i1 = dict(name='Joseph\xa3', age='22') + i2 = self.item_class(name='Maria', age=i1) + i3 = dict(name='Jesus', age=i2) self.ie.start_exporting() self.ie.export_item(i3) self.ie.finish_exporting() exported = json.loads(to_unicode(self.output.getvalue())) - expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}} + expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}} self.assertEqual(exported, [expected]) def test_nonstring_types_item(self): @@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase): else: return super(CustomItemExporter, self).serialize_field(field, name, value) - i = self.item_class(name=u'John', age='22') + i = self.item_class(name='John', age='22') a = ItemAdapter(i) ie = CustomItemExporter() self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John') self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23') - i2 = {'name': u'John', 'age': '22'} + i2 = {'name': 'John', 'age': '22'} self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John') self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c49b2e92f..b57349848 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_encoding(self): - items = [dict({'foo': u'Test\xd6'})] + items = [dict({'foo': 'Test\xd6'})] formats = { 'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'), @@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'})] formats = { 'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_configs(self): - items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})] + items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})] formats = { 'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'), @@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_batch_item_count_feeds_setting(self): - items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})] + items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})] formats = { 'json': ['[{"foo": "FOO"}]'.encode('utf-8'), '[{"foo": "FOO1"}]'.encode('utf-8')], diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index cf3fc8496..64ff7a73d 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase): assert h.getlist('X-Forwarded-For') is not hlist def test_encode_utf8(self): - h = Headers({u'key': u'\xa3'}, encoding='utf-8') + h = Headers({'key': '\xa3'}, encoding='utf-8') key, val = dict(h).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] self.assertEqual(val[0], b'\xc2\xa3') def test_encode_latin1(self): - h = Headers({u'key': u'\xa3'}, encoding='latin1') + h = Headers({'key': '\xa3'}, encoding='latin1') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xa3') def test_encode_multiple(self): - h = Headers({u'key': [u'\xa3']}, encoding='utf-8') + h = Headers({'key': ['\xa3']}, encoding='utf-8') key, val = dict(h).popitem() self.assertEqual(val[0], b'\xc2\xa3') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 63014b22d..f5cf4e798 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase): self.assertFalse(p.headers is r.headers) # headers must not be unicode - h = Headers({'key1': u'val1', u'key2': 'val2'}) - h[u'newkey'] = u'newval' + h = Headers({'key1': 'val1', 'key2': 'val2'}) + h['newkey'] = 'newval' for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: @@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase): self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") def test_url_encoding(self): - r = self.request_class(url=u"http://www.scrapy.org/price/£") + r = self.request_class(url="http://www.scrapy.org/price/£") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_other(self): # encoding affects only query part of URI, not path # path part should always be UTF-8 encoded before percent-escaping - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") - r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1") + r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1") self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") def test_url_encoding_query(self): - r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ") + r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ") self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") # should be same as above - r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") + r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8") self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5") def test_url_encoding_query_latin1(self): # encoding is used for encoding query-string before percent-escaping; # path is still UTF-8 encoded before percent-escaping - r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1") + r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1") self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3") def test_url_encoding_nonutf8_untouched(self): @@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase): # characters. Otherwise, in the future the IRI will be mapped to # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different # URI from "http://www.example.org/r%E9sum%E9.html". - r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3") + r1 = self.request_class(url="http://www.scrapy.org/price/%a3") self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3") - r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") + r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3") + r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3") self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3") - r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html") + r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html") def test_body(self): @@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase): assert isinstance(r2.body, bytes) self.assertEqual(r2.encoding, 'utf-8') # default encoding - r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') + r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) self.assertEqual(r3.body, b"Price: \xc2\xa3100") - r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1') + r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1') assert isinstance(r4.body, bytes) self.assertEqual(r4.body, b"Price: \xa3100") @@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase): r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") # unicode url - r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value") + r = self.request_class(url="http://www.example.com/ajax.html#!key=value") self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue") def test_copy(self): @@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase): assert r4.dont_filter is False def test_method_always_str(self): - r = self.request_class("http://www.example.com", method=u"POST") + r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) def test_immutable_attributes(self): @@ -381,7 +381,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_textual_data(self): # using default encoding (utf-8) - data = {u'µ one': u'two', u'price': u'£ 100'} + data = {'µ one': 'two', 'price': '£ 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -390,7 +390,7 @@ class FormRequestTest(RequestTest): def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'} + data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'} r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') @@ -406,14 +406,14 @@ class FormRequestTest(RequestTest): self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded') def test_custom_encoding_textual_data(self): - data = {'price': u'£ 100'} + data = {'price': '£ 100'} r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1') self.assertEqual(r3.encoding, 'latin1') self.assertEqual(r3.body, b'price=%A3+100') def test_multi_key_values(self): # using multiples values for a single key - data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} + data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') @@ -450,10 +450,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_bytes_latin1(self): response = _buildresponse( @@ -471,14 +471,14 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_post_nonascii_unicode(self): response = _buildresponse( - u""" + """ @@ -490,10 +490,10 @@ class FormRequestTest(RequestTest): self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.url, "http://www.example.com/this/post.php") fs = _qs(req, to_unicode=True) - self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'}) - self.assertEqual(set(fs[u'one']), {u'two', u'three'}) - self.assertEqual(fs[u'test2'], [u'xxx µ']) - self.assertEqual(fs[u'six'], [u'seven']) + self.assertEqual(set(fs['test £']), {'val1', 'val2'}) + self.assertEqual(set(fs['one']), {'two', 'three'}) + self.assertEqual(fs['test2'], ['xxx µ']) + self.assertEqual(fs['six'], ['seven']) def test_from_response_duplicate_form_key(self): response = _buildresponse( @@ -685,7 +685,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, clickdata={u'name': u'clickable', u'value': u'clicked2'} + response, clickdata={'name': 'clickable', 'value': 'clicked2'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -694,21 +694,21 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_clickdata(self): response = _buildresponse( - u"""
    + """
    """) req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a3'} + response, clickdata={'name': 'price in \u00a3'} ) fs = _qs(req, to_unicode=True) - self.assertTrue(fs[u'price in \u00a3']) + self.assertTrue(fs['price in \u00a3']) def test_from_response_unicode_clickdata_latin1(self): response = _buildresponse( - u"""
    + """ @@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
    """, encoding='latin1') req = self.request_class.from_response( - response, clickdata={u'name': u'price in \u00a5'} + response, clickdata={'name': 'price in \u00a5'} ) fs = _qs(req, to_unicode=True, encoding='latin1') - self.assertTrue(fs[u'price in \u00a5']) + self.assertTrue(fs['price in \u00a5']) def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( @@ -733,7 +733,7 @@ class FormRequestTest(RequestTest): """) req = self.request_class.from_response( - response, formname='form2', clickdata={u'name': u'clickable'} + response, formname='form2', clickdata={'name': 'clickable'} ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) @@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest): def test_from_response_unicode_xpath(self): response = _buildresponse(b'
    ') - r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']") + r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']") fs = _qs(r) self.assertEqual(fs, {}) - xpath = u"//form[@name='\u03b1']" + xpath = "//form[@name='\u03b1']" self.assertRaisesRegex(ValueError, re.escape(xpath), self.request_class.from_response, response, formxpath=xpath) @@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest): self._test_request(params=('value',)) self._test_request(params=('username', 'password'), methodname='login') self._test_request(params=('response', ), methodresponse='login') - self._test_request(params=(u'pas£',), encoding='utf-8') + self._test_request(params=('pas£',), encoding='utf-8') self._test_request(params=(None,), allow_none=1) self.assertRaises(TypeError, self._test_request) self.assertRaises(TypeError, self._test_request, params=(None,)) def test_latin1(self): - self._test_request(params=(u'pas£',), encoding='latin1') + self._test_request(params=('pas£',), encoding='latin1') class JsonRequestTest(RequestTest): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e0ca3c0e6..56d017de6 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest): def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) - resp = self.response_class(u"http://www.example.com/") + resp = self.response_class("http://www.example.com/") self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str - resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/", encoding='utf-8') assert isinstance(resp.url, str) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8') self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') + resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) - resp = self.response_class(u"http://www.example.com/price/\xa3", + resp = self.response_class("http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') def test_unicode_body(self): unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 ' '\u0442\u0435\u043a\u0441\u0442') - self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body') + self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body') original_string = unicode_string.encode('cp1251') r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') @@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest): def test_encoding(self): r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") + r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3") r3 = self.response_class("http://www.example.com", body=b"\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3") @@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r5._headers_encoding(), None) self._assert_response_encoding(r5, "utf-8") assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' - self._assert_response_values(r1, 'utf-8', u"\xa3") - self._assert_response_values(r2, 'utf-8', u"\xa3") - self._assert_response_values(r3, 'iso-8859-1', u"\xa3") - self._assert_response_values(r6, 'gb18030', u"\u2015") - self._assert_response_values(r7, 'gb18030', u"\u2015") + self._assert_response_values(r1, 'utf-8', "\xa3") + self._assert_response_values(r2, 'utf-8', "\xa3") + self._assert_response_values(r3, 'iso-8859-1', "\xa3") + self._assert_response_values(r6, 'gb18030', "\u2015") + self._assert_response_values(r7, 'gb18030', "\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies - self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3") def test_declared_encoding_invalid(self): """Check that unknown declared encodings are ignored""" @@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=UKNOWN"]}, body=b"\xc2\xa3") self.assertEqual(r._declared_encoding(), None) - self._assert_response_values(r, 'utf-8', u"\xa3") + self._assert_response_values(r, 'utf-8', "\xa3") def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" r = self.response_class("http://www.example.com", body=b'\xff\xfeh\x00i\x00', encoding='utf-16') - self._assert_response_values(r, 'utf-16', u"hi") + self._assert_response_values(r, 'utf-16', "hi") def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): r6 = self.response_class("http://www.example.com", @@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest): body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') self.assertIn(r6.text, { - u'WORD\ufffd\ufffd', # w3lib < 1.19.0 - u'WORD\ufffd', # w3lib >= 1.19.0 + 'WORD\ufffd\ufffd', # w3lib < 1.19.0 + 'WORD\ufffd', # w3lib >= 1.19.0 }) def test_bom_is_removed_from_body(self): @@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest): # Test response without content-type and BOM encoding response = self.response_class(url, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') # Body caching sideeffect isn't triggered when encoding is declared in @@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest): # body response = self.response_class(url, headers=headers, body=body) self.assertEqual(response.encoding, 'utf-8') - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') response = self.response_class(url, headers=headers, body=body) - self.assertEqual(response.text, u'WORD') + self.assertEqual(response.text, 'WORD') self.assertEqual(response.encoding, 'utf-8') def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX') # XXX: Policy for replacing invalid chars may suffer minor variations - # but it should always contain the unicode replacement char (u'\ufffd') - assert u'\ufffd' in r.text, repr(r.text) - assert u'PREFIX' in r.text, repr(r.text) - assert u'SUFFIX' in r.text, repr(r.text) + # but it should always contain the unicode replacement char ('\ufffd') + assert '\ufffd' in r.text, repr(r.text) + assert 'PREFIX' in r.text, repr(r.text) + assert 'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs r = self.response_class("http://example.com", encoding='utf-8', body=b'\xf0value') - assert u'value' in r.text, repr(r.text) + assert 'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX') - # assert u'\ufffd' in r.text, repr(r.text) + # assert '\ufffd' in r.text, repr(r.text) def test_selector(self): body = b"Some page" @@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest): self.assertEqual( response.selector.xpath("//title/text()").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.css("title::text").getall(), - [u'Some page'] + ['Some page'] ) self.assertEqual( response.selector.re("Some (.*)"), - [u'page'] + ['page'] ) def test_selector_shortcuts(self): @@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body=u'click me'.encode('utf8') + body='click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body=u'click me'.encode('cp1251') + body='click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], @@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest): def test_body_as_unicode_deprecation_warning(self): with catch_warnings(record=True) as warnings: - r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8') - self.assertEqual(r1.body_as_unicode(), u'Hello') + r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8') + self.assertEqual(r1.body_as_unicode(), 'Hello') self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) @@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest): self.assertEqual( response.selector.xpath("//elem/text()").getall(), - [u'value'] + ['value'] ) def test_selector_shortcuts(self): diff --git a/tests/test_item.py b/tests/test_item.py index 60468971c..0ce78f8c0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'name' - self.assertEqual(i['name'], u'name') + i['name'] = 'name' + self.assertEqual(i['name'], 'name') def test_init(self): class TestItem(Item): @@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.__getitem__, 'name') - i2 = TestItem(name=u'john doe') - self.assertEqual(i2['name'], u'john doe') + i2 = TestItem(name='john doe') + self.assertEqual(i2['name'], 'john doe') - i3 = TestItem({'name': u'john doe'}) - self.assertEqual(i3['name'], u'john doe') + i3 = TestItem({'name': 'john doe'}) + self.assertEqual(i3['name'], 'john doe') i4 = TestItem(i3) - self.assertEqual(i4['name'], u'john doe') + self.assertEqual(i4['name'], 'john doe') - self.assertRaises(KeyError, TestItem, {'name': u'john doe', - 'other': u'foo'}) + self.assertRaises(KeyError, TestItem, {'name': 'john doe', + 'other': 'foo'}) def test_invalid_field(self): class TestItem(Item): @@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase): number = Field() i = TestItem() - i['name'] = u'John Doe' + i['name'] = 'John Doe' i['number'] = 123 itemrepr = repr(i) @@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase): i = TestItem() self.assertRaises(KeyError, i.get_name) - i['name'] = u'lala' - self.assertEqual(i.get_name(), u'lala') - i.change_name(u'other') + i['name'] = 'lala' + self.assertEqual(i.get_name(), 'lala') + i.change_name('other') self.assertEqual(i.get_name(), 'other') def test_metaclass(self): @@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase): values = Field() i = TestItem() - i['name'] = u'John' + i['name'] = 'John' self.assertEqual(list(i.keys()), ['name']) self.assertEqual(list(i.values()), ['John']) - i['keys'] = u'Keys' - i['values'] = u'Values' + i['keys'] = 'Keys' + i['values'] = 'Values' self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name']) - self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John']) + self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John']) def test_metaclass_with_fields_attribute(self): class TestItem(Item): fields = {'new': Field(default='X')} - item = TestItem(new=u'New') + item = TestItem(new='New') self.assertSortedEqual(list(item.keys()), ['new']) - self.assertSortedEqual(list(item.values()), [u'New']) + self.assertSortedEqual(list(item.values()), ['New']) def test_metaclass_inheritance(self): class ParentItem(Item): @@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase): name = Field() i = TestItem() - i['name'] = u'John' - self.assertEqual(dict(i), {'name': u'John'}) + i['name'] = 'John' + self.assertEqual(dict(i), {'name': 'John'}) def test_copy(self): class TestItem(Item): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 8d4538eed..a0bafa5e5 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -31,31 +31,31 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) def test_extract_filter_allow(self): lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=('sample', ), unique=False) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) @@ -63,10 +63,10 @@ class Base: lx = self.extractor_cls(allow=('sample', ), unique=False, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') ]) @@ -74,22 +74,22 @@ class Base: lx = self.extractor_cls(allow=('sample',), unique=True, canonicalize=True) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=('google.com', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_extraction_using_single_values(self): @@ -97,27 +97,27 @@ class Base: lx = self.extractor_cls(allow='sample') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(allow_domains='google.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) lx = self.extractor_cls(deny_domains='example.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), + Link(url='http://www.google.com/something', text=''), ]) def test_nofollow(self): @@ -145,11 +145,11 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/about.html', text=u'About us'), - Link(url='http://example.org/follow.html', text=u'Follow this link'), - Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), - Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://example.org/about.html', text='About us'), + Link(url='http://example.org/follow.html', text='Follow this link'), + Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True), + Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ]) def test_matches(self): @@ -183,8 +183,8 @@ class Base: def test_restrict_xpaths(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) def test_restrict_xpaths_encoding(self): @@ -202,14 +202,14 @@ class Base: lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + [Link(url='http://example.org/about.html', text='About us\xa3')]) def test_restrict_xpaths_with_html_entities(self): html = b'

    text

    ' response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')]) + [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')]) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" @@ -217,22 +217,22 @@ class Base: response = HtmlResponse("http://example.org", body=body, encoding='gb18030') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c', fragment='', nofollow=False)]) def test_restrict_css(self): lx = self.extractor_cls(restrict_css=('#subwrapper a',)) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2') + Link(url='http://example.com/sample2.html', text='sample 2') ]) def test_restrict_css_and_restrict_xpaths_together(self): lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), restrict_css=('#subwrapper + a', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), ]) def test_area_tag_with_unicode_present(self): @@ -243,7 +243,7 @@ class Base: lx.extract_links(response) lx.extract_links(response) self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'', + [Link(url='http://example.org/foo', text='', fragment='', nofollow=False)]) def test_encoded_url(self): @@ -251,7 +251,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_encoded_url_in_restricted_xpath(self): @@ -259,7 +259,7 @@ class Base: response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False), ]) def test_ignored_extensions(self): @@ -268,7 +268,7 @@ class Base: response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), + Link(url='http://example.org/page.html', text='asd'), ]) # override denied extensions @@ -308,25 +308,25 @@ class Base: page4_url = 'http://example.com/page%204.html' self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()) self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url=page4_url, text=u'href with whitespaces'), + Link(url='http://www.google.com/something', text=''), + Link(url='http://example.com/innertag.html', text='inner tag'), + Link(url=page4_url, text='href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) @@ -344,24 +344,24 @@ class Base: lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample1.html', text=''), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags="area") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample1.html', text=''), ]) lx = self.extractor_cls(tags="a") self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample2.html', text='sample 2'), ]) lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=()) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), + Link(url='http://example.com/sample2.html', text='sample 2'), + Link(url='http://example.com/sample2.jpg', text=''), ]) def test_tags_attrs(self): @@ -375,14 +375,14 @@ class Base: lx = self.extractor_cls(tags='div', attrs='data-url') self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False) ]) def test_xhtml(self): @@ -420,13 +420,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -436,13 +436,13 @@ class Base: self.assertEqual( lx.extract_links(response), [ - Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', + Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text='Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', + Link(url='http://example.com/nofollow2.html', text='Choose to follow or not', fragment='', nofollow=False), - Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True), + Link(url='http://google.com/something', text='External link not to follow', nofollow=True), ] ) @@ -455,8 +455,8 @@ class Base: response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_ftp_links(self): @@ -467,7 +467,7 @@ class Base: response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ - Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False), ]) def test_pickle_extractor(self): @@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): response = HtmlResponse("http://example.org/index.html", body=html) lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + Link(url='http://example.org/item1.html', text='Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text='Item 3', nofollow=False), ]) def test_link_restrict_text(self): @@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): # Simple text inclusion test lx = self.extractor_cls(restrict_text='dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Unique regex test lx = self.extractor_cls(restrict_text=r'of.*dog') self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) # Multiple regex test lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat']) self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False), - Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), + Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False), + Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False), ]) def test_restrict_xpaths_with_html_entities(self): diff --git a/tests/test_loader.py b/tests/test_loader.py index 581183625..2ed6f365f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value_on_unknown_field(self): il = TestItemLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) + self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo']) def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) class InitializationTestMixin: @@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, l.get_css, '#name::text') def test_init_method_with_selector(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_selector_css(self): - sel = Selector(text=u"
    marta
    ") + sel = Selector(text="
    marta
    ") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) def test_init_method_with_response_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) # combining/accumulating CSS selectors and XPath expressions l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta']) l.add_xpath('url', '//img/@src') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png']) def test_add_xpath_re(self): l = TestItemLoader(response=self.response) l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_replace_xpath(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//p/text()') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_xpath('name', ['//p/text()', '//div/text()']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) def test_get_xpath(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph']) - self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph') - self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_xpath('//p/text()'), ['paragraph']) + self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph') + self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta']) + self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta']) def test_replace_xpath_multi_fields(self): l = TestItemLoader(response=self.response) l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) def test_add_css_re(self): l = TestItemLoader(response=self.response) l.add_css('name', 'div::text', re='ma') - self.assertEqual(l.get_output_value('name'), [u'Ma']) + self.assertEqual(l.get_output_value('name'), ['Ma']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) def test_replace_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('name', 'div::text') - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css('name', 'p::text') - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.replace_css('name', ['p::text', 'div::text']) - self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta']) + self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta']) l.add_css('url', 'a::attr(href)', re='http://(.+)') - self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['www.scrapy.org']) l.replace_css('url', 'img::attr(src)') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_get_css(self): l = TestItemLoader(response=self.response) - self.assertEqual(l.get_css('p::text'), [u'paragraph']) - self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph') - self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa') + self.assertEqual(l.get_css('p::text'), ['paragraph']) + self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph') + self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa') - self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta']) + self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta']) self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']), - [u'http://www.scrapy.org', u'/images/logo.png']) + ['http://www.scrapy.org', '/images/logo.png']) def test_replace_css_multi_fields(self): l = TestItemLoader(response=self.response) l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Marta']) + self.assertEqual(l.get_output_value('name'), ['Marta']) l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x}) - self.assertEqual(l.get_output_value('name'), [u'Paragraph']) + self.assertEqual(l.get_output_value('name'), ['Paragraph']) l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x}) - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) def test_replace_css_re(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') - self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['scrapy.org']) class SubselectorLoaderTest(unittest.TestCase): @@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase): nl.add_css('name_div', '#id') nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) - self.assertEqual(l.get_output_value('name'), [u'marta']) - self.assertEqual(l.get_output_value('name_div'), [u'
    marta
    ']) - self.assertEqual(l.get_output_value('name_value'), [u'marta']) + self.assertEqual(l.get_output_value('name'), ['marta']) + self.assertEqual(l.get_output_value('name_div'), ['
    marta
    ']) + self.assertEqual(l.get_output_value('name_value'), ['marta']) self.assertEqual(l.get_output_value('name'), nl.get_output_value('name')) self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div')) @@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase): nl2 = nl1.nested_xpath('a') l.add_xpath('url', '//footer/a/@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) nl1.replace_xpath('url', 'img/@src') - self.assertEqual(l.get_output_value('url'), [u'/images/logo.png']) + self.assertEqual(l.get_output_value('url'), ['/images/logo.png']) nl2.replace_xpath('url', '@href') - self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) + self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org']) def test_nested_ordering(self): l = NestedItemLoader(response=self.response) @@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase): l.add_xpath('url', '//footer/a/@href') self.assertEqual(l.get_output_value('url'), [ - u'/images/logo.png', - u'http://www.scrapy.org', - u'homepage', - u'http://www.scrapy.org', + '/images/logo.png', + 'http://www.scrapy.org', + 'homepage', + 'http://www.scrapy.org', ]) def test_nested_load_item(self): @@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase): assert item is nl1.item assert item is nl2.item - self.assertEqual(item['name'], [u'marta']) - self.assertEqual(item['url'], [u'http://www.scrapy.org']) - self.assertEqual(item['image'], [u'/images/logo.png']) + self.assertEqual(item['name'], ['marta']) + self.assertEqual(item['url'], ['http://www.scrapy.org']) + self.assertEqual(item['image'], ['/images/logo.png']) # Functions as processors diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d0a59e8cd..eb14de14f 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_load_item_using_default_loader(self): i = TestItem() - i['summary'] = u'lala' + i['summary'] = 'lala' il = ItemLoader(item=i) - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() assert item is i - self.assertEqual(item['summary'], [u'lala']) - self.assertEqual(item['name'], [u'marta']) + self.assertEqual(item['summary'], ['lala']) + self.assertEqual(item['name'], ['marta']) def test_load_item_using_custom_loader(self): il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') item = il.load_item() - self.assertEqual(item['name'], [u'Marta']) + self.assertEqual(item['name'], ['Marta']) def test_load_item_ignore_none_field_values(self): def validate_sku(value): @@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase): price_out = Compose(TakeFirst(), float) sku_out = Compose(TakeFirst(), validate_sku) - valid_fragment = u'SKU: 1234' - invalid_fragment = u'SKU: not available' + valid_fragment = 'SKU: 1234' + invalid_fragment = 'SKU: not available' sku_re = 'SKU: (.+)' il = MyLoader(item={}) # Should not return "sku: None". il.add_value('sku', [invalid_fragment], re=sku_re) # Should not ignore empty values. - il.add_value('name', u'') - il.add_value('price', [u'0']) + il.add_value('name', '') + il.add_value('price', ['0']) self.assertEqual(il.load_item(), { - 'name': u'', + 'name': '', 'price': 0.0, }) il.replace_value('sku', [valid_fragment], re=sku_re) - self.assertEqual(il.load_item()['sku'], u'1234') + self.assertEqual(il.load_item()['sku'], '1234') def test_self_referencing_loader(self): class MyLoader(ItemLoader): @@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase): def test_add_value(self): il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.add_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.add_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe']) + self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe']) # test add object value il.add_value('summary', {'key': 1}) self.assertEqual(il.get_collected_values('summary'), [{'key': 1}]) - il.add_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim']) + il.add_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim']) def test_add_zero(self): il = NameItemLoader() @@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase): def test_replace_value(self): il = TestItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'Marta']) - self.assertEqual(il.get_output_value('name'), [u'Marta']) - il.replace_value('name', u'pepe') - self.assertEqual(il.get_collected_values('name'), [u'Pepe']) - self.assertEqual(il.get_output_value('name'), [u'Pepe']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['Marta']) + self.assertEqual(il.get_output_value('name'), ['Marta']) + il.replace_value('name', 'pepe') + self.assertEqual(il.get_collected_values('name'), ['Pepe']) + self.assertEqual(il.get_output_value('name'), ['Pepe']) - il.replace_value(None, u'Jim', lambda x: {'name': x}) - self.assertEqual(il.get_collected_values('name'), [u'Jim']) + il.replace_value(None, 'Jim', lambda x: {'name': x}) + self.assertEqual(il.get_collected_values('name'), ['Jim']) def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) - self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) - self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) + self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper)) + self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$')) + self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')) - il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$') - self.assertEqual([u'foo'], il.get_collected_values('name')) - il.replace_value('name', u'name:bar', re=u'name:(.*)$') - self.assertEqual([u'bar'], il.get_collected_values('name')) + il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$') + self.assertEqual(['foo'], il.get_collected_values('name')) + il.replace_value('name', 'name:bar', re='name:(.*)$') + self.assertEqual(['bar'], il.get_collected_values('name')) def test_iter_on_input_processor_input(self): class NameFirstItemLoader(NameItemLoader): name_in = TakeFirst() il = NameFirstItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.add_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', 'marta') + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.replace_value('name', [u'marta', u'jose']) - self.assertEqual(il.get_collected_values('name'), [u'marta']) + il.replace_value('name', ['marta', 'jose']) + self.assertEqual(il.get_collected_values('name'), ['marta']) il = NameFirstItemLoader() - il.add_value('name', u'marta') - il.add_value('name', [u'jose', u'pedro']) - self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose']) + il.add_value('name', 'marta') + il.add_value('name', ['jose', 'pedro']) + self.assertEqual(il.get_collected_values('name'), ['marta', 'jose']) def test_map_compose_filter(self): def filter_world(x): @@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase): name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Mart']) item = il.load_item() - self.assertEqual(item['name'], [u'Mart']) + self.assertEqual(item['name'], ['Mart']) def test_default_input_processor(self): il = DefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_inherited_default_input_processor(self): class InheritDefaultedItemLoader(DefaultedItemLoader): pass il = InheritDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mart']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mart']) def test_input_processor_inheritance(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(lambda v: v.lower()) il = ChildItemLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'HTTP://scrapy.ORG') + self.assertEqual(il.get_output_value('url'), ['http://scrapy.org']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) class ChildChildItemLoader(ChildItemLoader): url_in = MapCompose(lambda v: v.upper()) summary_in = MapCompose(lambda v: v) il = ChildChildItemLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'Marta']) + il.add_value('url', 'http://scrapy.org') + self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['Marta']) def test_empty_map_compose(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_identity_input_processor(self): class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = Identity() il = IdentityDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'marta']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['marta']) def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'mARTA']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['mARTA']) def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_output_value('name'), [u'MART']) + il.add_value('name', 'marta') + self.assertEqual(il.get_output_value('name'), ['MART']) def test_output_processor_using_function(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): - name_out = u" ".join + name_out = " ".join il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') def test_output_processor_error(self): class TestItemLoader(ItemLoader): @@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = MapCompose(float) il = TestItemLoader() - il.add_value('name', [u'$10']) + il.add_value('name', ['$10']) try: - float(u'$10') + float('$10') except Exception as e: expected_exc_str = str(e) @@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase): def test_output_processor_using_classes(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class TakeFirstItemLoader(TestItemLoader): name_out = Join() il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar Ta') class TakeFirstItemLoader(TestItemLoader): name_out = Join("
    ") il = TakeFirstItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), u'Mar
    Ta') + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), 'Mar
    Ta') def test_default_output_processor(self): il = TestItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) class LalaItemLoader(TestItemLoader): default_output_processor = Identity() il = LalaItemLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta']) + il.add_value('name', ['mar', 'ta']) + self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta']) def test_loader_context_on_declaration(self): class ChildItemLoader(TestItemLoader): - url_in = MapCompose(processor_with_args, key=u'val') + url_in = MapCompose(processor_with_args, key='val') il = ChildItemLoader() - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_instantiation(self): class ChildItemLoader(TestItemLoader): url_in = MapCompose(processor_with_args) - il = ChildItemLoader(key=u'val') - il.add_value('url', u'text') + il = ChildItemLoader(key='val') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_loader_context_on_assign(self): @@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase): url_in = MapCompose(processor_with_args) il = ChildItemLoader() - il.context['key'] = u'val' - il.add_value('url', u'text') + il.context['key'] = 'val' + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['val']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['val']) def test_item_passed_to_input_processor_functions(self): @@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase): it = TestItem(name='marta') il = ChildItemLoader(item=it) - il.add_value('url', u'text') + il.add_value('url', 'text') self.assertEqual(il.get_output_value('url'), ['marta']) - il.replace_value('url', u'text2') + il.replace_value('url', 'text2') self.assertEqual(il.get_output_value('url'), ['marta']) def test_compose_processor(self): @@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1]) il = TestItemLoader() - il.add_value('name', [u'marta', u'other']) - self.assertEqual(il.get_output_value('name'), u'Mart') + il.add_value('name', ['marta', 'other']) + self.assertEqual(il.get_output_value('name'), 'Mart') item = il.load_item() - self.assertEqual(item['name'], u'Mart') + self.assertEqual(item['name'], 'Mart') def test_partial_processor(self): def join(values, sep=None, loader_context=None, ignored=None): @@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase): summary_out = Compose(partial(join, ignored='foo')) il = TestItemLoader() - il.add_value('name', [u'rabbit', u'hole']) - il.add_value('url', [u'rabbit', u'hole']) - il.add_value('summary', [u'rabbit', u'hole']) + il.add_value('name', ['rabbit', 'hole']) + il.add_value('url', ['rabbit', 'hole']) + il.add_value('summary', ['rabbit', 'hole']) item = il.load_item() - self.assertEqual(item['name'], u'rabbit+hole') - self.assertEqual(item['url'], u'rabbit.hole') - self.assertEqual(item['summary'], u'rabbithole') + self.assertEqual(item['name'], 'rabbit+hole') + self.assertEqual(item['url'], 'rabbit.hole') + self.assertEqual(item['summary'], 'rabbithole') def test_error_input_processor(self): class TestItem(Item): @@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other']) + ['marta', 'other']) def test_error_output_processor(self): class TestItem(Item): @@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase): name_out = Compose(Join(), float) il = TestItemLoader() - il.add_value('name', u'marta') + il.add_value('name', 'marta') with self.assertRaises(ValueError): il.load_item() @@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() self.assertRaises(ValueError, il.add_value, 'name', - [u'marta', u'other'], Compose(float)) + ['marta', 'other'], Compose(float)) class InitializationFromDictTest(unittest.TestCase): @@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase): def test_join(self): proc = Join() self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) - self.assertEqual(proc(['', 'hello', 'world']), u' hello world') - self.assertEqual(proc(['hello', 'world']), u'hello world') + self.assertEqual(proc(['', 'hello', 'world']), ' hello world') + self.assertEqual(proc(['hello', 'world']), 'hello world') self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): @@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase): def filter_world(x): return None if x == 'world' else x proc = MapCompose(filter_world, str.upper) - self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), - [u'HELLO', u'THIS', u'IS', u'SCRAPY']) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) proc = MapCompose(filter_world, str.upper) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 7064337ad..b771e7d79 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase): def test_dropped(self): item = {} - exception = Exception(u"\u2018") + exception = Exception("\u2018") response = Response("http://www.example.com") logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Dropped: \u2018", '{}']) + self.assertEqual(lines, ["Dropped: \u2018", '{}']) def test_item_error(self): # In practice, the complete traceback is shown by passing the @@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, u"Error processing {'key': 'value'}") + self.assertEqual(logline, "Error processing {'key': 'value'}") def test_spider_error(self): # In practice, the complete traceback is shown by passing the @@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase): def test_scraped(self): item = CustomItem() - item['name'] = u'\xa3' + item['name'] = '\xa3' response = Response("http://www.example.com") logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() assert all(isinstance(x, str) for x in lines) - self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3']) class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) CRAWLEDMSG = ( - u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" + "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) log_args = kwargs['args'] log_args['flags'] = str(request.flags) diff --git a/tests/test_mail.py b/tests/test_mail.py index 53dbc0686..9b248fbfa 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase): self.catched_msg = dict(**kwargs) def test_send_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject=subject, body=body, charset='utf-8', _callback=self._catch_mail_sent) @@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"') def test_send_attach_utf8(self): - subject = u'sübjèçt' - body = u'bödÿ-àéïöñß' + subject = 'sübjèçt' + body = 'bödÿ-àéïöñß' attach = BytesIO() attach.write(body.encode('utf-8')) attach.seek(0) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index dd19a69d5..a175f88ca 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ (b'attachment; filename="data.xml"', XmlResponse), (b'attachment; filename=data.xml', XmlResponse), - (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response), - (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), - (u'attachment;filename=data高.doc'.encode('gbk'), Response), - (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), - (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), + ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response), + ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response), + ('attachment;filename=data高.doc'.encode('gbk'), Response), + ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse), + ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse), ] for source, cls in mappings: diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 24aaaf7ec..9d8c201dd 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -93,7 +93,7 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) def test_unicode_url_and_useragent(self): - robotstxt_robotstxt_body = u""" + robotstxt_robotstxt_body = """ User-Agent: * Disallow: /admin/ Disallow: /static/ @@ -107,11 +107,11 @@ class BaseRobotParserTest: self.assertTrue(rp.allowed("https://site.local/", "*")) self.assertFalse(rp.allowed("https://site.local/admin/", "*")) self.assertFalse(rp.allowed("https://site.local/static/", "*")) - self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt")) + self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt")) self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*")) - self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*")) + self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*")) self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) - self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt")) + self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")) class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): diff --git a/tests/test_selector.py b/tests/test_selector.py index 00e663c11..62036ad8c 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase): ) self.assertEqual( [x.get() for x in sel.xpath("//input[@name='a']/@name")], - [u'a'] + ['a'] ) self.assertEqual( [x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], - [u'12.0'] + ['12.0'] ) self.assertEqual( sel.xpath("concat('xpath', 'rules')").getall(), - [u'xpathrules'] + ['xpathrules'] ) self.assertEqual( [x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], - [u'12'] + ['12'] ) def test_root_base_url(self): @@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') self.assertEqual(sel.xpath("//div").getall(), - [u'

    Hello

    ']) + ['

    Hello

    ']) def test_http_header_encoding_precedence(self): - # u'\xa3' = pound symbol in unicode - # u'\xc2\xa3' = pound symbol in utf-8 - # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + # '\xa3' = pound symbol in unicode + # '\xc2\xa3' = pound symbol in utf-8 + # '\xa3' = pound symbol in latin-1 (iso-8859-1) - meta = u'' - head = u'' + meta + u'' - body_content = u'\xa3' - body = u'' + body_content + u'' - html = u'' + head + body + u'' + meta = '' + head = '' + meta + '' + body_content = '\xa3' + body = '' + body_content + '' + html = '' + head + body + '' encoding = 'utf-8' html_utf8 = html.encode(encoding) headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence @@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, 'received both response and text'): - Selector(TextResponse(url='http://example.com', body=b''), text=u'') + Selector(TextResponse(url='http://example.com', body=b''), text='') diff --git a/tests/test_spider.py b/tests/test_spider.py index 83c10a3c3..78157a9b9 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest): output = list(spider._parse(response)) self.assertEqual(len(output), 2, iterator) self.assertEqual(output, [ - {'loc': [u'http://www.example.com/Special-Offers.html'], - 'updated': [u'2009-08-16'], - 'custom': [u'fuu'], - 'other': [u'bar']}, + {'loc': ['http://www.example.com/Special-Offers.html'], + 'updated': ['2009-08-16'], + 'custom': ['fuu'], + 'other': ['bar']}, {'loc': [], - 'updated': [u'2009-08-16'], - 'other': [u'foo'], + 'updated': ['2009-08-16'], + 'other': ['foo'], 'custom': []}, ], iterator) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index d17bb2cbc..298178f08 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = u""" + body = """ <þingflokkar> <þingflokkur id="26"> @@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase): XmlResponse(url="http://example.com", body=body, encoding='utf-8'), ): attrs = [] - for x in self.xmliter(r, u'þingflokkur'): + for x in self.xmliter(r, 'þingflokkur'): attrs.append((x.attrib['id'], - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), - x.xpath(u'./tímabil/fyrstaþing/text()').getall())) + x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath('./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [(u'26', [u'-'], [u'80']), - (u'21', [u'Ab'], [u'76']), - (u'27', [u'A'], [u'27'])]) + [('26', ['-'], ['80']), + ('21', ['Ab'], ['76']), + ('27', ['A'], ['27'])]) def test_xmliter_text(self): body = ( @@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase): ) self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], - [[u'one'], [u'two']]) + [['one'], ['two']]) def test_xmliter_namespaces(self): body = b""" @@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse('http://www.example.com', body=body) self.assertEqual( next(self.xmliter(response, 'item')).get(), - u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' + 'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase): result = [row for row in csv] self.assertEqual(result, - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: @@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') @@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase): csv1 = csviter(response1, quotechar="'") self.assertEqual([row for row in csv1], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) csv2 = csviter(response2, delimiter="|", quotechar="'") self.assertEqual([row for row in csv2], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): body = get_testdata('feeds', 'feed-sample6.csv') @@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"}, - {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"}, - {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"}, - {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) + [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"}, + {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"}, + {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"}, + {"'id'": "4", "'name'": "'empty'", "'value'": ""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') @@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, delimiter='\t') self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() @@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': 'foo\nbar'}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase): csv = csviter(response) self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, - {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, - {u'id': u'4', u'name': u'empty', u'value': u''}]) + [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, + {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, + {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): body = get_testdata('feeds', 'feed-sample3.csv') @@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + {'id': '1', 'name': 'latin1', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'}, ] ) @@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual( list(csv), [ - {u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + {'id': '1', 'name': 'cp852', 'value': 'test'}, + {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, ] ) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..3f93f509e 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase): class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e') def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e') def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): - self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') + self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e') def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_unicode, 423) @@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( to_unicode(b'a\xedb', 'utf-8', errors='replace'), - u'a\ufffdb' + 'a\ufffdb' ) class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49') + self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49') def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49') + self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49') def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') @@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase): def test_errors_argument(self): self.assertEqual( - to_bytes(u'a\ufffdb', 'latin-1', errors='replace'), + to_bytes('a\ufffdb', 'latin-1', errors='replace'), b'a?b' ) @@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase): assert binary_is_text(b"hello") def test_utf_16_strings_contain_null_bytes(self): - assert binary_is_text(u"hello".encode('utf-16')) + assert binary_is_text("hello".encode('utf-16')) def test_one_with_encoding(self): assert binary_is_text(b"
    Price \xa3
    ") diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 450e4bdca..de94ec960 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase): method="POST", body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, - cookies={'currency': u'руб'}, + cookies={'currency': 'руб'}, encoding='latin-1', priority=20, meta={'a': 'b'}, diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 5a52dd695..5ff2e41ef 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): def test_simple_render(self): context = dict(project_name='proj', name='spi', classname='TheSpider') - template = u'from ${project_name}.spiders.${name} import ${classname}' - rendered = u'from proj.spiders.spi import TheSpider' + template = 'from ${project_name}.spiders.${name} import ${classname}' + rendered = 'from proj.spiders.spi import TheSpider' template_path = os.path.join(self.tmp_path, 'templ.py.tmpl') render_path = os.path.join(self.tmp_path, 'templ.py') From 6f4ccec5675c00b0ec7877a0b1fcc234d5983490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Jul 2020 14:03:14 +0200 Subject: [PATCH 113/125] Cover our deprecation policy in the documentation --- docs/contributing.rst | 5 +++++ docs/versioning.rst | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 7b901dd00..525ad3497 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -108,6 +108,11 @@ Well-written patches should: tox -e docs-coverage +* if you are removing deprecated code, first make sure that at least 1 year + (12 months) has passed since the release that introduced the deprecation. + See :ref:`deprecation-policy`. + + .. _submitting-patches: Submitting patches diff --git a/docs/versioning.rst b/docs/versioning.rst index 227085f02..57643ea9a 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,7 +1,7 @@ .. _versioning: ============================ -Versioning and API Stability +Versioning and API stability ============================ Versioning @@ -34,7 +34,7 @@ For example: production) -API Stability +API stability ============= API stability was one of the major goals for the *1.0* release. @@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the same way. +.. _deprecation-policy: + +Deprecation policy +================== + +We aim to maintain support for deprecated Scrapy features for at least 1 year. + +For example, if a feature is deprecated in a Scrapy version released on +June 15th 2020, that feature should continue to work in versions released on +June 14th 2021 or before that. + +Any new Scrapy release after a year *may* remove support for that deprecated +feature. + +All deprecated features removed in a Scrapy release are explicitly mentioned in +the :ref:`release notes `. + + .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases From 3600582f56071dd9f40f31ed44e09294a78dc13f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:05:56 +0200 Subject: [PATCH 114/125] Cover Scrapy 2.2.1 and 2.3 in the release notes (#4708) --- docs/news.rst | 133 ++++++++++++++++++++++++++++++++ docs/topics/commands.rst | 2 + docs/topics/developer-tools.rst | 6 +- docs/topics/feed-exports.rst | 36 ++++++++- scrapy/utils/curl.py | 3 +- 5 files changed, 176 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 80d130e4a..850b323ef 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,139 @@ Release notes ============= +.. _release-2.3.0: + +Scrapy 2.3.0 (2020-08-04) +------------------------- + +Highlights: + +* :ref:`Feed exports ` now support :ref:`Google Cloud + Storage ` as a storage backend + +* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver + output items in batches of up to the specified number of items. + + It also serves as a workaround for :ref:`delayed file delivery + `, which causes Scrapy to only start item delivery + after the crawl has finished when using certain storage backends + (:ref:`S3 `, :ref:`FTP `, + and now :ref:`GCS `). + +* The base implementation of :ref:`item loaders ` has been + moved into a separate library, :doc:`itemloaders `, + allowing usage from outside Scrapy and a separate release schedule + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* Removed the following classes and their parent modules from + ``scrapy.linkextractors``: + + * ``htmlparser.HtmlParserLinkExtractor`` + * ``regex.RegexLinkExtractor`` + * ``sgml.BaseSgmlLinkExtractor`` + * ``sgml.SgmlLinkExtractor`` + + Use + :class:`LinkExtractor ` + instead (:issue:`4356`, :issue:`4679`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated + (:issue:`4683`) + + +New features +~~~~~~~~~~~~ + +* :ref:`Feed exports ` support :ref:`Google Cloud + Storage ` (:issue:`685`, :issue:`3608`) + +* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries + (:issue:`4250`, :issue:`4434`) + +* The :command:`parse` command now allows specifying an output file + (:issue:`4317`, :issue:`4377`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support + ``--data-raw`` (:issue:`4612`) + +* A ``parse`` callback may now be used in built-in spider subclasses, such + as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`, + :issue:`781`, :issue:`4254` ) + + +Bug fixes +~~~~~~~~~ + +* Fixed the :ref:`CSV exporting ` of + :ref:`dataclass items ` and :ref:`attr.s items + ` (:issue:`4667`, :issue:`4668`) + +* :meth:`Request.from_curl ` and + :func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request + method to ``POST`` when a request body is specified and no request method + is specified (:issue:`4612`) + +* The processing of ANSI escape sequences in enabled in Windows 10.0.14393 + and later, where it is required for colored output (:issue:`4393`, + :issue:`4403`) + + +Documentation +~~~~~~~~~~~~~ + +* Updated the `OpenSSL cipher list format`_ link in the documentation about + the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`) + +* Simplified the code example in :ref:`topics-loaders-dataclass` + (:issue:`4652`) + +.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* The base implementation of :ref:`item loaders ` has been + moved into :doc:`itemloaders ` (:issue:`4005`, + :issue:`4516`) + +* Fixed a silenced error in some scheduler tests (:issue:`4644`, + :issue:`4645`) + +* Renewed the localhost certificate used for SSL tests (:issue:`4650`) + +* Removed cookie-handling code specific to Python 2 (:issue:`4682`) + +* Stopped using Python 2 unicode literal syntax (:issue:`4704`) + +* Stopped using a backlash for line continuation (:issue:`4673`) + +* Removed unneeded entries from the MyPy exception list (:issue:`4690`) + +* Automated tests now pass on Windows as part of our continuous integration + system (:issue:`4458`) + +* Automated tests now pass on the latest PyPy version for supported Python + versions in our continuous integration system (:issue:`4504`) + + +.. _release-2.2.1: + +Scrapy 2.2.1 (2020-07-17) +------------------------- + +* The :command:`startproject` command no longer makes unintended changes to + the permissions of files in the destination folder, such as removing + execution permissions (:issue:`4662`, :issue:`4666`) + + .. _release-2.2.0: Scrapy 2.2.0 (2020-06-24) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 4fce51abc..9638a2322 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -493,6 +493,8 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file + .. versionadded:: 2.3 + .. skip: start Usage example:: diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 4e87a00f2..101aa159c 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -289,8 +289,10 @@ request:: "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") Alternatively, if you want to know the arguments needed to recreate that -request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` -function to get a dictionary with the equivalent arguments. +request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments: + +.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs Note that to translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index dd4eb3c61..37b7096f6 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -100,6 +100,7 @@ The storages backends supported out of the box are: * :ref:`topics-feed-storage-fs` * :ref:`topics-feed-storage-ftp` * :ref:`topics-feed-storage-s3` (requires botocore_) + * :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) * :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are @@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-s3: S3 @@ -194,11 +198,16 @@ You can also define a custom ACL for exported feeds using this setting: * :setting:`FEED_STORAGE_S3_ACL` +This storage backend uses :ref:`delayed file delivery `. + + .. _topics-feed-storage-gcs: Google Cloud Storage (GCS) -------------------------- +.. versionadded:: 2.3 + The feeds are stored on `Google Cloud Storage`_. * URI scheme: ``gs`` @@ -206,7 +215,7 @@ The feeds are stored on `Google Cloud Storage`_. * ``gs://mybucket/path/to/export.csv`` - * Required external libraries: `google-cloud-storage `_. + * Required external libraries: `google-cloud-storage`_. For more information about authentication, please refer to `Google Cloud documentation `_. @@ -215,6 +224,11 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following * :setting:`FEED_STORAGE_GCS_ACL` * :setting:`GCS_PROJECT_ID` +This storage backend uses :ref:`delayed file delivery `. + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + + .. _topics-feed-storage-stdout: Standard output @@ -227,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process. * Required external libraries: none +.. _delayed-file-delivery: + +Delayed file delivery +--------------------- + +As indicated above, some of the described storage backends use delayed file +delivery. + +These storage backends do not upload items to the feed URI as those items are +scraped. Instead, Scrapy writes items into a temporary local file, and only +once all the file contents have been written (i.e. at the end of the crawl) is +that file uploaded to the feed URI. + +If you want item delivery to start earlier when using one of these storage +backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items +in multiple files, with the specified maximum item count per file. That way, as +soon as a file reaches the maximum item count, that file is delivered to the +feed URI, allowing item delivery to start way before the end of the crawl. + + Settings ======== diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index aa681522f..9c0efcec4 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -39,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): :param str curl_command: string containing the curl command :param bool ignore_unknown_options: If true, only a warning is emitted when - cURL options are unknown. Otherwise raises an error. (default: True) + cURL options are unknown. Otherwise + raises an error. (default: True) :return: dictionary of Request kwargs """ From 1278e76d9093b1c5c9ec810768d1066772a8a134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:07:02 +0200 Subject: [PATCH 115/125] =?UTF-8?q?Bump=20version:=202.2.0=20=E2=86=92=202?= =?UTF-8?q?.3.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8d4d74bc5..3c1c8f891 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0 +current_version = 2.3.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index ccbccc3dc..276cbf9e2 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.2.0 +2.3.0 From 4ee538e44b2650c49054b1f7f4c87ac70350471a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Aug 2020 20:34:11 +0200 Subject: [PATCH 116/125] Update unicode references from Python 2 times in the documentation (#4703) --- docs/topics/exporters.rst | 8 ++------ docs/topics/loaders.rst | 4 ++-- docs/topics/request-response.rst | 30 ++++++++++++++++-------------- docs/topics/selectors.rst | 7 ++++--- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e5c99e5b1..8c84b85fc 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -166,8 +166,7 @@ BaseItemExporter By default, this method looks for a serializer :ref:`declared in the item field ` and returns the result of applying that serializer to the value. If no serializer is found, it returns the - value unchanged except for ``unicode`` values which are encoded to - ``str`` using the encoding declared in the :attr:`encoding` attribute. + value unchanged. :param field: the field being serialized. If the source :ref:`item object ` does not define field metadata, *field* is an empty @@ -217,10 +216,7 @@ BaseItemExporter .. attribute:: encoding - The encoding that will be used to encode unicode values. This only - affects unicode values (which are always serialized to str using this - encoding). Other value types are passed unchanged to the specific - serialization library. + The output character encoding. .. attribute:: indent diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 29d9c5805..c0f534493 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -193,10 +193,10 @@ Item Loaders are declared using a class definition syntax. Here is an example:: default_output_processor = TakeFirst() - name_in = MapCompose(unicode.title) + name_in = MapCompose(str.title) name_out = Join() - price_in = MapCompose(unicode.strip) + price_in = MapCompose(str.strip) # ... diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index fbd8e4b73..1dffd1d55 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,12 +51,12 @@ Request objects given, the dict passed in this parameter will be shallow copied. :type meta: dict - :param body: the request body. If a ``unicode`` is passed, then it's encoded to - ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If - ``body`` is not given, an empty string is stored. Regardless of the - type of this argument, the final value stored will be a ``str`` (never - ``unicode`` or ``None``). - :type body: str or unicode + :param body: the request body. If a string is passed, then it's encoded as + bytes using the ``encoding`` passed (which defaults to ``utf-8``). If + ``body`` is not given, an empty bytes object is stored. Regardless of the + type of this argument, the final value stored will be a bytes object + (never a string or ``None``). + :type body: bytes or str :param headers: the headers of this request. The dict values can be strings (for single valued headers) or lists (for multi-valued headers). If @@ -106,7 +106,7 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the - body to ``str`` (if given as ``unicode``). + body to bytes (if given as a string). :type encoding: string :param priority: the priority of this request (defaults to ``0``). @@ -721,7 +721,7 @@ Response objects .. attribute:: Response.body The body of this Response. Keep in mind that Response.body - is always a bytes object. If you want the unicode version use + is always a bytes object. If you want the string version use :attr:`TextResponse.text` (only available in :class:`TextResponse` and subclasses). @@ -842,9 +842,9 @@ TextResponse objects is the same as for the :class:`Response` class and is not documented here. :param encoding: is a string which contains the encoding to use for this - response. If you create a :class:`TextResponse` object with a unicode + response. If you create a :class:`TextResponse` object with a string as body, it will be encoded using this encoding (remember the body attribute - is always a string). If ``encoding`` is ``None`` (default value), the + is always a bytes object). If ``encoding`` is ``None`` (default value), the encoding will be looked up in the response headers and body instead. :type encoding: string @@ -853,7 +853,7 @@ TextResponse objects .. attribute:: TextResponse.text - Response body, as unicode. + Response body, as a string. The same as ``response.body.decode(response.encoding)``, but the result is cached after the first call, so you can access @@ -861,9 +861,11 @@ TextResponse objects .. note:: - ``unicode(response.body)`` is not a correct way to convert response - body to unicode: you would be using the system default encoding - (typically ``ascii``) instead of the response encoding. + ``str(response.body)`` is not a correct way to convert the response + body into a string: + + >>> str(b'body') + "b'body'" .. attribute:: TextResponse.encoding diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 5014df6ac..9e2c6ba42 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``: Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or -markup as an unicode string (in ``text`` argument). +markup as a string (in ``text`` argument). + Usually there is no need to construct Scrapy selectors manually: ``response`` object is available in Spider callbacks, so in most cases it is more convenient to use ``response.css()`` and ``response.xpath()`` @@ -383,7 +384,7 @@ Using selectors with regular expressions :class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting data using regular expressions. However, unlike using ``.xpath()`` or -``.css()`` methods, ``.re()`` returns a list of unicode strings. So you +``.css()`` methods, ``.re()`` returns a list of strings. So you can't construct nested ``.re()`` calls. Here's an example used to extract image names from the :ref:`HTML code @@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: sel.xpath("//h1") 2. Extract the text of all ``

    `` elements from an HTML response body, - returning a list of unicode strings:: + returning a list of strings:: sel.xpath("//h1").getall() # this includes the h1 tag sel.xpath("//h1/text()").getall() # this excludes the h1 tag From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Tue, 4 Aug 2020 20:42:01 +0200 Subject: [PATCH 117/125] Change super syntax (#4707) --- extras/qpsclient.py | 2 +- scrapy/commands/view.py | 2 +- scrapy/contracts/default.py | 2 +- scrapy/core/downloader/contextfactory.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 10 +++++----- scrapy/core/downloader/tls.py | 2 +- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/exceptions.py | 4 ++-- scrapy/exporters.py | 2 +- scrapy/http/headers.py | 8 ++++---- scrapy/http/request/form.py | 2 +- scrapy/http/request/json_request.py | 4 ++-- scrapy/http/request/rpc.py | 2 +- scrapy/http/response/text.py | 10 +++++----- scrapy/item.py | 10 +++++----- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 3 +-- scrapy/resolver.py | 10 +++++----- scrapy/selector/unified.py | 2 +- scrapy/settings/__init__.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spiders/crawl.py | 4 ++-- scrapy/spiders/init.py | 2 +- scrapy/spiders/sitemap.py | 2 +- scrapy/squeues.py | 12 ++++++------ scrapy/statscollectors.py | 2 +- scrapy/utils/datatypes.py | 14 +++++++------- scrapy/utils/deprecate.py | 8 ++++---- scrapy/utils/log.py | 2 +- scrapy/utils/serialize.py | 2 +- scrapy/utils/testsite.py | 4 ++-- tests/spiders.py | 20 ++++++++++---------- tests/test_command_parse.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_contracts.py | 2 +- tests/test_downloader_handlers.py | 4 ++-- tests/test_downloadermiddleware_httpcache.py | 4 ++-- tests/test_downloadermiddleware_robotstxt.py | 4 ++-- tests/test_exporters.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_http_response.py | 2 +- tests/test_item.py | 2 +- tests/test_linkextractors.py | 2 +- tests/test_loader.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_media.py | 12 ++++++------ tests/test_request_left.py | 2 +- tests/test_robotstxt_interface.py | 8 ++++---- tests/test_scheduler.py | 4 ++-- tests/test_spidermiddleware_httperror.py | 2 +- 56 files changed, 117 insertions(+), 118 deletions(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 7554f7eec..fe1f96cbb 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -27,7 +27,7 @@ class QPSSpider(Spider): slots = 1 def __init__(self, *a, **kw): - super(QPSSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) if self.qps is not None: self.qps = float(self.qps) self.download_delay = 1 / self.qps diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 908bee966..c8f873334 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): - super(Command, self).add_options(parser) + super().add_options(parser) parser.remove_option("--headers") def _print_response(self, response, opts): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 34f0d36d4..cfdcc7c25 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -56,7 +56,7 @@ class ReturnsContract(Contract): } def __init__(self, *args, **kwargs): - super(ReturnsContract, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: raise ValueError( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..8a7d656a1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): - super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging if tls_ciphers: @@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) # # * getattr() for `_ssl_method` attribute for context factories - # not calling super(..., self).__init__ + # not calling super().__init__ return CertificateOptions( verify=False, method=getattr(self, 'method', getattr(self, '_ssl_method', None)), diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..fb04d1fb7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf - super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) + super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() self._tunneledHost = host self._tunneledPort = port @@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def connect(self, protocolFactory): self._protocolFactory = protocolFactory - connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) + connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) connectDeferred.addErrback(self.connectFailed) return self._tunnelReadyDeferred @@ -215,7 +215,7 @@ class TunnelingAgent(Agent): def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): - super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory @@ -235,7 +235,7 @@ class TunnelingAgent(Agent): # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy key = key + self._proxyConf - return super(TunnelingAgent, self)._requestWithEndpoint( + return super()._requestWithEndpoint( key=key, endpoint=endpoint, method=method, @@ -249,7 +249,7 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): - super(ScrapyProxyAgent, self).__init__( + super().__init__( reactor=reactor, connectTimeout=connectTimeout, bindAddress=bindAddress, diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e43a3c83e..d9f3750d5 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): """ def __init__(self, hostname, ctx, verbose_logging=False): - super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + super().__init__(hostname, ctx) self.verbose_logging = verbose_logging def _identityVerifyingInfoCallback(self, connection, where, ret): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 35264a92b..5a99b96be 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): - super(SpiderMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f43771e2..48f19424c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner): """ def __init__(self, settings=None, install_root_handler=True): - super(CrawlerProcess, self).__init__(settings) + super().__init__(settings) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 366d60dcb..4053fecc5 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = 'METAREFRESH_ENABLED' def __init__(self, settings): - super(MetaRefreshMiddleware, self).__init__(settings) + super().__init__(settings) self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 45f152321..0c410f035 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -37,7 +37,7 @@ class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" def __init__(self, reason='cancelled'): - super(CloseSpider, self).__init__() + super().__init__() self.reason = reason @@ -74,7 +74,7 @@ class UsageError(Exception): def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) - super(UsageError, self).__init__(*a, **kw) + super().__init__(*a, **kw) class ScrapyDeprecationWarning(Warning): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0aba1c904..95518b3ac 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter): def _configure(self, options, dont_fail=False): self.binary = options.pop('binary', True) - super(PythonItemExporter, self)._configure(options, dont_fail) + super()._configure(options, dont_fail) if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index dcaaeddfa..6bf9e5346 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -8,7 +8,7 @@ class Headers(CaselessDict): def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding - super(Headers, self).__init__(seq) + super().__init__(seq) def normkey(self, key): """Normalize key to bytes""" @@ -37,19 +37,19 @@ class Headers(CaselessDict): def __getitem__(self, key): try: - return super(Headers, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] except IndexError: return None def get(self, key, def_val=None): try: - return super(Headers, self).get(key, def_val)[-1] + return super().get(key, def_val)[-1] except IndexError: return None def getlist(self, key, def_val=None): try: - return super(Headers, self).__getitem__(key) + return super().__getitem__(key) except KeyError: if def_val is not None: return self.normvalue(def_val) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a260798ac..59af81321 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,7 @@ class FormRequest(Request): if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' - super(FormRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f08b25280..eae3f9f6b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -32,7 +32,7 @@ class JsonRequest(Request): if 'method' not in kwargs: kwargs['method'] = 'POST' - super(JsonRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -47,7 +47,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - return super(JsonRequest, self).replace(*args, **kwargs) + return super().replace(*args, **kwargs) def _dumps(self, data): """Convert to JSON """ diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 811d3ad6b..c70912e49 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -31,5 +31,5 @@ class XmlRpcRequest(Request): if encoding is not None: kwargs['encoding'] = encoding - super(XmlRpcRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'text/xml') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0f300c8da..a7bb34d48 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,13 +35,13 @@ class TextResponse(Response): self._cached_benc = None self._cached_ubody = None self._cached_selector = None - super(TextResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def _set_url(self, url): if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: - super(TextResponse, self)._set_url(url) + super()._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection @@ -51,7 +51,7 @@ class TextResponse(Response): type(self).__name__) self._body = body.encode(self._encoding) else: - super(TextResponse, self)._set_body(body) + super()._set_body(body) def replace(self, *args, **kwargs): kwargs.setdefault('encoding', self.encoding) @@ -166,7 +166,7 @@ class TextResponse(Response): elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding - return super(TextResponse, self).follow( + return super().follow( url=url, callback=callback, method=method, @@ -226,7 +226,7 @@ class TextResponse(Response): for sel in selectors: with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) - return super(TextResponse, self).follow_all( + return super().follow_all( urls=urls, callback=callback, method=method, diff --git a/scrapy/item.py b/scrapy/item.py index 4ab83d1a0..c262a153c 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta): if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(BaseItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) class Field(dict): @@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta): def __new__(mcs, class_name, bases, attrs): classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) - _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) + _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs) fields = getattr(_class, 'fields', {}) new_attrs = {} @@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta): new_attrs['_class'] = _class if classcell is not None: new_attrs['__classcell__'] = classcell - return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) + return super().__new__(mcs, class_name, bases, new_attrs) class DictItem(MutableMapping, BaseItem): @@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem): if issubclass(cls, DictItem) and not issubclass(cls, Item): warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(DictItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): self._values = {} @@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) - super(DictItem, self).__setattr__(name, value) + super().__setattr__(name, value) def __len__(self): return len(self._values) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 984a5c4e1..08a6ca1e8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -65,7 +65,7 @@ class FilteringLinkExtractor: warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super(FilteringLinkExtractor, cls).__new__(cls) + return super().__new__(cls) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8b9f961ee..e941c4321 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super(LxmlLinkExtractor, self).__init__( + super().__init__( link_extractor=lx, allow=allow, deny=deny, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 487382a38..6bc5d46eb 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) + super().__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 46f2bfb58..e2dd70215 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): - super(ImagesPipeline, self).__init__(store_uri, settings=settings, - download_func=download_func) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: settings = Settings(settings) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..f191deac6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver): """ def __init__(self, reactor, cache_size, timeout): - super(CachingThreadedResolver, self).__init__(reactor) + super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver): # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) - d = super(CachingThreadedResolver, self).getHostByName(name, timeout) + d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d @@ -80,16 +80,16 @@ class CachingHostnameResolver: class CachingResolutionReceiver(resolutionReceiver): def resolutionBegan(self, resolution): - super(CachingResolutionReceiver, self).resolutionBegan(resolution) + super().resolutionBegan(resolution) self.resolution = resolution self.resolved = False def addressResolved(self, address): - super(CachingResolutionReceiver, self).addressResolved(address) + super().addressResolved(address) self.resolved = True def resolutionComplete(self): - super(CachingResolutionReceiver, self).resolutionComplete() + super().resolutionComplete() if self.resolved: dnscache[hostName] = self.resolution diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 85a9bb526..f12c61081 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault('base_url', response.url) self.response = response - super(Selector, self).__init__(text=text, type=st, root=root, **kwargs) + super().__init__(text=text, type=st, root=root, **kwargs) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ff8317cd1..b8ae32d7c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -439,7 +439,7 @@ class Settings(BaseSettings): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user - super(Settings, self).__init__() + super().__init__() self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..db9d0f2ae 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -15,7 +15,7 @@ class HttpError(IgnoreRequest): def __init__(self, response, *args, **kwargs): self.response = response - super(HttpError, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class HttpErrorMiddleware: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index cb7260892..c9fbce08d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -75,7 +75,7 @@ class CrawlSpider(Spider): rules = () def __init__(self, *a, **kw): - super(CrawlSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._compile_rules() def _parse(self, response, **kwargs): @@ -145,6 +145,6 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) return spider diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index fd41133ea..fe8c94e78 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -6,7 +6,7 @@ class InitSpider(Spider): """Base Spider with initialization facilities""" def start_requests(self): - self._postinit_reqs = super(InitSpider, self).start_requests() + self._postinit_reqs = super().start_requests() return iterate_spider_output(self.init_request()) def initialized(self, response=None): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c5360bfa7..1f72e76b7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -18,7 +18,7 @@ class SitemapSpider(Spider): sitemap_alternate_links = False def __init__(self, *a, **kw): - super(SitemapSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, str): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index c7ad4d53d..77ffda6f7 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -20,7 +20,7 @@ def _with_mkdir(queue_class): if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def push(self, obj): s = serialize(obj) - super(SerializableQueue, self).push(s) + super().push(s) def pop(self): - s = super(SerializableQueue, self).pop() + s = super().pop() if s: return deserialize(s) @@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class): def __init__(self, crawler, key): self.spider = crawler.spider - super(ScrapyRequestQueue, self).__init__(key) + super().__init__(key) @classmethod def from_crawler(cls, crawler, key, *args, **kwargs): @@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class): def push(self, request): request = request_to_dict(request, self.spider) - return super(ScrapyRequestQueue, self).push(request) + return super().push(request) def pop(self): - request = super(ScrapyRequestQueue, self).pop() + request = super().pop() if not request: return None diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 579c60180..ba7d1a6bf 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -54,7 +54,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler): - super(MemoryStatsCollector, self).__init__(crawler) + super().__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2a92d0588..e31284a7f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -15,7 +15,7 @@ class CaselessDict(dict): __slots__ = () def __init__(self, seq=None): - super(CaselessDict, self).__init__() + super().__init__() if seq: self.update(seq) @@ -53,7 +53,7 @@ class CaselessDict(dict): def update(self, seq): seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) - super(CaselessDict, self).update(iseq) + super().update(iseq) @classmethod def fromkeys(cls, keys, value=None): @@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict): """ def __init__(self, limit=None): - super(LocalCache, self).__init__() + super().__init__() self.limit = limit def __setitem__(self, key, value): if self.limit: while len(self) >= self.limit: self.popitem(last=False) - super(LocalCache, self).__setitem__(key, value) + super().__setitem__(key, value) class LocalWeakReferencedCache(weakref.WeakKeyDictionary): @@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ def __init__(self, limit=None): - super(LocalWeakReferencedCache, self).__init__() + super().__init__() self.data = LocalCache(limit=limit) def __setitem__(self, key, value): try: - super(LocalWeakReferencedCache, self).__setitem__(key, value) + super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching def __getitem__(self, key): try: - return super(LocalWeakReferencedCache, self).__getitem__(key) + return super().__getitem__(key) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3dbea5fee..3c8e3c8b5 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,7 +57,7 @@ def create_deprecated_class( warned_on_subclass = False def __new__(metacls, name, bases, clsdict_): - cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_) + cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -73,7 +73,7 @@ def create_deprecated_class( if warn_once: msg += ' (warning only on first subclass, there may be others)' warnings.warn(msg, warn_category, stacklevel=2) - super(DeprecatedClass, cls).__init__(name, bases, clsdict_) + super().__init__(name, bases, clsdict_) # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks @@ -88,7 +88,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super(DeprecatedClass, cls).__subclasscheck__(sub) + return super().__subclasscheck__(sub) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -102,7 +102,7 @@ def create_deprecated_class( msg = instance_warn_message.format(cls=_clspath(cls, old_class_path), new=_clspath(new_class, new_class_path)) warnings.warn(msg, warn_category, stacklevel=2) - return super(DeprecatedClass, cls).__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {}) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..1d6a2c39d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" def __init__(self, crawler, *args, **kwargs): - super(LogCounterHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.crawler = crawler def emit(self, record): diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index dc9604578..cc3263602 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif isinstance(o, Response): return "<%s %s %s>" % (type(o).__name__, o.status, o.url) else: - return super(ScrapyJSONEncoder, self).default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 66930ad2c..397e54703 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -7,12 +7,12 @@ class SiteTest: def setUp(self): from twisted.internet import reactor - super(SiteTest, self).setUp() + super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.baseurl = "http://localhost:%d/" % self.site.getHost().port def tearDown(self): - super(SiteTest, self).tearDown() + super().tearDown() self.site.stopListening() def url(self, path): diff --git a/tests/spiders.py b/tests/spiders.py index 3eb681819..63bd726fb 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): - super(MockServerSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.mockserver = mockserver @@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider): name = 'meta' def __init__(self, *args, **kwargs): - super(MetaSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.meta = {} def closed(self, reason): @@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): - super(FollowAllSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} @@ -60,7 +60,7 @@ class DelaySpider(MetaSpider): name = 'delay' def __init__(self, n=1, b=0, *args, **kwargs): - super(DelaySpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.n = n self.b = b self.t1 = self.t2 = self.t2_err = 0 @@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider): name = 'simple' def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(SimpleSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [url] def parse(self, response): @@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider): name = 'item' def parse(self, response): - for request in super(ItemSpider, self).parse(response): + for request in super().parse(response): yield request yield Item() yield {} @@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider): raise self.exception_cls('Expected exception') def parse(self, response): - for request in super(ErrorSpider, self).parse(response): + for request in super().parse(response): yield request self.raise_exception() @@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): fail_yielding = False def __init__(self, *a, **kw): - super(BrokenStartRequestsSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self.seedsseen = [] def start_requests(self): @@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get('seed')) - for req in super(BrokenStartRequestsSpider, self).parse(response): + for req in super().parse(response): yield req @@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def parse(self, response): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5754a5478..e115f420f 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' def setUp(self): - super(ParseCommandTest, self).setUp() + super().setUp() self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: diff --git a/tests/test_commands.py b/tests/test_commands.py index 42091ab00..8938156fc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None): class StartprojectTemplatesTest(ProjectTest): def setUp(self): - super(StartprojectTemplatesTest, self).setUp() + super().setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') @@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest): class CommandTest(ProjectTest): def setUp(self): - super(CommandTest, self).setUp() + super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 99120b128..2e7e3ccc4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase): name = 'test_same_url' def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def start_requests(s): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57d4cdd6b..13063d106 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https11InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https11InvalidDNSPattern, self).setUp() + super().setUp() class Https11CustomCiphers(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9b77c97a8..299fb0eb8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) - return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded @@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_GZIP', True) - return super(FilesystemStorageTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) class DummyPolicyTest(_BaseTest): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f9936baba..858138f81 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): skip = "Rerp parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithRerpTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') @@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): skip = "Reppy parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithReppyTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 660c99ce1..6c25a0064 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase): if name == 'age': return str(int(value) + 1) else: - return super(CustomItemExporter, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index f5cf4e798..0a303dbe2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest): def setUp(self): warnings.simplefilter("always") - super(JsonRequestTest, self).setUp() + super().setUp() def test_data(self): r1 = self.request_class(url="http://www.example.com/") @@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() - super(JsonRequestTest, self).tearDown() + super().tearDown() if __name__ == "__main__": diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 56d017de6..f831ef5dc 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest): response_class = TextResponse def test_replace(self): - super(TextResponseTest, self).test_replace() + super().test_replace() r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") diff --git a/tests/test_item.py b/tests/test_item.py index 0ce78f8c0..66fa761f0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase): # requirement. When not done properly raises an error: # TypeError: __class__ set to # defining 'MyItem' as - super(MyItem, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class DictItemTest(unittest.TestCase): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a0bafa5e5..6f133d77a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ]) def test_restrict_xpaths_with_html_entities(self): - super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + super().test_restrict_xpaths_with_html_entities() def test_filteringlinkextractor_deprecation_warning(self): """Make sure the FilteringLinkExtractor deprecation warning is not diff --git a/tests/test_loader.py b/tests/test_loader.py index 2ed6f365f..b0bc82f4e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase): temp = Field() def __init__(self, *args, **kwargs): - super(TempItem, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index eb14de14f..624dd9ab8 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase): class TempDict(dict): def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index b771e7d79..41ff3651d 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase): class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) + kwargs = super().crawled(request, response, spider) CRAWLEDMSG = ( "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3364d2258..b2b75ef20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager): return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): - super(TestMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process'): self.methods['process'].append(mw.process) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 19ff00350..4f130c0c9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase): class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipeline, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def download(self, request, info): self._mockcalled.append('download') - return super(MockedMediaPipeline, self).download(request, info) + return super().download(request, info) def media_to_download(self, request, info): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') - return super(MockedMediaPipeline, self).media_to_download(request, info) + return super().media_to_download(request, info) def get_media_requests(self, item, info): self._mockcalled.append('get_media_requests') @@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline): def media_downloaded(self, response, request, info): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipeline, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): self._mockcalled.append('media_failed') - return super(MockedMediaPipeline, self).media_failed(failure, request, info) + return super().media_failed(failure, request, info) def item_completed(self, results, item, info): self._mockcalled.append('item_completed') - item = super(MockedMediaPipeline, self).item_completed(results, item, info) + item = super().item_completed(results, item, info) item['results'] = results return item diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5cfef8e7d..373b2e49c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider): name = 'signal_catcher' def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9d8c201dd..4b15d0fab 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -117,7 +117,7 @@ class BaseRobotParserTest: class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser - super(PythonRobotParserTest, self)._setUp(PythonRobotParser) + super()._setUp(PythonRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.") @@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ReppyRobotParser - super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) + super()._setUp(ReppyRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Reppy does not support order based directives precedence.") @@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import RerpRobotParser - super(RerpRobotParserTest, self)._setUp(RerpRobotParser) + super()._setUp(RerpRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("Rerp does not support length based directives precedence.") @@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ProtegoRobotParser - super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser) + super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2b6cb0902..512a7460e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,7 +53,7 @@ class MockCrawler(Crawler): JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', ) - super(MockCrawler, self).__init__(Spider, settings) + super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -296,7 +296,7 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls - super(StartUrlsSpider, self).__init__(name='StartUrlsSpider') + super().__init__(name='StartUrlsSpider') def parse(self, response): pass diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..e449cd706 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider): bypass_status_codes = set() def __init__(self, *args, **kwargs): - super(_HttpErrorSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [ self.mockserver.url("/status?n=200"), self.mockserver.url("/status?n=404"), From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:11:59 +0530 Subject: [PATCH 118/125] deprecated weakkeycache by specifying in __init__ --- scrapy/utils/python.py | 4 +++- scrapy/utils/tester.py | 3 +++ tests/test_utils_python.py | 22 +++++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index c8f921ff3..4756b07b6 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,10 +7,12 @@ import inspect import re import sys import weakref +import warnings from functools import partial, wraps from itertools import chain from scrapy.utils.decorators import deprecated +from scrapy.exceptions import ScrapyDeprecationWarning def flatten(x): @@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes): return True -@deprecated class WeakKeyCache: def __init__(self, default_factory): + warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py new file mode 100644 index 000000000..691e9bc1a --- /dev/null +++ b/scrapy/utils/tester.py @@ -0,0 +1,3 @@ +from scrapy.utils.decorators import deprecated + + diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5a53d89e4..ebce3c079 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,15 +1,18 @@ import functools +import gc import operator import platform import unittest +from itertools import count from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) + __doctests__ = ['scrapy.utils.python'] @@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) + def test_weakkeycache(self): + class _Weakme: + pass + + _values = count() + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() + v = wk[k] + self.assertEqual(v, wk[k]) + self.assertNotEqual(v, wk[_Weakme()]) + self.assertEqual(v, wk[k]) + del k + for _ in range(100): + if wk._weakdict: + gc.collect() + self.assertFalse(len(wk._weakdict)) + def test_get_func_args(self): def f1(a, b, c): pass From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:14:04 +0530 Subject: [PATCH 119/125] deleted tester.py --- scrapy/utils/tester.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py deleted file mode 100644 index 691e9bc1a..000000000 --- a/scrapy/utils/tester.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.utils.decorators import deprecated - - From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 16:13:52 +0530 Subject: [PATCH 120/125] aesthetic fixes --- scrapy/utils/python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 4756b07b6..59f1b8371 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,13 +6,13 @@ import gc import inspect import re import sys -import weakref import warnings +import weakref from functools import partial, wraps from itertools import chain -from scrapy.utils.decorators import deprecated from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import deprecated def flatten(x): @@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes): class WeakKeyCache: def __init__(self, default_factory): - warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() From 4dc09f09aa9698b02f2cbf2e3001202388eba043 Mon Sep 17 00:00:00 2001 From: linchiwei123 <40888469+linchiwei123@users.noreply.github.com> Date: Wed, 5 Aug 2020 22:23:19 +0800 Subject: [PATCH 121/125] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 58090f7a2..d0880051f 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ install_requires = [ 'cryptography>=2.0', 'cssselect>=0.9.1', 'itemloaders>=1.0.1', - 'lxml>=3.5.0', 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', 'pyOpenSSL>=16.2.0', From 1c4b4cc6b046e121be33a57031a0900ca1347e16 Mon Sep 17 00:00:00 2001 From: Ajay Mittur Date: Tue, 11 Aug 2020 17:42:44 +0530 Subject: [PATCH 122/125] Support defining file path based on item in media pipelines (#4686) --- docs/topics/media-pipeline.rst | 24 ++++--- scrapy/pipelines/files.py | 16 ++--- scrapy/pipelines/images.py | 14 ++-- scrapy/pipelines/media.py | 69 +++++++++++++++--- tests/test_pipeline_files.py | 13 ++++ tests/test_pipeline_media.py | 123 ++++++++++++++++++++++++++++++++- 6 files changed, 225 insertions(+), 34 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1f995ce14..487e26b8e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. @@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. - .. method:: file_path(self, request, response=None, info=None) + .. method:: file_path(self, request, response=None, info=None, *, item=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified :class:`response `. In addition to ``response``, this method receives the original - :class:`request ` and - :class:`info `. + :class:`request `, + :class:`info ` and + :class:`item ` You can override this method to customize the download path of each file. @@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): return 'files/' + os.path.basename(urlparse(request.url).path) + Similarly, you can use the ``item`` to determine the file path based on some item + property. + By default the :meth:`file_path` method returns ``full/.``. diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 6bc5d46eb..5a2184681 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): def _onsuccess(result): if not result: return # returning None force download @@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline): checksum = result.get('checksum', None) return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'} - path = self.file_path(request, info=info) + path = self.file_path(request, info=info, item=item) dfd = defer.maybeDeferred(self.store.stat_file, path, info) dfd.addCallbacks(_onsuccess, lambda _: None) dfd.addErrback( @@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline): raise FileException - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): referer = referer_str(request) if response.status != 200: @@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline): self.inc_stats(info.spider, status) try: - path = self.file_path(request, response=response, info=info) - checksum = self.file_downloaded(response, request, info) + path = self.file_path(request, response=response, info=info, item=item) + checksum = self.file_downloaded(response, request, info, item=item) except FileException as exc: logger.warning( 'File (error): Error processing file from %(request)s ' @@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline): urls = ItemAdapter(item).get(self.files_urls_field, []) return [Request(u) for u in urls] - def file_downloaded(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def file_downloaded(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) buf = BytesIO(response.body) checksum = md5sum(buf) buf.seek(0) @@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline): ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() media_ext = os.path.splitext(request.url)[1] # Handles empty and wild extensions by trying to guess the diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e2dd70215..0a67a0b1d 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) - def file_downloaded(self, response, request, info): - return self.image_downloaded(response, request, info) + def file_downloaded(self, response, request, info, *, item=None): + return self.image_downloaded(response, request, info, item=item) - def image_downloaded(self, response, request, info): + def image_downloaded(self, response, request, info, *, item=None): checksum = None - for path, image, buf in self.get_images(response, request, info): + for path, image, buf in self.get_images(response, request, info, item=item): if checksum is None: buf.seek(0) checksum = md5sum(buf) @@ -119,8 +119,8 @@ class ImagesPipeline(FilesPipeline): headers={'Content-Type': 'image/jpeg'}) return checksum - def get_images(self, response, request, info): - path = self.file_path(request, response=response, info=info) + def get_images(self, response, request, info, *, item=None): + path = self.file_path(request, response=response, info=info, item=item) orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size @@ -166,7 +166,7 @@ class ImagesPipeline(FilesPipeline): ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok] return item - def file_path(self, request, response=None, info=None): + def file_path(self, request, response=None, info=None, *, item=None): image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return 'full/%s.jpg' % (image_guid) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index aa65f4f0e..2439de9a5 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,12 +1,16 @@ import functools import logging from collections import defaultdict +from inspect import signature +from warnings import warn + from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure from scrapy.settings import Settings from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter from scrapy.utils.log import failure_to_exc_info @@ -27,6 +31,7 @@ class MediaPipeline: def __init__(self, download_func=None, settings=None): self.download_func = download_func + self._expects_item = {} if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -38,6 +43,9 @@ class MediaPipeline: ) self._handle_statuses(self.allow_redirects) + # Check if deprecated methods are being used and make them compatible + self._make_compatible() + def _handle_statuses(self, allow_redirects): self.handle_httpstatus_list = None if allow_redirects: @@ -77,11 +85,11 @@ class MediaPipeline: def process_item(self, item, spider): info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) - dlist = [self._process_request(r, info) for r in requests] + dlist = [self._process_request(r, info, item) for r in requests] dfd = DeferredList(dlist, consumeErrors=1) return dfd.addCallback(self.item_completed, item, info) - def _process_request(self, request, info): + def _process_request(self, request, info, item): fp = request_fingerprint(request) cb = request.callback or (lambda _: _) eb = request.errback @@ -102,34 +110,73 @@ class MediaPipeline: # Download request checking media_to_download hook output first info.downloading.add(fp) - dfd = mustbe_deferred(self.media_to_download, request, info) - dfd.addCallback(self._check_media_to_download, request, info) + dfd = mustbe_deferred(self.media_to_download, request, info, item=item) + dfd.addCallback(self._check_media_to_download, request, info, item=item) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(lambda f: logger.error( f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider}) ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _make_compatible(self): + """Make overridable methods of MediaPipeline and subclasses backwards compatible""" + methods = [ + "file_path", "media_to_download", "media_downloaded", + "file_downloaded", "image_downloaded", "get_images" + ] + + for method_name in methods: + method = getattr(self, method_name, None) + if callable(method): + setattr(self, method_name, self._compatible(method)) + + def _compatible(self, func): + """Wrapper for overridable methods to allow backwards compatibility""" + self._check_signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if self._expects_item[func.__name__]: + return func(*args, **kwargs) + + kwargs.pop('item', None) + return func(*args, **kwargs) + + return wrapper + + def _check_signature(self, func): + sig = signature(func) + self._expects_item[func.__name__] = True + + if 'item' not in sig.parameters: + old_params = str(sig)[1:-1] + new_params = old_params + ", *, item=None" + warn('%s(self, %s) is deprecated, ' + 'please use %s(self, %s)' + % (func.__name__, old_params, func.__name__, new_params), + ScrapyDeprecationWarning, stacklevel=2) + self._expects_item[func.__name__] = False + def _modify_media_request(self, request): if self.handle_httpstatus_list: request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - def _check_media_to_download(self, result, request, info): + def _check_media_to_download(self, result, request, info, item): if result is not None: return result if self.download_func: # this ugly code was left only to support tests. TODO: remove dfd = mustbe_deferred(self.download_func, request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) else: self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( - callback=self.media_downloaded, callbackArgs=(request, info), + callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item}, errback=self.media_failed, errbackArgs=(request, info)) return dfd @@ -171,7 +218,7 @@ class MediaPipeline: defer_result(result).chainDeferred(wad) # Overridable Interface - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): """Check request before starting download""" pass @@ -179,7 +226,7 @@ class MediaPipeline: """Returns the media requests to download""" pass - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): """Handler for success downloads""" return response @@ -199,3 +246,7 @@ class MediaPipeline: extra={'spider': info.spider} ) return item + + def file_path(self, request, response=None, info=None, *, item=None): + """Returns the path where downloaded media should be stored""" + pass diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a023dfcc8..b19b4ff2a 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase): for p in patchers: p.stop() + def test_file_path_from_item(self): + """ + Custom file path based on item data, overriding default implementation + """ + class CustomFilesPipeline(FilesPipeline): + def file_path(self, request, response=None, info=None, item=None): + return 'full/%s' % item.get('path') + + file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path + item = dict(path='path-to-store-file') + request = Request("http://example.com") + self.assertEqual(file_path(request, item=item), 'full/path-to-store-file') + class FilesPipelineTestCaseFieldsMixin: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 4f130c0c9..6afd47497 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.request import request_fingerprint +from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info @@ -169,7 +171,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('download') return super().download(request, info) - def media_to_download(self, request, info): + def media_to_download(self, request, info, *, item=None): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') @@ -179,7 +181,7 @@ class MockedMediaPipeline(MediaPipeline): self._mockcalled.append('get_media_requests') return item.get('requests') - def media_downloaded(self, response, request, info): + def media_downloaded(self, response, request, info, *, item=None): self._mockcalled.append('media_downloaded') return super().media_downloaded(response, request, info) @@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): ['get_media_requests', 'media_to_download', 'item_completed']) +class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): + + def __init__(self, *args, **kwargs): + super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs) + self._mockcalled = [] + + def get_media_requests(self, item, info): + item_url = item['image_urls'][0] + return Request( + item_url, + meta={'response': Response(item_url, status=200, body=b'data')} + ) + + def inc_stats(self, *args, **kwargs): + return True + + def media_to_download(self, request, info): + self._mockcalled.append('media_to_download') + return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info) + + def media_downloaded(self, response, request, info): + self._mockcalled.append('media_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info) + + def file_downloaded(self, response, request, info): + self._mockcalled.append('file_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info) + + def file_path(self, request, response=None, info=None): + self._mockcalled.append('file_path') + return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info) + + def get_images(self, response, request, info): + self._mockcalled.append('get_images') + return [] + + def image_downloaded(self, response, request, info): + self._mockcalled.append('image_downloaded') + return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info) + + +class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): + + def setUp(self): + self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func) + self.pipe.open_spider(None) + self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[]) + + def _assert_method_called_with_warnings(self, method, message, warnings): + self.assertIn(method, self.pipe._mockcalled) + warningShown = False + for warning in warnings: + if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning: + warningShown = True + self.assertTrue(warningShown) + + @inlineCallbacks + def test_media_to_download_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_to_download(self, request, info) is deprecated, ' + 'please use media_to_download(self, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_to_download', message, warnings) + + @inlineCallbacks + def test_media_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'media_downloaded(self, response, request, info) is deprecated, ' + 'please use media_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('media_downloaded', message, warnings) + + @inlineCallbacks + def test_file_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_downloaded(self, response, request, info) is deprecated, ' + 'please use file_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('file_downloaded', message, warnings) + + @inlineCallbacks + def test_file_path_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'file_path(self, request, response=None, info=None) is deprecated, ' + 'please use file_path(self, request, response=None, info=None, *, item=None)' + ) + self._assert_method_called_with_warnings('file_path', message, warnings) + + @inlineCallbacks + def test_get_images_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'get_images(self, response, request, info) is deprecated, ' + 'please use get_images(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('get_images', message, warnings) + + @inlineCallbacks + def test_image_downloaded_called(self): + yield self.pipe.process_item(self.item, None) + warnings = self.flushWarnings([MediaPipeline._compatible]) + message = ( + 'image_downloaded(self, response, request, info) is deprecated, ' + 'please use image_downloaded(self, response, request, info, *, item=None)' + ) + self._assert_method_called_with_warnings('image_downloaded', message, warnings) + + class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): def _assert_request_no3xx(self, pipeline_class, settings): From 5f4df622a17b1d8f9c3b9b693cb2f7119ffec27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 05:41:06 +0200 Subject: [PATCH 123/125] test_utils_iterators.py: support Windows the right way --- tests/__init__.py | 2 +- tests/test_utils_iterators.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 12ce79fa9..4253017fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb') as f: + with open(path, 'rb', newline='') as f: return f.read() diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 298178f08..50190d4d1 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata -FOOBAR_NL = "foo{}bar".format(os.linesep) - - class XmliterTestCase(unittest.TestCase): xmliter = staticmethod(xmliter) @@ -267,7 +264,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual(result, [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr @@ -283,7 +280,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_quotechar(self): @@ -296,7 +293,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv1], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) response2 = TextResponse(url="http://example.com/", body=body2) @@ -305,7 +302,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv2], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_wrong_quotechar(self): @@ -327,7 +324,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_headers(self): @@ -353,7 +350,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{'id': '1', 'name': 'alpha', 'value': 'foobar'}, {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'}, - {'id': '3', 'name': 'multi', 'value': FOOBAR_NL}, + {'id': '3', 'name': 'multi', 'value': "foo\nbar"}, {'id': '4', 'name': 'empty', 'value': ''}]) def test_csviter_exception(self): From 24ba5a71aca9b368eee21bd7d9043a7d26dba403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 06:35:09 +0200 Subject: [PATCH 124/125] Maybe the problem is not in the code after all --- .gitattributes | 1 + tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..dfbdf4208 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/sample_data/** binary diff --git a/tests/__init__.py b/tests/__init__.py index 4253017fa..12ce79fa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -28,5 +28,5 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - with open(path, 'rb', newline='') as f: + with open(path, 'rb') as f: return f.read() From 65e0abaea5eddb1dbb28b2bca9cd00e9b9409471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 09:05:51 +0200 Subject: [PATCH 125/125] Document FEED_URI_PARAMS --- docs/topics/feed-exports.rst | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 37b7096f6..0f0f258dc 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -321,13 +321,14 @@ The following is a list of the accepted keys and the setting that is used as a fallback value if that key is not provided for a specific feed definition. * ``format``: the serialization format to be used for the feed. - See :ref:`topics-feed-format` for possible values. + See :ref:`topics-feed-format` for possible values. Mandatory, no fallback setting +* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` * ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` * ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` * ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` * ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` -* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` +* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS` .. setting:: FEED_EXPORT_ENCODING @@ -500,7 +501,7 @@ generated: * ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created (e.g. ``2020-03-28T14-45-08.237134``) -* ``%(batch_id)d`` - gets replaced by the sequence number of the batch. +* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch. Use :ref:`printf-style string formatting ` to alter the number format. For example, to make the batch ID a 5-digit @@ -517,16 +518,74 @@ And your :command:`crawl` command line is:: The command line above can generate a directory tree like:: -->projectname --->dirname ---->1-filename2020-03-28T14-45-08.237134.json ---->2-filename2020-03-28T14-45-09.148903.json ---->3-filename2020-03-28T14-45-10.046092.json + ->projectname + -->dirname + --->1-filename2020-03-28T14-45-08.237134.json + --->2-filename2020-03-28T14-45-09.148903.json + --->3-filename2020-03-28T14-45-10.046092.json Where the first and second files contain exactly 100 items. The last one contains 100 items or fewer. +.. setting:: FEED_URI_PARAMS + +FEED_URI_PARAMS +--------------- + +Default: ``None`` + +A string with the import path of a function to set the parameters to apply with +:ref:`printf-style string formatting ` to the +feed URI. + +The function signature should be as follows: + +.. function:: uri_params(params, spider) + + Return a :class:`dict` of key-value pairs to apply to the feed URI using + :ref:`printf-style string formatting `. + + :param params: default key-value pairs + + Specifically: + + - ``batch_id``: ID of the file batch. See + :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id`` + is always ``1``. + + - ``batch_time``: UTC date and time, in ISO format with ``:`` + replaced with ``-``. + + See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`. + + - ``time``: ``batch_time``, with microseconds set to ``0``. + :type params: dict + + :param spider: source spider of the feed items + :type spider: scrapy.spiders.Spider + +For example, to include the :attr:`name ` of the +source spider in the feed URI: + +#. Define the following function somewhere in your project:: + + # myproject/utils.py + def uri_params(params, spider): + return {**params, 'spider_name': spider.name} + +#. Point :setting:`FEED_URI_PARAMS` to that function in your settings:: + + # myproject/settings.py + FEED_URI_PARAMS = 'myproject.utils.uri_params' + +#. Use ``%(spider_name)s`` in your feed URI:: + + scrapy crawl -o "%(spider_name)s.jl" + + .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore