RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 4307-use-f-strings 

Resolve Conflicts:
	tests/test_middleware.py
This commit is contained in:
Ammar Najjar 2020-08-26 13:46:26 +02:00
parent 9aaddcde0a cf50561b86
commit 7597193dbe
No known key found for this signature in database
GPG Key ID: 4E358D38135EE7EA
10 changed files with 101 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/topics/settings.rst
View File

@ -98,6 +98,32 @@ class.
The global defaults are located in the ``scrapy.settings.default_settings``
module and documented in the :ref:`topics-settings-ref` section.
Import paths and classes
========================
.. versionadded:: VERSION
When a setting references a callable object to be imported by Scrapy, such as a
class or a function, there are two different ways you can specify that object:
- As a string containing the import path of that object
- As the object itself
For example::
from mybot.pipelines.validate import ValidateMyItem
ITEM_PIPELINES = {
# passing the classname...
ValidateMyItem: 300,
# ...equals passing the class path
'mybot.pipelines.validate.ValidateMyItem': 300,
}
.. note:: Passing non-callable objects is not supported.
How to access settings
======================

2
scrapy/utils/deprecate.py
View File

@ -134,7 +134,7 @@ DEPRECATION_RULES = [
def update_classpath(path):
"""Update a deprecated path from an object with its new location"""
for prefix, replacement in DEPRECATION_RULES:
if path.startswith(prefix):
if isinstance(path, str) and path.startswith(prefix):
new_path = path.replace(prefix, replacement, 1)
warnings.warn(f"`{path}` class is deprecated, use `{new_path}` instead",
ScrapyDeprecationWarning)

14
scrapy/utils/misc.py
View File

@ -39,10 +39,20 @@ def arg_to_iter(arg):
def load_object(path):
"""Load an object given its absolute object path, and return it.
object can be the import path of a class, function, variable or an
instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
The object can be the import path of a class, function, variable or an
instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.
If ``path`` is not a string, but is a callable object, such as a class or
a function, then return it as is.
"""
if not isinstance(path, str):
if callable(path):
return path
else:
raise TypeError("Unexpected argument type, expected string "
"or object, got: %s" % type(path))
try:
dot = path.rindex('.')
except ValueError:

2
tests/test_crawler.py
View File

@ -142,7 +142,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
def test_spider_manager_verify_interface(self):
settings = Settings({
'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface,
})
with warnings.catch_warnings(record=True) as w:
self.assertRaises(AttributeError, CrawlerRunner, settings)

6
tests/test_downloader_handlers.py
View File

@ -61,7 +61,7 @@ class OffDH:
class LoadTestCase(unittest.TestCase):
def test_enabled_handler(self):
handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'}
handlers = {'scheme': DummyDH}
crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
dh = DownloadHandlers(crawler)
self.assertIn('scheme', dh._schemes)
@ -69,7 +69,7 @@ class LoadTestCase(unittest.TestCase):
self.assertNotIn('scheme', dh._notconfigured)
def test_not_configured_handler(self):
handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'}
handlers = {'scheme': OffDH}
crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
dh = DownloadHandlers(crawler)
self.assertIn('scheme', dh._schemes)
@ -87,7 +87,7 @@ class LoadTestCase(unittest.TestCase):
self.assertIn('scheme', dh._notconfigured)
def test_lazy_handlers(self):
handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'}
handlers = {'scheme': DummyLazyDH}
crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
dh = DownloadHandlers(crawler)
self.assertIn('scheme', dh._schemes)

22
tests/test_feedexport.py
View File

@ -844,7 +844,7 @@ class FeedExportTest(FeedExportTestBase):
self._random_temp_filename(): {'format': 'xml'},
self._random_temp_filename(): {'format': 'csv'},
},
'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
'FEED_STORAGES': {'file': LogOnStoreFileStorage},
'FEED_STORE_EMPTY': False
}
@ -1188,8 +1188,8 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_init_exporters_storages_with_crawler(self):
settings = {
'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'},
'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'},
'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter},
'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage},
'FEEDS': {
self._random_temp_filename(): {'format': 'csv'},
},
@ -1218,7 +1218,7 @@ class FeedExportTest(FeedExportTestBase):
self._random_temp_filename(): {'format': 'xml'},
self._random_temp_filename(): {'format': 'csv'},
},
'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},
'FEED_STORAGES': {'file': DummyBlockingFeedStorage},
}
items = [
{'foo': 'bar1', 'baz': ''},
@ -1239,7 +1239,7 @@ class FeedExportTest(FeedExportTestBase):
self._random_temp_filename(): {'format': 'xml'},
self._random_temp_filename(): {'format': 'csv'},
},
'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},
'FEED_STORAGES': {'file': FailingBlockingFeedStorage},
}
items = [
{'foo': 'bar1', 'baz': ''},
@ -1664,7 +1664,7 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
settings_dict = {
'FEED_URI': 'file:///tmp/foobar',
'FEED_STORAGES': {
'file': 'tests.test_feedexport.StdoutFeedStorageWithoutFeedOptions'
'file': StdoutFeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
@ -1705,7 +1705,7 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
settings_dict = {
'FEED_URI': 'file:///tmp/foobar',
'FEED_STORAGES': {
'file': 'tests.test_feedexport.FileFeedStorageWithoutFeedOptions'
'file': FileFeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
@ -1753,7 +1753,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
settings_dict = {
'FEED_URI': 'file:///tmp/foobar',
'FEED_STORAGES': {
'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptions'
'file': S3FeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
@ -1781,7 +1781,7 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
settings_dict = {
'FEED_URI': 'file:///tmp/foobar',
'FEED_STORAGES': {
'file': 'tests.test_feedexport.S3FeedStorageWithoutFeedOptionsWithFromCrawler'
'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
},
}
crawler = get_crawler(settings_dict=settings_dict)
@ -1830,7 +1830,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
settings_dict = {
'FEED_URI': 'file:///tmp/foobar',
'FEED_STORAGES': {
'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptions'
'file': FTPFeedStorageWithoutFeedOptions
},
}
crawler = get_crawler(settings_dict=settings_dict)
@ -1858,7 +1858,7 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
settings_dict = {
'FEED_URI': 'file:///tmp/foobar',
'FEED_STORAGES': {
'file': 'tests.test_feedexport.FTPFeedStorageWithoutFeedOptionsWithFromCrawler'
'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
},
}
crawler = get_crawler(settings_dict=settings_dict)

2
tests/test_middleware.py
View File

@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager):
@classmethod
def _get_mwlist_from_settings(cls, settings):
return [f'tests.test_middleware.{x}' for x in ['M1', 'MOff', 'M3']]
return [M1, MOff, M3]
def _add_middleware(self, mw):
super()._add_middleware(mw)

32
tests/test_settings/__init__.py
View File

@ -385,6 +385,38 @@ class SettingsTest(unittest.TestCase):
self.assertIn('key', mydict)
self.assertEqual(mydict['key'], 'val')
def test_passing_objects_as_values(self):
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.utils.misc import create_instance
from scrapy.utils.test import get_crawler
class TestPipeline():
def process_item(self, i, s):
return i
settings = Settings({
'ITEM_PIPELINES': {
TestPipeline: 800,
},
'DOWNLOAD_HANDLERS': {
'ftp': FileDownloadHandler,
},
})
self.assertIn('ITEM_PIPELINES', settings.attributes)
mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem()
self.assertEqual(priority, 800)
self.assertEqual(mypipeline, TestPipeline)
self.assertIsInstance(mypipeline(), TestPipeline)
self.assertEqual(mypipeline().process_item('item', None), 'item')
myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp')
self.assertEqual(myhandler, FileDownloadHandler)
myhandler_instance = create_instance(myhandler, None, get_crawler())
self.assertIsInstance(myhandler_instance, FileDownloadHandler)
self.assertTrue(hasattr(myhandler_instance, 'download_request'))
if __name__ == "__main__":
unittest.main()

2
tests/test_spidermiddleware_referer.py
View File

@ -385,7 +385,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
class TestSettingsCustomPolicy(TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
settings = {'REFERRER_POLICY': CustomPythonOrgPolicy}
scenarii = [
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),

15
tests/test_utils_misc/__init__.py
View File

@ -12,11 +12,22 @@ __doctests__ = ['scrapy.utils.misc']
class UtilsMiscTestCase(unittest.TestCase):
def test_load_object(self):
def test_load_object_class(self):
obj = load_object(Field)
self.assertIs(obj, Field)
obj = load_object('scrapy.item.Field')
self.assertIs(obj, Field)
def test_load_object_function(self):
obj = load_object(load_object)
self.assertIs(obj, load_object)
obj = load_object('scrapy.utils.misc.load_object')
assert obj is load_object
self.assertIs(obj, load_object)
def test_load_object_exceptions(self):
self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
self.assertRaises(TypeError, load_object, dict())
def test_walk_modules(self):
mods = walk_modules('tests.test_utils_misc.test_walk_modules')