From a2602df90b6954b6e764ab2b49d897fc2cdb0457 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Wed, 21 Nov 2012 10:44:35 -0400 Subject: [PATCH 1/3] httpcache: allow to import submodules within packages as HTTPCACHE_DBM_MODULE setting. --- scrapy/contrib/httpcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 0891c6355..49e82dc8c 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -163,7 +163,8 @@ class DbmCacheStorage(object): def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE']) + self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], + {}, {}, ['']) self.db = None def open_spider(self, spider): From 6ed44a3a67e6e11063a53be9c0a8684cf2f688b2 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Fri, 1 Feb 2013 12:41:52 -0400 Subject: [PATCH 2/3] httpcache: added tests for custom dbm module. And removed the break in the line to improve readability as there is no strict 80-chars line width convention. --- scrapy/contrib/httpcache.py | 3 +-- scrapy/tests/mocks/__init__.py | 0 scrapy/tests/mocks/dummydbm.py | 23 +++++++++++++++++++ .../test_downloadermiddleware_httpcache.py | 14 +++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 scrapy/tests/mocks/__init__.py create mode 100644 scrapy/tests/mocks/dummydbm.py diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 49e82dc8c..be5af1604 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -163,8 +163,7 @@ class DbmCacheStorage(object): def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], - {}, {}, ['']) + self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], {}, {}, ['']) self.db = None def open_spider(self, spider): diff --git a/scrapy/tests/mocks/__init__.py b/scrapy/tests/mocks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/mocks/dummydbm.py b/scrapy/tests/mocks/dummydbm.py new file mode 100644 index 000000000..36c2e8637 --- /dev/null +++ b/scrapy/tests/mocks/dummydbm.py @@ -0,0 +1,23 @@ +"""DBM-like dummy module""" +import collections + + +class DummyDB(dict): + """Provide dummy DBM-like interface.""" + def close(self): + pass + + +class error(Exception): + pass + + +_DATABASES = collections.defaultdict(DummyDB) + +def open(file, flag='r', mode=0666): + """Open or create a dummy database compatible. + + Arguments `flag` and `mode` are ignored. + """ + # return same instance for same file argument + return _DATABASES[file] diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 8944ded0a..af4d4bc22 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -116,6 +116,20 @@ class DbmStorageTest(DefaultStorageTest): storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' +class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): + + dbm_module = 'scrapy.tests.mocks.dummydbm' + + def _get_settings(self, **new_settings): + new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) + return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings) + + def test_custom_dbm_module_loaded(self): + # make sure our dbm module has been loaded + with self._storage() as storage: + self.assertEqual(storage.dbmodule.__name__, self.dbm_module) + + class FilesystemStorageTest(DefaultStorageTest): storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage' From d00e6cf1d7e7c2d8bb6c9822f6094e164a8478e2 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Fri, 1 Feb 2013 12:52:59 -0400 Subject: [PATCH 3/3] tests: dummydbm.error should be KeyError. --- scrapy/tests/mocks/dummydbm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/tests/mocks/dummydbm.py b/scrapy/tests/mocks/dummydbm.py index 36c2e8637..6818708ac 100644 --- a/scrapy/tests/mocks/dummydbm.py +++ b/scrapy/tests/mocks/dummydbm.py @@ -8,8 +8,7 @@ class DummyDB(dict): pass -class error(Exception): - pass +error = KeyError _DATABASES = collections.defaultdict(DummyDB)