From b800fdcb4de9e8eb8176be1f47dec92aee0f120d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 4 Sep 2010 03:49:46 -0300 Subject: [PATCH] SqliteSpiderQueue: failback to in-memory SQLite if database cannot be opened (typically due to missing write permissions) --- scrapy/spiderqueue.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/spiderqueue.py b/scrapy/spiderqueue.py index 03b84fe91..409a85181 100644 --- a/scrapy/spiderqueue.py +++ b/scrapy/spiderqueue.py @@ -1,5 +1,8 @@ +import sqlite3 + from zope.interface import implements +from scrapy import log from scrapy.interfaces import ISpiderQueue from scrapy.utils.sqlite import JsonSqlitePriorityQueue @@ -9,7 +12,12 @@ class SqliteSpiderQueue(object): implements(ISpiderQueue) def __init__(self, database=None, table='spider_queue'): - self.q = JsonSqlitePriorityQueue(database, table) + try: + self.q = JsonSqlitePriorityQueue(database, table) + except sqlite3.Error, e: + self.q = JsonSqlitePriorityQueue(':memory:', table) + log.msg("Cannot open SQLite %r - using in-memory spider queue " \ + "instead. Error was: %r" % (database, str(e)), log.WARNING) @classmethod def from_settings(cls, settings):