moved scrapy.utils.sqlite to scrapyd.sqlite

--HG--
rename : scrapy/utils/sqlite.py => scrapyd/sqlite.py
rename : scrapy/tests/test_utils_sqlite.py => scrapyd/tests/test_sqlite.py
This commit is contained in:
Pablo Hoffman 2011-08-27 01:20:57 -03:00
parent e1aff779da
commit 91b9d89ffd
5 changed files with 2 additions and 31 deletions

View File

@ -40,24 +40,3 @@ def data_path(path):
otherwise return the path unmodified
"""
return path if isabs(path) else join(project_data_dir(), path)
def sqlite_db(path, nonwritable_fallback=True):
"""Get the SQLite database to use. If path is relative, returns the given
path inside the project data dir, otherwise returns the path unmodified. If
not inside a project returns :memory: to use an in-memory database.
If nonwritable_fallback is True, and the path is not writable it issues a
warning and returns :memory:
"""
if not inside_project() or path == ':memory:':
db = ':memory:'
else:
try:
db = data_path(path)
except NotConfigured:
db = ':memory:'
else:
if not is_writable(db) and nonwritable_fallback:
warnings.warn("%r is not writable - using in-memory SQLite instead" % db)
db = ':memory:'
return db

View File

@ -49,9 +49,6 @@ class IPoller(Interface):
class ISpiderQueue(Interface):
def from_settings(settings):
"""Class method to instantiate from settings"""
def add(name, **spider_args):
"""Add a spider to the queue given its name a some spider arguments.

View File

@ -1,8 +1,7 @@
from zope.interface import implements
from scrapyd.interfaces import ISpiderQueue
from scrapy.utils.sqlite import JsonSqlitePriorityQueue
from scrapy.utils.project import sqlite_db
from scrapyd.sqlite import JsonSqlitePriorityQueue
class SqliteSpiderQueue(object):
@ -12,10 +11,6 @@ class SqliteSpiderQueue(object):
def __init__(self, database=None, table='spider_queue'):
self.q = JsonSqlitePriorityQueue(database, table)
@classmethod
def from_settings(cls, settings):
return cls(sqlite_db(settings['SQLITE_DB']))
def add(self, name, **spider_args):
d = spider_args.copy()
d['name'] = name

View File

@ -3,7 +3,7 @@ from datetime import datetime
from decimal import Decimal
from scrapy.http import Request
from scrapy.utils.sqlite import SqlitePriorityQueue, JsonSqlitePriorityQueue, \
from scrapyd.sqlite import SqlitePriorityQueue, JsonSqlitePriorityQueue, \
PickleSqlitePriorityQueue, SqliteDict, JsonSqliteDict, PickleSqliteDict