mirror of https://github.com/scrapy/scrapy.git
moved scrapy.utils.sqlite to scrapyd.sqlite
--HG-- rename : scrapy/utils/sqlite.py => scrapyd/sqlite.py rename : scrapy/tests/test_utils_sqlite.py => scrapyd/tests/test_sqlite.py
This commit is contained in:
parent
e1aff779da
commit
91b9d89ffd
|
|
@ -40,24 +40,3 @@ def data_path(path):
|
|||
otherwise return the path unmodified
|
||||
"""
|
||||
return path if isabs(path) else join(project_data_dir(), path)
|
||||
|
||||
def sqlite_db(path, nonwritable_fallback=True):
|
||||
"""Get the SQLite database to use. If path is relative, returns the given
|
||||
path inside the project data dir, otherwise returns the path unmodified. If
|
||||
not inside a project returns :memory: to use an in-memory database.
|
||||
|
||||
If nonwritable_fallback is True, and the path is not writable it issues a
|
||||
warning and returns :memory:
|
||||
"""
|
||||
if not inside_project() or path == ':memory:':
|
||||
db = ':memory:'
|
||||
else:
|
||||
try:
|
||||
db = data_path(path)
|
||||
except NotConfigured:
|
||||
db = ':memory:'
|
||||
else:
|
||||
if not is_writable(db) and nonwritable_fallback:
|
||||
warnings.warn("%r is not writable - using in-memory SQLite instead" % db)
|
||||
db = ':memory:'
|
||||
return db
|
||||
|
|
|
|||
|
|
@ -49,9 +49,6 @@ class IPoller(Interface):
|
|||
|
||||
class ISpiderQueue(Interface):
|
||||
|
||||
def from_settings(settings):
|
||||
"""Class method to instantiate from settings"""
|
||||
|
||||
def add(name, **spider_args):
|
||||
"""Add a spider to the queue given its name a some spider arguments.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
from zope.interface import implements
|
||||
|
||||
from scrapyd.interfaces import ISpiderQueue
|
||||
from scrapy.utils.sqlite import JsonSqlitePriorityQueue
|
||||
from scrapy.utils.project import sqlite_db
|
||||
from scrapyd.sqlite import JsonSqlitePriorityQueue
|
||||
|
||||
|
||||
class SqliteSpiderQueue(object):
|
||||
|
|
@ -12,10 +11,6 @@ class SqliteSpiderQueue(object):
|
|||
def __init__(self, database=None, table='spider_queue'):
|
||||
self.q = JsonSqlitePriorityQueue(database, table)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
return cls(sqlite_db(settings['SQLITE_DB']))
|
||||
|
||||
def add(self, name, **spider_args):
|
||||
d = spider_args.copy()
|
||||
d['name'] = name
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ from datetime import datetime
|
|||
from decimal import Decimal
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.sqlite import SqlitePriorityQueue, JsonSqlitePriorityQueue, \
|
||||
from scrapyd.sqlite import SqlitePriorityQueue, JsonSqlitePriorityQueue, \
|
||||
PickleSqlitePriorityQueue, SqliteDict, JsonSqliteDict, PickleSqliteDict
|
||||
|
||||
|
||||
Loading…
Reference in New Issue