diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8f50b45a9..c14d0d488 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -13,6 +13,44 @@ just call "commands", or "Scrapy commands". The Scrapy tool provides several commands, for multiple purposes, and each one accepts a different set of arguments and options. +.. _topics-project-structure: + +Default structure of Scrapy projects +==================================== + +Before delving into the command-line tool and its sub-commands, let's first +understand the directory structure of a Scrapy project. + +Even thought it can be modified, all Scrapy projects have the same file +structure by default, similar to this:: + + scrapy.cfg + myproject/ + __init__.py + items.py + pipelines.py + settings.py + spiders/ + __init__.py + spider1.py + spider2.py + ... + .scrapy/ + scrapy.db + +The directory where the ``scrapy.cfg`` file resides is known as the *project +root directory*. That file contains the name of the python module that defines +the project settings. Here is an example:: + + [settings] + default = myproject.settings + +By default, Scrapy projects use a SQLite_ database to store persistent runtime +data of the project, such as the spider queue (the list of spiders that are +scheduled to run). By default, this SQLite database is stored in the *project +data directory* which, by default, is the ``.scrapy`` directory inside the +project root directory mentioned above. + Using the ``scrapy`` tool ========================= @@ -444,3 +482,5 @@ commands for your Scrapy project. Example:: COMMANDS_MODULE = 'mybot.commands' + +.. _SQLite: http://en.wikipedia.org/wiki/SQLite diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0f2f99811..2a1d9aeb2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -878,6 +878,18 @@ Example:: SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev'] +.. setting:: SQLITE_DB + +SQLITE_DB +--------- + +Default: ``'scrapy.db'`` + +The location of the project SQLite database, used for storing the spider queue +and other persistent data of the project. If a relative path is given, is taken +relative to the project data dir. For more info see: +:ref:`topics-project-structure`. + .. setting:: STATS_CLASS STATS_CLASS diff --git a/scrapy/contrib/spidercontext.py b/scrapy/contrib/spidercontext.py index 5c369a172..29a309183 100644 --- a/scrapy/contrib/spidercontext.py +++ b/scrapy/contrib/spidercontext.py @@ -1,12 +1,11 @@ -import sqlite3 - from zope.interface import Interface, implements from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.sqlite import JsonSqliteDict -from scrapy import log, signals +from scrapy.utils.project import sqlite_db +from scrapy import signals class ISpiderContextStorage(Interface): @@ -24,16 +23,11 @@ class SqliteSpiderContextStorage(object): sqlite_dict_class = JsonSqliteDict def __init__(self, database=None, table='contexts'): - try: - self.d = self.sqlite_dict_class(database, table) - except sqlite3.Error, e: - self.d = self.sqlite_dict_class(':memory:', table) - log.msg("Cannot open SQLite %r - using in-memory context storage " \ - "instead. Error was: %r" % (database, str(e)), log.WARNING) + self.d = self.sqlite_dict_class(database, table) @classmethod def from_settings(cls, settings): - return cls(settings['SQLITE_DB']) + return cls(sqlite_db(settings['SQLITE_DB'])) def get(self, spider): if spider.name in self.d: diff --git a/scrapy/spiderqueue.py b/scrapy/spiderqueue.py index 409a85181..e19d10fc9 100644 --- a/scrapy/spiderqueue.py +++ b/scrapy/spiderqueue.py @@ -1,10 +1,8 @@ -import sqlite3 - from zope.interface import implements -from scrapy import log from scrapy.interfaces import ISpiderQueue from scrapy.utils.sqlite import JsonSqlitePriorityQueue +from scrapy.utils.project import sqlite_db class SqliteSpiderQueue(object): @@ -12,16 +10,11 @@ class SqliteSpiderQueue(object): implements(ISpiderQueue) def __init__(self, database=None, table='spider_queue'): - try: - self.q = JsonSqlitePriorityQueue(database, table) - except sqlite3.Error, e: - self.q = JsonSqlitePriorityQueue(':memory:', table) - log.msg("Cannot open SQLite %r - using in-memory spider queue " \ - "instead. Error was: %r" % (database, str(e)), log.WARNING) + self.q = JsonSqlitePriorityQueue(database, table) @classmethod def from_settings(cls, settings): - return cls(settings['SQLITE_DB']) + return cls(sqlite_db(settings['SQLITE_DB'])) def add(self, name, **spider_args): d = spider_args.copy() diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py new file mode 100644 index 000000000..596d8380f --- /dev/null +++ b/scrapy/utils/project.py @@ -0,0 +1,48 @@ +from os.path import join, dirname, abspath, isabs, exists +from os import makedirs +import warnings + +from scrapy.utils.conf import closest_scrapy_cfg, get_config +from scrapy.utils.python import is_writable + +DATADIR_CFG_SECTION = 'datadir' + +def inside_project(): + return bool(closest_scrapy_cfg()) + +def project_data_dir(project='default'): + """Return the current project data dir, creating it if it doesn't exist""" + assert inside_project(), "Not inside project" + scrapy_cfg = closest_scrapy_cfg() + d = abspath(join(dirname(scrapy_cfg), '.scrapy')) + cfg = get_config() + if cfg.has_option(DATADIR_CFG_SECTION, project): + d = cfg.get(DATADIR_CFG_SECTION, project) + if not exists(d): + makedirs(d) + return d + +def expand_data_path(path): + """If path is relative, return the given path inside the project data dir, + otherwise return the path unmodified + """ + if isabs(path): + return path + return join(project_data_dir(), path) + +def sqlite_db(path, nonwritable_fallback=True): + """Get the SQLite database to use. If path is relative, returns the given + path inside the project data dir, otherwise returns the path unmodified. If + not inside a project returns :memory: to use an in-memory database. + + If nonwritable_fallback is True, and the path is not writable it issues a + warning and returns :memory: + """ + if not inside_project() or path == ':memory:': + db = ':memory:' + else: + db = expand_data_path(path) + if not is_writable(db) and nonwritable_fallback: + warnings.warn("%r is not writable - using in-memory SQLite instead" % db) + db = ':memory:' + return db diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 24e4476e6..486aece73 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -5,6 +5,7 @@ It also contains functions (or functionality) which is in Python versions higher than 2.5 which is the lowest version supported by Scrapy. """ +import os import re import inspect import weakref @@ -205,3 +206,12 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): v = v.encode(encoding) if isinstance(v, unicode) else v d[k] = v return d + +def is_writable(path): + """Return True if the given path can be written (if it exists) or created + (if it doesn't exist) + """ + if os.path.exists(path): + return os.access(path, os.W_OK) + else: + return os.access(os.path.dirname(path), os.W_OK)