From 8e18ecd5ce5a4d7896095c8198b9045439a86d26 Mon Sep 17 00:00:00 2001 From: olveyra Date: Fri, 25 Jul 2008 19:11:01 +0000 Subject: [PATCH] first version of scrapy-admin --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40117 --- scrapy/trunk/scrapy/bin/scrapy-admin.py | 50 +++++++++++ scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl | 24 ++++++ scrapy/trunk/scrapy/templates/settings.tmpl | 84 +++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 scrapy/trunk/scrapy/bin/scrapy-admin.py create mode 100644 scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl create mode 100644 scrapy/trunk/scrapy/templates/settings.tmpl diff --git a/scrapy/trunk/scrapy/bin/scrapy-admin.py b/scrapy/trunk/scrapy/bin/scrapy-admin.py new file mode 100644 index 000000000..5e5735079 --- /dev/null +++ b/scrapy/trunk/scrapy/bin/scrapy-admin.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +"""Scrapy admin script""" + +import os, stat +from optparse import OptionParser + +import scrapy + +usage = """ +Usage: scrapy-admin.py [options] [command] + +Available commands: + + startproject + Starts a new project with name 'project_name' +""" + +def main(): + parser = OptionParser(usage = usage) + opts, args = parser.parse_args() + + if args: + if args[0] == "startproject": + + try: + project_name = args[1] + except IndexError: + parser.print_help() + else: + os.mkdir(project_name) + os.mknod(os.path.join(project_name, "__init__.py")) + + for subdir in ["spiders", "conf", os.path.join("conf", "sites"), "commands", "templates"]: + os.mkdir(os.path.join(project_name, subdir)) + os.mknod(os.path.join(project_name, subdir, "__init__.py")) + + settings_template = open(os.path.join(scrapy.__path__[0], "templates", "settings.tmpl"), "r").read() + settings = settings_template.replace("__project_name__", project_name) + open(os.path.join(project_name, "conf", "scrapy_settings.py"), "w").write(settings) + + control_template = open(os.path.join(scrapy.__path__[0], "templates", "scrapy-ctl.tmpl"), "r").read() + control = control_template.replace("__project_name__", project_name) + open(os.path.join(project_name, "scrapy-ctl.py"), "w").write(control) + os.chmod(os.path.join(project_name, "scrapy-ctl.py"), stat.S_IRWXU) + + return + parser.print_help() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl b/scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl new file mode 100644 index 000000000..605ed892d --- /dev/null +++ b/scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import os, sys + +# hack to avoid using chardet which hits performance very badly +sys.modules['chardet'] = None + +# to avoid (write) permission problems with images, cache, etc +#os.umask(0002) + +SCRAPING_TRUNK = os.environ.get('SCRAPING_TRUNK', os.environ.get('PWD/..')).split(":") + +try: + import __project_name__ +except: + sys.path.extend(SCRAPING_TRUNK) + +try: + import scrapy_settings +except: + sys.path.extend(['%s/__project_name__/conf' % spath for spath in SCRAPING_TRUNK]) + +from scrapy.command.cmdline import execute +execute() \ No newline at end of file diff --git a/scrapy/trunk/scrapy/templates/settings.tmpl b/scrapy/trunk/scrapy/templates/settings.tmpl new file mode 100644 index 000000000..60c2ff624 --- /dev/null +++ b/scrapy/trunk/scrapy/templates/settings.tmpl @@ -0,0 +1,84 @@ +import __project_name__ + +# --------------------------------------------------------------------------- +# - Scrapy settings for __project_name__ - +# --------------------------------------------------------------------------- + +BOT_NAME = 'scrapy-bot' +BOT_VERSION = '1.0' + +COMMANDS_MODULE = '__project_name__.commands' +COMMANDS_SETTINGS_MODULE = '__project_name__.conf.commands' +SPIDER_MODULES = ['__project_name__.spiders'] +NEWSPIDER_MODULE = '__project_name__.spiders' +TEMPLATES_DIR = '%s/templates' % __project_name__.__path__[0] +ENABLED_SPIDERS_FILE = '%s/conf/sites/production.list' % __project_name__.__path__[0] +DEFAULT_ITEM_CLASS = '__project_name__.product.Product' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) +DOWNLOAD_TIMEOUT = 600 + +SCHEDULER = 'scrapy.core.scheduler.Scheduler' +SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO + +#CACHE2_DIR = '/tmp/cache2' # if set, enables HTTP cache +#CACHE2_IGNORE_MISSING = 0 # ignore requests not in cache +#CACHE2_SECTORIZE = 1 # sectorize domains to distribute storage among servers + +#STATS_ENABLED = 1 # enable stats +#STATS_CLEANUP = 0 # cleanup domain stats when a domain is closed (saves memory) +#STATS_DEBUG = 0 # log stats on domain closed + +EXTENSIONS = ( + 'scrapy.management.web.WebConsole', + 'scrapy.management.telnet.TelnetConsole', +) + +DOWNLOADER_MIDDLEWARES = ( + # Engine side + #'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', + #'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', + #'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', + #'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', + #'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', + #'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', + #'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', + #'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware', + #'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', + #'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', + # Downloader side +) + +SPIDER_MIDDLEWARES = ( + # Engine side + #'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', + #'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', + #'scrapy.contrib.spidermiddleware.referer.CrawlMiddleware', + #'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', + #'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', + # Spider side +) + +# Item pipelines are usually configured by commands (see conf/commands) +#ITEM_PIPELINES = ( +#) + +#DEPTH_LIMIT = 10 # limit the maximum link depth to follow +#DEPTH_STATS = 1 # enable depth stats + +# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html +URLLENGTH_LIMIT = 2083 + +WEBCONSOLE_ENABLED = 1 +WEBCONSOLE_PORT = 8060 # if not set uses a dynamic port + +#TELNETCONSOLE_ENABLED = 1 +#TELNETCONSOLE_PORT = 2020 # if not set uses a dynamic port + +# global mail sending settings +#MAIL_HOST = 'localhost' +#MAIL_FROM = 'scrapybot@localhost' + +# scrapy webservice +WS_ENABLED = 0 + +SPIDERPROFILER_ENABLED = 0 \ No newline at end of file