first version of scrapy-admin

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40117

scrapy/trunk/scrapy/bin/scrapy-admin.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""Scrapy admin script"""
+
+import os, stat
+from optparse import OptionParser
+
+import scrapy
+
+usage = """
+Usage: scrapy-admin.py [options] [command]
+ 
+Available commands:
+     
+    startproject <project_name>
+      Starts a new project with name 'project_name'
+"""
+
+def main():
+    parser = OptionParser(usage = usage)
+    opts, args = parser.parse_args()
+    
+    if args:
+        if args[0] == "startproject":
+            
+            try:
+                project_name = args[1]
+            except IndexError:
+                parser.print_help()
+            else:
+                os.mkdir(project_name)
+                os.mknod(os.path.join(project_name, "__init__.py"))
+                
+                for subdir in ["spiders", "conf", os.path.join("conf", "sites"), "commands", "templates"]:
+                    os.mkdir(os.path.join(project_name, subdir))
+                    os.mknod(os.path.join(project_name, subdir, "__init__.py"))
+                
+                settings_template = open(os.path.join(scrapy.__path__[0], "templates", "settings.tmpl"), "r").read()
+                settings = settings_template.replace("__project_name__", project_name)
+                open(os.path.join(project_name, "conf", "scrapy_settings.py"), "w").write(settings)
+                
+                control_template = open(os.path.join(scrapy.__path__[0], "templates", "scrapy-ctl.tmpl"), "r").read()
+                control = control_template.replace("__project_name__", project_name)
+                open(os.path.join(project_name, "scrapy-ctl.py"), "w").write(control)
+                os.chmod(os.path.join(project_name, "scrapy-ctl.py"), stat.S_IRWXU)
+                
+            return
+    parser.print_help()
+
+if __name__ == '__main__':
+    main()

scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import os, sys
+
+# hack to avoid using chardet which hits performance very badly
+sys.modules['chardet'] = None
+
+# to avoid (write) permission problems with images, cache, etc
+#os.umask(0002)
+
+SCRAPING_TRUNK = os.environ.get('SCRAPING_TRUNK', os.environ.get('PWD/..')).split(":")
+
+try:
+    import __project_name__
+except:
+    sys.path.extend(SCRAPING_TRUNK)
+
+try:
+    import scrapy_settings
+except:
+    sys.path.extend(['%s/__project_name__/conf' % spath for spath in SCRAPING_TRUNK])
+
+from scrapy.command.cmdline import execute
+execute()

scrapy/trunk/scrapy/templates/settings.tmpl

@@ -0,0 +1,84 @@
+import __project_name__
+
+# ---------------------------------------------------------------------------
+# - Scrapy settings for __project_name__                                             -
+# ---------------------------------------------------------------------------
+
+BOT_NAME = 'scrapy-bot'
+BOT_VERSION = '1.0'
+
+COMMANDS_MODULE = '__project_name__.commands'
+COMMANDS_SETTINGS_MODULE = '__project_name__.conf.commands'
+SPIDER_MODULES = ['__project_name__.spiders']
+NEWSPIDER_MODULE = '__project_name__.spiders'
+TEMPLATES_DIR = '%s/templates' % __project_name__.__path__[0]
+ENABLED_SPIDERS_FILE = '%s/conf/sites/production.list' % __project_name__.__path__[0]
+DEFAULT_ITEM_CLASS = '__project_name__.product.Product'
+USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
+DOWNLOAD_TIMEOUT = 600
+
+SCHEDULER = 'scrapy.core.scheduler.Scheduler'
+SCHEDULER_ORDER = 'BFO'   # available orders: BFO (default), DFO
+
+#CACHE2_DIR = '/tmp/cache2'  # if set, enables HTTP cache
+#CACHE2_IGNORE_MISSING = 0   # ignore requests not in cache
+#CACHE2_SECTORIZE = 1         # sectorize domains to distribute storage among servers
+
+#STATS_ENABLED = 1   # enable stats
+#STATS_CLEANUP = 0   # cleanup domain stats when a domain is closed (saves memory)
+#STATS_DEBUG = 0     # log stats on domain closed
+
+EXTENSIONS = (
+    'scrapy.management.web.WebConsole',
+    'scrapy.management.telnet.TelnetConsole',
+)
+
+DOWNLOADER_MIDDLEWARES = (
+    # Engine side
+    #'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
+    #'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
+    #'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
+    #'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
+    #'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
+    #'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
+    #'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
+    #'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware',
+    #'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
+    #'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
+    # Downloader side
+)
+
+SPIDER_MIDDLEWARES = (
+    # Engine side
+    #'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
+    #'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
+    #'scrapy.contrib.spidermiddleware.referer.CrawlMiddleware',
+    #'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
+    #'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
+    # Spider side
+)
+
+# Item pipelines are usually configured by commands (see conf/commands)
+#ITEM_PIPELINES = (
+#)
+
+#DEPTH_LIMIT = 10  # limit the maximum link depth to follow
+#DEPTH_STATS = 1    # enable depth stats
+
+# Limit URL length. See: http://www.boutell.com/newfaq/misc/urllength.html
+URLLENGTH_LIMIT = 2083
+
+WEBCONSOLE_ENABLED = 1
+WEBCONSOLE_PORT = 8060  # if not set uses a dynamic port
+
+#TELNETCONSOLE_ENABLED = 1
+#TELNETCONSOLE_PORT = 2020  # if not set uses a dynamic port
+
+# global mail sending settings
+#MAIL_HOST = 'localhost'
+#MAIL_FROM = 'scrapybot@localhost'
+
+# scrapy webservice
+WS_ENABLED = 0
+
+SPIDERPROFILER_ENABLED = 0