From 24df5d16029d12b5a292c0d69b48682c4b0aac3f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 5 Oct 2008 07:37:21 +0000 Subject: [PATCH] improved scrapy-admin.py script and default project template --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40300 --- scrapy/trunk/scrapy/bin/scrapy-admin.py | 52 +++++++------------ .../scrapy/conf/project_template/__init__.py | 0 .../scrapy/conf/project_template/items.py | 6 +++ .../conf/project_template/scrapy-ctl.py | 4 ++ .../project_template/scrapy_settings.py} | 52 +++++++++---------- .../conf/project_template/spiders/__init__.py | 1 + scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl | 24 --------- 7 files changed, 57 insertions(+), 82 deletions(-) create mode 100644 scrapy/trunk/scrapy/conf/project_template/__init__.py create mode 100644 scrapy/trunk/scrapy/conf/project_template/items.py create mode 100755 scrapy/trunk/scrapy/conf/project_template/scrapy-ctl.py rename scrapy/trunk/scrapy/{templates/settings.tmpl => conf/project_template/scrapy_settings.py} (53%) create mode 100644 scrapy/trunk/scrapy/conf/project_template/spiders/__init__.py delete mode 100644 scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl diff --git a/scrapy/trunk/scrapy/bin/scrapy-admin.py b/scrapy/trunk/scrapy/bin/scrapy-admin.py index ee3bf17fb..f2dc1d5c3 100755 --- a/scrapy/trunk/scrapy/bin/scrapy-admin.py +++ b/scrapy/trunk/scrapy/bin/scrapy-admin.py @@ -1,13 +1,15 @@ #!/usr/bin/env python -"""Scrapy admin script""" +"""Scrapy admin script is used to create new scrapy projects and similar +tasks""" -import os, stat +import os +import shutil from optparse import OptionParser import scrapy usage = """ -Usage: scrapy-admin.py [options] [command] +scrapy-admin.py [options] [command] Available commands: @@ -16,36 +18,22 @@ Available commands: """ def main(): - parser = OptionParser(usage = usage) + parser = OptionParser(usage=usage) opts, args = parser.parse_args() - if args: - if args[0] == "startproject": - - try: - project_name = args[1] - - except IndexError: - parser.print_help() - else: - os.mkdir(project_name) - os.mknod(os.path.join(project_name, "__init__.py")) - - for subdir in ["spiders", "conf", "commands", "templates"]: - os.mkdir(os.path.join(project_name, subdir)) - os.mknod(os.path.join(project_name, subdir, "__init__.py")) - - settings_template = open(os.path.join(scrapy.__path__[0], "templates", "settings.tmpl"), "r").read() - settings = settings_template.replace("__project_name__", project_name) - open(os.path.join(project_name, "conf", "scrapy_settings.py"), "w").write(settings) - - control_template = open(os.path.join(scrapy.__path__[0], "templates", "scrapy-ctl.tmpl"), "r").read() - control = control_template.replace("__project_name__", project_name) - open(os.path.join(project_name, "scrapy-ctl.py"), "w").write(control) - os.chmod(os.path.join(project_name, "scrapy-ctl.py"), stat.S_IRWXU) - - return - parser.print_help() + if not args: + parser.print_help() + + cmd = args[0] + if cmd == "startproject": + if len(args) >= 2: + project_name = args[1] + project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template") + shutil.copytree(project_tplpath, project_name) + else: + print "scrapy-admin.py: missing project name" + else: + print "scrapy-admin.py: unknown command: %s" % cmd if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/scrapy/trunk/scrapy/conf/project_template/__init__.py b/scrapy/trunk/scrapy/conf/project_template/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/conf/project_template/items.py b/scrapy/trunk/scrapy/conf/project_template/items.py new file mode 100644 index 000000000..d78c4d149 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/project_template/items.py @@ -0,0 +1,6 @@ +# Define here the models for your scraped items + +from scrapy.item import ScrapedItem + +class MyItem(ScrapedItem): + pass diff --git a/scrapy/trunk/scrapy/conf/project_template/scrapy-ctl.py b/scrapy/trunk/scrapy/conf/project_template/scrapy-ctl.py new file mode 100755 index 000000000..e8e56eaf2 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/project_template/scrapy-ctl.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python + +from scrapy.command.cmdline import execute +execute() diff --git a/scrapy/trunk/scrapy/templates/settings.tmpl b/scrapy/trunk/scrapy/conf/project_template/scrapy_settings.py similarity index 53% rename from scrapy/trunk/scrapy/templates/settings.tmpl rename to scrapy/trunk/scrapy/conf/project_template/scrapy_settings.py index 35e3a0e64..8936ae590 100644 --- a/scrapy/trunk/scrapy/templates/settings.tmpl +++ b/scrapy/trunk/scrapy/conf/project_template/scrapy_settings.py @@ -1,20 +1,20 @@ -import __project_name__ +import $project_name # --------------------------------------------------------------------------- -# - Scrapy settings for __project_name__ - +# - Scrapy settings for $project_name - # --------------------------------------------------------------------------- -BOT_NAME = 'scrapy-bot' +BOT_NAME = 'scrapybot' BOT_VERSION = '1.0' DEFAULT_DATA_ENCODING = 'utf-8' -COMMANDS_MODULE = '__project_name__.commands' -COMMANDS_SETTINGS_MODULE = '__project_name__.conf.commands' -SPIDER_MODULES = ['__project_name__.spiders'] -NEWSPIDER_MODULE = '__project_name__.spiders' -TEMPLATES_DIR = '%s/templates' % __project_name__.__path__[0] -ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % __project_name__.__path__[0] +COMMANDS_MODULE = '$project_name.commands' +COMMANDS_SETTINGS_MODULE = '$project_name.conf.commands' +SPIDER_MODULES = ['$project_name.spiders'] +NEWSPIDER_MODULE = '$project_name.spiders' +TEMPLATES_DIR = '%s/templates' % $project_name.__path__[0] +ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % $project_name.__path__[0] DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) DOWNLOAD_TIMEOUT = 600 @@ -44,28 +44,28 @@ EXTENSIONS = ( DOWNLOADER_MIDDLEWARES = ( # Engine side - #'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', - #'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', - #'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', - #'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', - #'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', - #'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', - #'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', - #'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware', - #'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', - #'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', + 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware', + 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware', + 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware', + 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware', + 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware', + 'scrapy.contrib.downloadermiddleware.common.CommonMiddleware', + 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware', + 'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware', + 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug', + 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware', # Downloader side ) SPIDER_MIDDLEWARES = ( # Engine side - #'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', - #'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', - #'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', - #'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', - #'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', - #'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', - #'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware', + 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware', + 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware', + 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware', + 'scrapy.contrib.spidermiddleware.referer.RefererMiddleware', + 'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware', + 'scrapy.contrib.spidermiddleware.depth.DepthMiddleware', + 'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware', # Spider side ) diff --git a/scrapy/trunk/scrapy/conf/project_template/spiders/__init__.py b/scrapy/trunk/scrapy/conf/project_template/spiders/__init__.py new file mode 100644 index 000000000..59d02a55f --- /dev/null +++ b/scrapy/trunk/scrapy/conf/project_template/spiders/__init__.py @@ -0,0 +1 @@ +# Place here all your scrapy spiders diff --git a/scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl b/scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl deleted file mode 100644 index 3eea158d3..000000000 --- a/scrapy/trunk/scrapy/templates/scrapy-ctl.tmpl +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python - -import os, sys - -# hack to avoid using chardet which hits performance very badly -sys.modules['chardet'] = None - -# to avoid (write) permission problems with images, cache, etc -#os.umask(0002) - -SCRAPING_TRUNK = os.environ.get('SCRAPING_TRUNK', os.environ.get('PWD') + '/..') - -try: - import __project_name__ -except: - sys.path.append(SCRAPING_TRUNK) - -try: - import scrapy_settings -except: - sys.path.append('%s/__project_name__/conf' % SCRAPING_TRUNK) - -from scrapy.command.cmdline import execute -execute() \ No newline at end of file