mirror of https://github.com/scrapy/scrapy.git
improved scrapy-admin.py script and default project template
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40300
This commit is contained in:
Pablo Hoffman
parent
8dc46c16ce
commit
24df5d1602
|
|
@ -1,13 +1,15 @@
|
|||
#!/usr/bin/env python
|
||||
"""Scrapy admin script"""
|
||||
"""Scrapy admin script is used to create new scrapy projects and similar
|
||||
tasks"""
|
||||
|
||||
import os, stat
|
||||
import os
|
||||
import shutil
|
||||
from optparse import OptionParser
|
||||
|
||||
import scrapy
|
||||
|
||||
usage = """
|
||||
Usage: scrapy-admin.py [options] [command]
|
||||
scrapy-admin.py [options] [command]
|
||||
|
||||
Available commands:
|
||||
|
||||
|
|
@ -16,36 +18,22 @@ Available commands:
|
|||
"""
|
||||
|
||||
def main():
|
||||
parser = OptionParser(usage = usage)
|
||||
parser = OptionParser(usage=usage)
|
||||
opts, args = parser.parse_args()
|
||||
|
||||
if args:
|
||||
if args[0] == "startproject":
|
||||
|
||||
try:
|
||||
project_name = args[1]
|
||||
|
||||
except IndexError:
|
||||
parser.print_help()
|
||||
else:
|
||||
os.mkdir(project_name)
|
||||
os.mknod(os.path.join(project_name, "__init__.py"))
|
||||
|
||||
for subdir in ["spiders", "conf", "commands", "templates"]:
|
||||
os.mkdir(os.path.join(project_name, subdir))
|
||||
os.mknod(os.path.join(project_name, subdir, "__init__.py"))
|
||||
|
||||
settings_template = open(os.path.join(scrapy.__path__[0], "templates", "settings.tmpl"), "r").read()
|
||||
settings = settings_template.replace("__project_name__", project_name)
|
||||
open(os.path.join(project_name, "conf", "scrapy_settings.py"), "w").write(settings)
|
||||
|
||||
control_template = open(os.path.join(scrapy.__path__[0], "templates", "scrapy-ctl.tmpl"), "r").read()
|
||||
control = control_template.replace("__project_name__", project_name)
|
||||
open(os.path.join(project_name, "scrapy-ctl.py"), "w").write(control)
|
||||
os.chmod(os.path.join(project_name, "scrapy-ctl.py"), stat.S_IRWXU)
|
||||
|
||||
return
|
||||
parser.print_help()
|
||||
if not args:
|
||||
parser.print_help()
|
||||
|
||||
cmd = args[0]
|
||||
if cmd == "startproject":
|
||||
if len(args) >= 2:
|
||||
project_name = args[1]
|
||||
project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template")
|
||||
shutil.copytree(project_tplpath, project_name)
|
||||
else:
|
||||
print "scrapy-admin.py: missing project name"
|
||||
else:
|
||||
print "scrapy-admin.py: unknown command: %s" % cmd
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
# Define here the models for your scraped items
|
||||
|
||||
from scrapy.item import ScrapedItem
|
||||
|
||||
class MyItem(ScrapedItem):
|
||||
pass
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
execute()
|
||||
|
|
@ -1,20 +1,20 @@
|
|||
import __project_name__
|
||||
import $project_name
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# - Scrapy settings for __project_name__ -
|
||||
# - Scrapy settings for $project_name -
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BOT_NAME = 'scrapy-bot'
|
||||
BOT_NAME = 'scrapybot'
|
||||
BOT_VERSION = '1.0'
|
||||
|
||||
DEFAULT_DATA_ENCODING = 'utf-8'
|
||||
|
||||
COMMANDS_MODULE = '__project_name__.commands'
|
||||
COMMANDS_SETTINGS_MODULE = '__project_name__.conf.commands'
|
||||
SPIDER_MODULES = ['__project_name__.spiders']
|
||||
NEWSPIDER_MODULE = '__project_name__.spiders'
|
||||
TEMPLATES_DIR = '%s/templates' % __project_name__.__path__[0]
|
||||
ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % __project_name__.__path__[0]
|
||||
COMMANDS_MODULE = '$project_name.commands'
|
||||
COMMANDS_SETTINGS_MODULE = '$project_name.conf.commands'
|
||||
SPIDER_MODULES = ['$project_name.spiders']
|
||||
NEWSPIDER_MODULE = '$project_name.spiders'
|
||||
TEMPLATES_DIR = '%s/templates' % $project_name.__path__[0]
|
||||
ENABLED_SPIDERS_FILE = '%s/conf/enabled_spiders.list' % $project_name.__path__[0]
|
||||
DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem'
|
||||
USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
||||
DOWNLOAD_TIMEOUT = 600
|
||||
|
|
@ -44,28 +44,28 @@ EXTENSIONS = (
|
|||
|
||||
DOWNLOADER_MIDDLEWARES = (
|
||||
# Engine side
|
||||
#'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware',
|
||||
#'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
|
||||
#'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.common.CommonMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.compression.CompressionMiddleware',
|
||||
'scrapy.contrib.downloadermiddleware.debug.CrawlDebug',
|
||||
'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware',
|
||||
# Downloader side
|
||||
)
|
||||
|
||||
SPIDER_MIDDLEWARES = (
|
||||
# Engine side
|
||||
#'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
|
||||
#'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
|
||||
#'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
|
||||
#'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
|
||||
#'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
|
||||
#'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
|
||||
#'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.referer.RefererMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.urllength.UrlLengthMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.depth.DepthMiddleware',
|
||||
'scrapy.contrib.spidermiddleware.urlfilter.UrlFilterMiddleware',
|
||||
# Spider side
|
||||
)
|
||||
|
||||
|
|
@ -0,0 +1 @@
|
|||
# Place here all your scrapy spiders
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os, sys
|
||||
|
||||
# hack to avoid using chardet which hits performance very badly
|
||||
sys.modules['chardet'] = None
|
||||
|
||||
# to avoid (write) permission problems with images, cache, etc
|
||||
#os.umask(0002)
|
||||
|
||||
SCRAPING_TRUNK = os.environ.get('SCRAPING_TRUNK', os.environ.get('PWD') + '/..')
|
||||
|
||||
try:
|
||||
import __project_name__
|
||||
except:
|
||||
sys.path.append(SCRAPING_TRUNK)
|
||||
|
||||
try:
|
||||
import scrapy_settings
|
||||
except:
|
||||
sys.path.append('%s/__project_name__/conf' % SCRAPING_TRUNK)
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
execute()
|
||||
Loading…
Reference in New Issue