mirror of https://github.com/scrapy/scrapy.git
81 lines
3.0 KiB
Python
81 lines
3.0 KiB
Python
import sys
|
|
import os
|
|
from importlib import import_module
|
|
|
|
from scrapy.utils.spider import iter_spider_classes
|
|
from scrapy.command import ScrapyCommand
|
|
from scrapy.exceptions import UsageError
|
|
from scrapy.utils.conf import arglist_to_dict
|
|
|
|
def _import_file(filepath):
|
|
abspath = os.path.abspath(filepath)
|
|
dirname, file = os.path.split(abspath)
|
|
fname, fext = os.path.splitext(file)
|
|
if fext != '.py':
|
|
raise ValueError("Not a Python source file: %s" % abspath)
|
|
if dirname:
|
|
sys.path = [dirname] + sys.path
|
|
try:
|
|
module = import_module(fname)
|
|
finally:
|
|
if dirname:
|
|
sys.path.pop(0)
|
|
return module
|
|
|
|
class Command(ScrapyCommand):
|
|
|
|
requires_project = False
|
|
|
|
def syntax(self):
|
|
return "[options] <spider_file>"
|
|
|
|
def short_desc(self):
|
|
return "Run a self-contained spider (without creating a project)"
|
|
|
|
def long_desc(self):
|
|
return "Run the spider defined in the given file"
|
|
|
|
def add_options(self, parser):
|
|
ScrapyCommand.add_options(self, parser)
|
|
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
|
|
help="set spider argument (may be repeated)")
|
|
parser.add_option("-o", "--output", metavar="FILE", \
|
|
help="dump scraped items into FILE (use - for stdout)")
|
|
parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \
|
|
help="format to use for dumping items with -o (default: %default)")
|
|
|
|
def process_options(self, args, opts):
|
|
ScrapyCommand.process_options(self, args, opts)
|
|
try:
|
|
opts.spargs = arglist_to_dict(opts.spargs)
|
|
except ValueError:
|
|
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
|
if opts.output:
|
|
if opts.output == '-':
|
|
self.settings.overrides['FEED_URI'] = 'stdout:'
|
|
else:
|
|
self.settings.overrides['FEED_URI'] = opts.output
|
|
valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
|
|
if opts.output_format not in valid_output_formats:
|
|
raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats))
|
|
self.settings.overrides['FEED_FORMAT'] = opts.output_format
|
|
|
|
def run(self, args, opts):
|
|
if len(args) != 1:
|
|
raise UsageError()
|
|
filename = args[0]
|
|
if not os.path.exists(filename):
|
|
raise UsageError("File not found: %s\n" % filename)
|
|
try:
|
|
module = _import_file(filename)
|
|
except (ImportError, ValueError) as e:
|
|
raise UsageError("Unable to load %r: %s\n" % (filename, e))
|
|
spclasses = list(iter_spider_classes(module))
|
|
if not spclasses:
|
|
raise UsageError("No spider found in file: %s\n" % filename)
|
|
spider = spclasses.pop()(**opts.spargs)
|
|
|
|
crawler = self.crawler_process.create_crawler()
|
|
crawler.crawl(spider)
|
|
self.crawler_process.start()
|