diff --git a/docs/faq.rst b/docs/faq.rst index ae8e2c519..c2cb0a986 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -113,3 +113,11 @@ the `Scrapy Recipes`_ page. .. _Community Spiders: http://dev.scrapy.org/wiki/CommunitySpiders .. _Scrapy Recipes: http://dev.scrapy.org/wiki/ScrapyRecipes +Can I run a spider without creating a project? +---------------------------------------------- + +Yes. You can use the ``runspider`` command. For example, if you have a spider +written in a ``my_spider.py`` file you can run it with:: + + scrapy-ctl.py runspider my_spider.py + diff --git a/scrapy/command/commands/runspider.py b/scrapy/command/commands/runspider.py new file mode 100644 index 000000000..e18f13cdd --- /dev/null +++ b/scrapy/command/commands/runspider.py @@ -0,0 +1,57 @@ +import sys +import os + +from scrapy.xlib.pydispatch import dispatcher +from scrapy.contrib.exporter import XmlItemExporter +from scrapy.command import ScrapyCommand +from scrapy.core.manager import scrapymanager +from scrapy.core import signals + +def _import_file(filepath): + abspath = os.path.abspath(filepath) + dirname, file = os.path.split(abspath) + fname, fext = os.path.splitext(file) + if fext != '.py': + raise ValueError("Only Python files supported: %s" % abspath) + if dirname: + sys.path = [dirname] + sys.path + try: + module = __import__(fname, {}, {}, ['']) + finally: + if dirname: + sys.path.pop(0) + return module + + +class Command(ScrapyCommand): + + requires_project = False + + def syntax(self): + return "[options] " + + def short_desc(self): + return "Run a spider" + + def long_desc(self): + return "Run the spider defined in the given file. The file must be a " \ + "Python module which defines a SPIDER variable with a instance of " \ + "the spider to run." + + def add_options(self, parser): + super(Command, self).add_options(parser) + parser.add_option("--output", dest="output", metavar="FILE", + help="store scraped items to FILE in XML format") + + def run(self, args, opts): + if len(args) != 1: + return False + if opts.output: + file = open(opts.output, 'w+b') + exporter = XmlItemExporter(file) + dispatcher.connect(exporter.export_item, signal=signals.item_passed) + exporter.start_exporting() + module = _import_file(args[0]) + scrapymanager.runonce(module.SPIDER) + if opts.output: + exporter.finish_exporting()