diff --git a/MANIFEST.in b/MANIFEST.in index 4d9441da1..6abbcea5a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,8 +3,6 @@ include AUTHORS include INSTALL include LICENSE include MANIFEST.in -include scrapy/core/downloader/responsetypes/mime.types -include scrapy/xlib/pydispatch/license.txt recursive-include scrapy/templates * recursive-include scrapy/tests/sample_data * recursive-include docs * diff --git a/debian/control b/debian/control index a6a1788c0..8e4097dd9 100644 --- a/debian/control +++ b/debian/control @@ -15,9 +15,11 @@ Description: Python web crawling and scraping framework It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Package: scrapy-service +Package: scrapyd Architecture: all -Depends: scrapy +Depends: scrapy, python-setuptools Description: Scrapy Service - This package provides support for running Scrapy as a system service, - controlled through an upstart script. + The Scrapy service allows you to deploy your Scrapy projects by building + Python eggs of them and uploading them to the Scrapy service using a JSON API + that you can also use for scheduling spider runs. It supports multiple + projects also. diff --git a/debian/scrapy-service.default b/debian/scrapy-service.default deleted file mode 100644 index 59852048d..000000000 --- a/debian/scrapy-service.default +++ /dev/null @@ -1,4 +0,0 @@ -# Defaults for Scrapy service - -export PYTHONPATH=/etc/scrapy -export SCRAPY_SETTINGS_MODULE=service_conf diff --git a/debian/scrapy-service.dirs b/debian/scrapy-service.dirs deleted file mode 100644 index a9eaa09f0..000000000 --- a/debian/scrapy-service.dirs +++ /dev/null @@ -1,2 +0,0 @@ -var/lib/scrapy -var/log/scrapy diff --git a/debian/scrapy-service.install b/debian/scrapy-service.install deleted file mode 100644 index 7ee823edc..000000000 --- a/debian/scrapy-service.install +++ /dev/null @@ -1,2 +0,0 @@ -debian/service_conf.py etc/scrapy -extras/scrapy.tac usr/share/scrapy diff --git a/debian/scrapy-service.prerm b/debian/scrapy-service.prerm deleted file mode 100644 index c0e87b571..000000000 --- a/debian/scrapy-service.prerm +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -set -e - -case "$1" in - remove|upgrade|deconfigure) - rm -f /etc/scrapy/service_conf.pyc - ;; - - failed-upgrade) - ;; - - *) - echo "prerm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -#DEBHELPER# - -exit 0 diff --git a/debian/scrapy-service.upstart b/debian/scrapy-service.upstart deleted file mode 100644 index 8e7d33c6a..000000000 --- a/debian/scrapy-service.upstart +++ /dev/null @@ -1,12 +0,0 @@ -# Scrapy service - -start on runlevel [2345] -stop on runlevel [06] - -script - [ -r /etc/default/scrapy-service ] && . /etc/default/scrapy-service - exec twistd -ny /usr/share/scrapy/scrapy.tac \ - -u scrapy -g nogroup \ - -l /var/log/scrapy/service.log \ - --pidfile /var/run/scrapy/scrapy.pid -end script diff --git a/debian/scrapy.1 b/debian/scrapy.1 index ca6e83ccc..e73ff5459 100644 --- a/debian/scrapy.1 +++ b/debian/scrapy.1 @@ -1,12 +1,12 @@ .TH SCRAPY 1 "October 17, 2009" .SH NAME -scrapy \- Python Scrapy control script +scrapy \- the Scrapy command-line tool .SH SYNOPSIS .B scrapy [\fIcommand\fR] [\fIOPTIONS\fR] ... .SH DESCRIPTION .PP -Scrapy is controlled through the \fBscrapy\fR control script. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options. +Scrapy is controlled through the \fBscrapy\fR command-line tool. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options. .SH OPTIONS .SS fetch\fR [\fIOPTION\fR] \fIURL\fR .TP @@ -50,8 +50,6 @@ Create new project with an initial project template .SS --help, -h Print command help and options -.SS --version -Print Scrapy version and exit .SS --logfile=FILE Log file. if omitted stderr will be used .SS --loglevel=LEVEL, -L LEVEL @@ -68,8 +66,6 @@ Write lsprof profiling stats to FILE Write process ID to FILE .SS --set=SET Set/override setting (may be repeated) -.SS --settings=MODULE -Python path to the Scrapy project settings .SH AUTHOR Scrapy was written by the Scrapy Developers diff --git a/debian/scrapy.install b/debian/scrapy.install index 2007a9d06..8aa6bc81d 100644 --- a/debian/scrapy.install +++ b/debian/scrapy.install @@ -1,2 +1,3 @@ -debian/tmp/usr +usr/lib/python*/*-packages/scrapy +usr/bin extras/scrapy_bash_completion etc/bash_completion.d/ diff --git a/debian/scrapyd.cfg b/debian/scrapyd.cfg new file mode 100644 index 000000000..d4b8ee4fa --- /dev/null +++ b/debian/scrapyd.cfg @@ -0,0 +1,7 @@ +[scrapyd] +http_port = 6800 +debug = off +#max_proc = 1 +eggs_dir = /var/lib/scrapyd/eggs +dbs_dir = /var/lib/scrapyd/dbs +logs_dir = /var/log/scrapyd diff --git a/debian/scrapyd.default b/debian/scrapyd.default new file mode 100644 index 000000000..f71fdb569 --- /dev/null +++ b/debian/scrapyd.default @@ -0,0 +1 @@ +# Defaults for Scrapy service diff --git a/debian/scrapyd.dirs b/debian/scrapyd.dirs new file mode 100644 index 000000000..ef91f28ed --- /dev/null +++ b/debian/scrapyd.dirs @@ -0,0 +1,4 @@ +var/lib/scrapyd +var/lib/scrapyd/eggs +var/lib/scrapyd/dbs +var/log/scrapyd diff --git a/debian/scrapyd.install b/debian/scrapyd.install new file mode 100644 index 000000000..29c82985e --- /dev/null +++ b/debian/scrapyd.install @@ -0,0 +1,3 @@ +usr/lib/python*/*-packages/scrapyd +debian/scrapyd.cfg etc +extras/scrapyd.tac usr/share/scrapyd diff --git a/debian/scrapy-service.lintian-overrides b/debian/scrapyd.lintian-overrides similarity index 86% rename from debian/scrapy-service.lintian-overrides rename to debian/scrapyd.lintian-overrides index 48868e9a7..52f5e4cb7 100644 --- a/debian/scrapy-service.lintian-overrides +++ b/debian/scrapyd.lintian-overrides @@ -1,2 +1,2 @@ new-package-should-close-itp-bug -script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapy-service +script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapyd diff --git a/debian/scrapy-service.postinst b/debian/scrapyd.postinst similarity index 52% rename from debian/scrapy-service.postinst rename to debian/scrapyd.postinst index 9dc708fff..7bd29260d 100644 --- a/debian/scrapy-service.postinst +++ b/debian/scrapyd.postinst @@ -6,16 +6,19 @@ case "$1" in configure) # Create user to run the service as if [ -z "`id -u scrapy 2> /dev/null`" ]; then - adduser --system --home /var/lib/scrapy --gecos "scrapy" \ + adduser --system --home /var/lib/scrapyd --gecos "scrapy" \ --no-create-home --disabled-password \ --quiet scrapy || true fi - if [ ! -d /var/run/scrapy ]; then - mkdir /var/run/scrapy - chown scrapy:nogroup /var/run/scrapy + if [ ! -d /var/run/scrapyd ]; then + mkdir /var/run/scrapyd + chown scrapy:nogroup /var/run/scrapyd fi - chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy + chown scrapy:nogroup /var/log/scrapyd /var/run/scrapyd \ + /var/lib/scrapyd /var/lib/scrapyd/eggs /var/lib/scrapyd/dbs + + update-python-modules -p # so upstart restart uses the new code ;; abort-upgrade|abort-remove|abort-deconfigure) diff --git a/debian/scrapy-service.postrm b/debian/scrapyd.postrm similarity index 91% rename from debian/scrapy-service.postrm rename to debian/scrapyd.postrm index b16d0c2b6..92d6fcf08 100644 --- a/debian/scrapy-service.postrm +++ b/debian/scrapyd.postrm @@ -8,7 +8,7 @@ if [ purge = "$1" ]; then else echo >&2 "not removing scrapy system account because deluser command was not found" fi - rm -rf /var/run/scrapy + rm -rf /var/run/scrapyd fi #DEBHELPER# diff --git a/debian/scrapyd.upstart b/debian/scrapyd.upstart new file mode 100644 index 000000000..cc6578c54 --- /dev/null +++ b/debian/scrapyd.upstart @@ -0,0 +1,13 @@ +# Scrapy service + +start on runlevel [2345] +stop on runlevel [06] + +script + [ -r /etc/default/scrapyd ] && . /etc/default/scrapyd + logdir=/var/log/scrapyd + exec twistd -ny /usr/share/scrapyd/scrapyd.tac \ + -u scrapy -g nogroup \ + --pidfile /var/run/scrapyd/scrapy.pid \ + -l $logdir/scrapyd.log >$logdir/scrapyd.out 2>$logdir/scrapyd.err +end script diff --git a/debian/service_conf.py b/debian/service_conf.py deleted file mode 100644 index 3eb3af931..000000000 --- a/debian/service_conf.py +++ /dev/null @@ -1,13 +0,0 @@ -# Scrapy service configuration - -# Directory where logs will be stored (one per crawler) -LOG_DIR = '/var/log/scrapy' - -# A dict containing the Scrapy projects that will be run by this service -# * Keys are paths to project settings modules -# * Values are number of processes that should be started for each project, or -# zero if you want to use the number of cores available. -PROJECTS = { -# 'mybot.settings': 0, -} - diff --git a/docs/index.rst b/docs/index.rst index 9c4f9ef1a..e50ffa15b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -125,6 +125,7 @@ Solving specific problems topics/leaks topics/images topics/ubuntu + topics/scrapyd :doc:`faq` Get answers to most frequently asked questions. @@ -144,6 +145,9 @@ Solving specific problems :doc:`topics/ubuntu` Install latest Scrapy packages easily on Ubuntu +:doc:`topics/scrapyd` + Deploying your Scrapy project in production. + .. _extending-scrapy: Extending Scrapy diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst new file mode 100644 index 000000000..c47bab0f7 --- /dev/null +++ b/docs/topics/scrapyd.rst @@ -0,0 +1,379 @@ +.. _topics-scrapyd: + +======================== +Scrapy Service (scrapyd) +======================== + +Scrapy comes with a built-in service, called "Scrapyd", which allows you to +deploy (aka. upload) your projects and control their spiders using a JSON web +service. + +Projects and versions +===================== + +Scrapyd can manage multiple projects and each project can have multiple +versions uploaded, but only the latest one will be used for launching new +spiders. + +A common (and useful) convention to use for the version name is the revision +number of the version control tool you're using to track your Scrapy project +code. For example: ``r23``. The versions are not compared alphabetically but +using a smarter algorithm (the same `distutils`_ uses) so ``r10`` compares +greater to ``r9``, for example. + +How Scrapyd works +================= + +Scrapyd is an application (typically run as a daemon) that continually polls +for projects that need to run (ie. those projects that have spiders enqueued). + +When a project needs to run, a Scrapy process is started for that project using +something similar to the typical ``scrapy crawl``` command, and it continues to +run until it finishes processing all spiders form the spider queue. + +Scrapyd also runs multiple processes in parallel, allocating them in a fixed +number of "slots", which defaults to the number of cpu processors available in +the system, but this can be changed with the ``max_proc`` option. It also +starts as many processes as possible to handle the load. + +In addition to dispatching and managing processes, Scrapyd provides a +:ref:`JSON web service ` to upload new project versions +(as eggs) and schedule spiders. This feature is optional and can be disabled if +you want to implement your own custom Scrapyd. The components are pluggable and +can be changed, if you're familiar with the `Twisted Application Framework`_ +which Scrapyd is implemented in. + +Starting Scrapyd +================ + +Scrapyd is implemented using the standard `Twisted Application Framework`_. To +start the service, use the ``extras/scrapyd.tac`` file provided in the Scrapy +distribution, like this:: + + twistd -ny extras/scrapyd.tac + +That should get your Scrapyd started. + +Installing Scrapyd +================== + +How to deploy Scrapyd on your servers depends on the platform your're using. +Scrapy comes with Ubuntu packages for Scrapyd ready for deploying it as a +system service, to ease the installation and administration, but you can create +packages for other distribution or operating systems (including Windows). If +you do so, and want to contribute them, send a message to +scrapy-developers@googlegroups.com and say hi. The community will appreciate +it. + +.. _topics-scrapyd-ubuntu: + +Installing Scrapyd in Ubuntu +---------------------------- + +When deploying Scrapyd, it's very useful to have a version already packaged for +your system. For this reason, Scrapyd comes with Ubuntu packages ready to use +in your Ubuntu servers. + +So, if you plan to deploy Scrapyd on a Ubuntu server, just add the Ubuntu +repositories as described in :ref:`topics-ubuntu` and then run:: + + aptitude install scrapyd + +This will install Scrapyd in your Ubuntu server creating a ``scrapy`` user +which Scrapyd will run as. It will also create some directories and files that +are listed below: + +/etc/scrapyd.cfg +~~~~~~~~~~~~~~~~ + +Scrapyd configuration file. See :ref:`topics-scrapyd-config`. + +/var/log/scrapyd/scrapyd.log +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Scrapyd main log file. + +/var/log/scrapyd/scrapyd.out +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The standard output captured from Scrapyd process and any +sub-process spawned from it. + +/var/log/scrapyd/scrapyd.err +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The standard error captured from Scrapyd and any sub-process spawned +from it. Remember to check this file if you're having problems, as the errors +may not get logged to the ``scrapyd.log`` file. + +/var/log/scrapyd/slotN.log +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The log files of Scrapy processes started from Scrapyd, one per slot. These are +standard :ref:`Scrapy log files `. + +/var/lib/scrapyd/ +~~~~~~~~~~~~~~~~~ + +Directory used to store data files (uploaded eggs and spider queues). + +.. _topics-scrapyd-config: + +Scrapyd Configuration file +========================== + +The Scrapyd configuration file supports the following options: + +http_port +--------- + +The TCP port where the HTTP JSON API will listen. Defaults to ``6800``. + +max_proc +-------- + +The maximum number of concurrent Scrapy process that will be started. If unset +or ``0`` it will use the number of cpus available in the system. + +debug +----- + +Whether debug mode is enabled. Defaults to ``off``. When debug mode is enabled +the full Python traceback will be returned (as plain text responses) when there +is an error processing a JSON API call. + +eggs_dir +-------- + +The directory where the project eggs will be stored. + +dbs_dir +------- + +The directory where the project databases will be stored (this includes the +spider queues). + +logs_dir +-------- + +The directory where the Scrapy processes logs (``slotN.log``) will be stored. + +Example configuration file +-------------------------- + +Here is an example configuration file with all the defaults: + +.. literalinclude:: ../../scrapyd/default_scrapyd.cfg + +Eggifying your project +====================== + +In order to upload your project to Scrapyd, you must first build a `Python +egg`_ of it. This is called "eggifying" your project. You'll need to install +`setuptools`_ for this. + +To eggify your project add a `setup.py`_ file to the root directory of your +project (where the ``scrapy.cfg`` resides) with the following contents:: + + #!/usr/bin/env python + + from setuptools import setup, find_packages + + setup( + name = 'myproject', + version = '1.0', + packages = find_packages(), + entry_points = {'scrapy': ['settings = myproject.settings']}, + ) + +And then the run the following command:: + + python setup.py bdist_egg + +This will generate an egg file and leave it in the ``dist`` directory, for +example:: + + dist/myproject-1.0-py2.6.egg + +Egg caveats +----------- + +There are some things to keep in mind when building eggs of your Scrapy +project: + +* make sure no local development settings are included in the egg when you + build it. The ``find_packages`` function may be picking up your custom + settings. In most cases you want to upload the egg with the default project + settings. + +* you shouldn't use ``__file__`` in your project code as it doesn't play well + with eggs. Consider using `pkgutil.get_data()`_ instead. + +* be careful when writing to disk in your project (in any spider, extension or + middleware) as Scrapyd will probably run with a different user which may not + have write access to certain directories. If you can, avoid writing to disk + and always use `tempfile`_ for temporary files. + +Uploading your project +====================== + +In these examples we'll be using `curl`_ for the web service interaction +examples, but you can use any command or library that speaks HTTP. + +Once you've built the egg, you can upload your project to Scrapyd, like this:: + + $ curl http://localhost:6800/addversion.json -F project=myproject -F version=r23 -F egg=@dist/myproject-1.0-py2.6.egg + {"status": "ok", "spiders": ["spider1", "spider2", "spider3"]} + +You'll see that the JSON response contains the spiders found in your project. + +Scheduling a spider run +======================= + +To schedule a spider run:: + + $ curl http://localhost:6800/schedule.json -d project=myproject -d spider=spider=spider2 + {"status": "ok"} + +For more resources see: :ref:`topics-scrapyd-jsonapi` for more available resources. + +.. _topics-scrapyd-jsonapi: + +JSON API reference +================== + +The following section describes the available resources in Scrapyd JSON API. + +addversion.json +--------------- + +Add a version to a project, creating the project if it doesn't exist. + +* Supported Request Methods: ``POST`` +* Parameters: + * ``project`` (string, required) - the project name + * ``version`` (string, required) - the project version + * ``egg`` (file, required) - a Python egg containing the project's code + +Example request:: + + $ curl http://localhost:6800/addversion.json -F project=myproject -F version=r23 -F egg=@myproject.egg + +Example reponse:: + + {"status": "ok", "spiders": ["spider1", "spider2", "spider3"]} + +schedule.json +------------- + +Schedule a spider run. + +* Supported Request Methods: ``POST`` +* Parameters: + * ``project`` (string, required) - the project name + * ``spider`` (string, required) - the spider name + * any other parameter is passed as spider argument + +Example request:: + + $ curl http://localhost:6800/schedule.json -d project=myproject -d spider=somespider + +Example response:: + + {"status": "ok"} + +listprojects.json +----------------- + +Get the list of projects uploaded to this Scrapy server. + +* Supported Request Methods: ``GET`` +* Parameters: none + +Example request:: + + $ curl http://localhost:6800/listprojects.json + +Example response:: + + {"status": "ok", "projects": ["myproject", "otherproject"]} + +listversions.json +----------------- + +Get the list of versions available for some project. The versions are returned +in order, the last one is the currently used version. + +* Supported Request Methods: ``GET`` +* Parameters: + * ``project`` (string, required) - the project name + +Example request:: + + $ curl http://localhost:6800/listversions.json?project=myproject + +Example response:: + + {"status": "ok", "versions": ["r99", "r156"]} + +listspiders.json +---------------- + +Get the list of spiders available in the last version of some project. + +* Supported Request Methods: ``GET`` +* Parameters: + * ``project`` (string, required) - the project name + +Example request:: + + $ curl http://localhost:6800/listspiders.json?project=myproject + +Example response:: + + {"status": "ok", "spiders": ["spider1", "spider2", "spider3"]} + +delversion.json +--------------- + +Delete a project version. If there are no more versions available for a given +project, that project will be deleted too. + +* Supported Request Methods: ``POST`` +* Parameters: + * ``project`` (string, required) - the project name + * ``version`` (string, required) - the project version + +Example request:: + + $ curl http://localhost:6800/delversion.json -d project=myproject -d version=r99 + +Example response:: + + {"status": "ok"} + +delproject.json +--------------- + +Delete a project and all its uploaded versions. + +* Supported Request Methods: ``POST`` +* Parameters: + * ``project`` (string, required) - the project name + +Example request:: + + $ curl http://localhost:6800/delproject.json?project=myproject + +Example response:: + + {"status": "ok"} + +.. _Python egg: http://peak.telecommunity.com/DevCenter/PythonEggs +.. _setup.py: http://docs.python.org/distutils/setupscript.html +.. _curl: http://en.wikipedia.org/wiki/CURL +.. _setuptools: http://pypi.python.org/pypi/setuptools +.. _pkgutil.get_data(): http://docs.python.org/library/pkgutil.html#pkgutil.get_data +.. _tempfile: http://docs.python.org/library/tempfile.html +.. _Twisted Application Framework: http://twistedmatrix.com/documents/current/core/howto/application.html +.. _distutils: http://docs.python.org/library/distutils.html diff --git a/extras/scrapy.tac b/extras/scrapy.tac deleted file mode 100644 index 35e25c56f..000000000 --- a/extras/scrapy.tac +++ /dev/null @@ -1,5 +0,0 @@ -from twisted.application.service import Application -from scrapy.service import ScrapyService - -application = Application("Scrapy") -ScrapyService().setServiceParent(application) diff --git a/extras/scrapyd.tac b/extras/scrapyd.tac new file mode 100644 index 000000000..bdf548d31 --- /dev/null +++ b/extras/scrapyd.tac @@ -0,0 +1,2 @@ +from scrapyd.app import get_application +application = get_application() diff --git a/scrapy/__init__.py b/scrapy/__init__.py index fd1628ef4..030b6c7f9 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -14,6 +14,11 @@ if sys.version_info < (2,5): # ignore noisy twisted deprecation warnings warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') +# prevents noisy (and innocent) dropin.cache errors when loading spiders from +# egg files using the old Spider Manager. TODO: Remove for Scrapy 0.11 +from twisted.python.zippath import ZipPath +ZipPath.setContent = lambda x, y: None + # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches diff --git a/scrapy/service.py b/scrapy/service.py deleted file mode 100644 index ce49d6f41..000000000 --- a/scrapy/service.py +++ /dev/null @@ -1,88 +0,0 @@ -import sys, os - -from twisted.python import log -from twisted.internet import reactor, defer, protocol, error -from twisted.application.service import Service - -from scrapy.utils.py26 import cpu_count -from scrapy.conf import settings - - -class ScrapyService(Service): - - def startService(self): - reactor.callWhenRunning(self.spawn_processes) - - def spawn_processes(self): - for settings_module, count in settings['PROJECTS'].items(): - for i in range(count or cpu_count()): - self.spawn_process(settings_module, i) - - def spawn_process(self, settings_module, position): - args = [sys.executable, '-m', 'scrapy.cmdline', 'runserver'] - env = os.environ.copy() - botname = self.get_bot_name(settings_module) - logfile = self.get_log_file(settings_module, position) - env['SCRAPY_SETTINGS_MODULE'] = settings_module - env['SCRAPY_LOG_FILE'] = logfile - pp = ScrapyProcessProtocol(botname, settings_module, logfile) - pp.deferred.addCallback(lambda _: reactor.callLater(5, \ - self.spawn_process, settings_module, position)) - reactor.spawnProcess(pp, sys.executable, args=args, env=env) - - def get_log_file(self, settings_module, position): - botname = self.get_bot_name(settings_module) - basename = "%s-%s" % (botname, position) if position else botname - return os.path.join(settings['LOG_DIR'], "%s.log" % basename) - - def get_bot_name(self, settings_module): - mod = __import__(settings_module, {}, {}, [''], -1) - return mod.BOT_NAME - -class ScrapyProcessProtocol(protocol.ProcessProtocol): - - TAIL_LINES = 100 - - def __init__(self, botname, settings_module, logfile): - self.botname = botname - self.settings_module = settings_module - self.logfile = logfile - self.pid = None - self.deferred = defer.Deferred() - self.outdata = '' - self.errdata = '' - - def outReceived(self, data): - self.outdata = self._tail(self.outdata + data) - - def errReceived(self, data): - self.errdata = self._tail(self.errdata + data) - - def connectionMade(self): - self.pid = self.transport.pid - log.msg("Crawler %r started: pid=%r settings=%r log=%r" % \ - (self.botname, self.pid, self.settings_module, self.logfile)) - - def processEnded(self, status): - if isinstance(status.value, error.ProcessDone): - msg = "Crawler %r finished: pid=%r settings=%r log=%r" % \ - (self.botname, self.pid, self.settings_module, self.logfile) - else: - msg = "Crawler %r died: exitstatus=%r pid=%r settings=%r log=%r" % \ - (self.botname, status.value.exitCode, self.pid, self.settings_module, \ - self.logfile) - self._log_ended(msg) - self.deferred.callback(self) - - def _log_ended(self, msg): - tolog = [msg] - if self.outdata: - tolog += [">>> stdout (last %d lines) <<<" % self.TAIL_LINES] - tolog += [self.outdata] - if self.errdata: - tolog += [">>> stderr (last %d lines) <<<" % self.TAIL_LINES] - tolog += [self.errdata] - log.msg(os.linesep.join(tolog)) - - def _tail(self, data, lines=TAIL_LINES): - return os.linesep.join(data.split(os.linesep)[-lines:]) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 0ec409903..ee9c838fd 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -49,9 +49,13 @@ def init_env(project='default', set_syspath=True): def get_config(use_closest=True): """Get Scrapy config file as a SafeConfigParser""" - sources = [os.path.expanduser('~/.scrapy.cfg'), '/etc/scrapy.cfg'] - if use_closest: - sources.insert(0, closest_scrapy_cfg()) + sources = get_sources(use_closest) cfg = SafeConfigParser() cfg.read(sources) return cfg + +def get_sources(use_closest=True): + sources = [os.path.expanduser('~/.scrapy.cfg'), '/etc/scrapy.cfg'] + if use_closest: + sources.insert(0, closest_scrapy_cfg()) + return sources diff --git a/scrapy/utils/txweb.py b/scrapy/utils/txweb.py new file mode 100644 index 000000000..b2124d8e1 --- /dev/null +++ b/scrapy/utils/txweb.py @@ -0,0 +1,17 @@ +from twisted.web import resource + +from scrapy.utils.py26 import json + +class JsonResource(resource.Resource): + + json_encoder = json.JSONEncoder() + + def render(self, txrequest): + r = resource.Resource.render(self, txrequest) + return self.render_object(r, txrequest) + + def render_object(self, obj, txrequest): + r = self.json_encoder.encode(obj) + "\n" + txrequest.setHeader('Content-Type', 'application/json') + txrequest.setHeader('Content-Length', len(r)) + return r diff --git a/scrapy/webservice.py b/scrapy/webservice.py index 6547747fc..7559656b4 100644 --- a/scrapy/webservice.py +++ b/scrapy/webservice.py @@ -5,7 +5,7 @@ See docs/topics/ws.rst """ from twisted.internet import reactor -from twisted.web import server, resource, error +from twisted.web import server, error from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured @@ -13,23 +13,15 @@ from scrapy import signals from scrapy.utils.jsonrpc import jsonrpc_server_call from scrapy.utils.serialize import ScrapyJSONEncoder, ScrapyJSONDecoder from scrapy.utils.misc import load_object +from scrapy.utils.txweb import JsonResource as JsonResource_ from scrapy.utils.conf import build_component_list from scrapy.conf import settings -class JsonResource(resource.Resource): +class JsonResource(JsonResource_): - ws_name = None json_encoder = ScrapyJSONEncoder() - def render(self, txrequest): - r = resource.Resource.render(self, txrequest) - r = self.json_encoder.encode(r) - txrequest.setHeader('Content-Type', 'application/json') - txrequest.setHeader('Content-Length', len(r)) - return r - - class JsonRpcResource(JsonResource): json_decoder = ScrapyJSONDecoder() diff --git a/scrapyd/__init__.py b/scrapyd/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapyd/app.py b/scrapyd/app.py new file mode 100644 index 000000000..abddfadc1 --- /dev/null +++ b/scrapyd/app.py @@ -0,0 +1,40 @@ +import sys, os + +from twisted.internet import reactor +from twisted.application.service import Application, Service +from twisted.application.internet import TimerService, TCPServer +from twisted.web import server + +from .interfaces import IEggStorage, IPoller, ISpiderScheduler, IEnvironment +from .launcher import Launcher +from .eggstorage import FilesystemEggStorage +from .scheduler import SpiderScheduler +from .poller import QueuePoller +from .environ import Environment +from .webservice import Root +from .config import Config + +def get_application(): + app = Application("Scrapy") + config = Config() + http_port = config.getint('http_port', 6800) + + poller = QueuePoller(config) + eggstorage = FilesystemEggStorage(config) + scheduler = SpiderScheduler(config) + environment = Environment(config) + + app.setComponent(IPoller, poller) + app.setComponent(IEggStorage, eggstorage) + app.setComponent(ISpiderScheduler, scheduler) + app.setComponent(IEnvironment, environment) + + launcher = Launcher(config, app) + timer = TimerService(5, poller.poll) + webservice = TCPServer(http_port, server.Site(Root(config, app))) + + launcher.setServiceParent(app) + timer.setServiceParent(app) + webservice.setServiceParent(app) + + return app diff --git a/scrapyd/config.py b/scrapyd/config.py new file mode 100644 index 000000000..de0b62d19 --- /dev/null +++ b/scrapyd/config.py @@ -0,0 +1,39 @@ +import pkgutil +from cStringIO import StringIO +from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError + +from scrapy.utils.conf import get_sources + +class Config(object): + """A ConfigParser wrapper to support defaults when calling instance + methods, and also tied to a single section""" + + SOURCES = ['scrapyd.cfg', '/etc/scrapyd.cfg'] + SECTION = 'scrapyd' + + def __init__(self): + sources = self.SOURCES + get_sources() + default_config = pkgutil.get_data(__package__, 'default_scrapyd.cfg') + self.cp = SafeConfigParser() + self.cp.readfp(StringIO(default_config)) + self.cp.read(sources) + + def _getany(self, method, option, default): + try: + return method(self.SECTION, option) + except (NoSectionError, NoOptionError): + if default is not None: + return default + raise + + def get(self, option, default=None): + return self._getany(self.cp.get, option, default) + + def getint(self, option, default=None): + return self._getany(self.cp.getint, option, default) + + def getfloat(self, option, default=None): + return self._getany(self.cp.getfloat, option, default) + + def getboolean(self, option, default=None): + return self._getany(self.cp.getboolean, option, default) diff --git a/scrapyd/default_scrapyd.cfg b/scrapyd/default_scrapyd.cfg new file mode 100644 index 000000000..ba34a7ab7 --- /dev/null +++ b/scrapyd/default_scrapyd.cfg @@ -0,0 +1,8 @@ +[scrapyd] +eggs_dir = eggs +logs_dir = logs +dbs_dir = dbs +max_proc = 0 +http_port = 6800 +debug = off +egg_runner = scrapyd.eggrunner diff --git a/scrapyd/eggrunner.py b/scrapyd/eggrunner.py new file mode 100644 index 000000000..846ee36cd --- /dev/null +++ b/scrapyd/eggrunner.py @@ -0,0 +1,27 @@ +""" +This module can be used to run a Scrapy project contained in an egg file + +To see all spiders in a project: + + python -m scrapyd.eggrunner myproject.egg list + +To crawl a spider: + + python -m scrapyd.eggrunner myproject.egg crawl somespider +""" + +import sys + +from scrapyd.eggutils import activate_egg + +def main(eggpath, args): + """Run scrapy for the settings module name passed""" + activate_egg(eggpath) + from scrapy.cmdline import execute + execute(['scrapy'] + list(args)) + +if __name__ == '__main__': + if len(sys.argv) < 2: + print "usage: %s [scrapy_command args ...]" % sys.argv[0] + sys.exit(1) + main(sys.argv[1], sys.argv[2:]) diff --git a/scrapyd/eggstorage.py b/scrapyd/eggstorage.py new file mode 100644 index 000000000..956b02409 --- /dev/null +++ b/scrapyd/eggstorage.py @@ -0,0 +1,49 @@ +from __future__ import with_statement + +from glob import glob +from os import path, makedirs, remove +from shutil import copyfileobj, rmtree +from distutils.version import LooseVersion + +from zope.interface import implements +from twisted.application.service import Service + +from .interfaces import IEggStorage + +class FilesystemEggStorage(Service): + + implements(IEggStorage) + + def __init__(self, config): + self.basedir = config.get('eggs_dir', 'eggs') + + def put(self, eggfile, project, version): + eggpath = self._eggpath(project, version) + eggdir = path.dirname(eggpath) + if not path.exists(eggdir): + makedirs(eggdir) + with open(eggpath, 'wb') as f: + copyfileobj(eggfile, f) + + def get(self, project, version=None): + if version is None: + version = self.list(project)[-1] + return version, open(self._eggpath(project, version), 'rb') + + def list(self, project): + eggdir = path.join(self.basedir, project) + versions = [path.splitext(path.basename(x))[0] \ + for x in glob("%s/*.egg" % eggdir)] + return sorted(versions, key=LooseVersion) + + def delete(self, project, version=None): + if version is None: + rmtree(path.join(self.basedir, project)) + else: + remove(self._eggpath(project, version)) + if not self.list(project): # remove project if no versions left + self.delete(project) + + def _eggpath(self, project, version): + x = path.join(self.basedir, project, "%s.egg" % version) + return x diff --git a/scrapyd/eggutils.py b/scrapyd/eggutils.py new file mode 100644 index 000000000..95ab2aca5 --- /dev/null +++ b/scrapyd/eggutils.py @@ -0,0 +1,32 @@ +import os, sys, shutil, pkg_resources +from subprocess import Popen, PIPE +from tempfile import NamedTemporaryFile, mkdtemp + +def get_spider_list_from_eggfile(eggfile, project): + # FIXME: we use a temporary directory here to avoid permissions problems + # when running as system service, as "scrapy list" command tries to write + # the scrapy.db sqlite database in current directory + tmpdir = mkdtemp() + try: + with NamedTemporaryFile(suffix='.egg', dir=tmpdir) as f: + shutil.copyfileobj(eggfile, f) + f.flush() + eggfile.seek(0) + pargs = [sys.executable, '-m', 'scrapyd.eggrunner', f.name, 'list'] + env = os.environ.copy() + env['SCRAPY_PROJECT'] = project + proc = Popen(pargs, stdout=PIPE, cwd=tmpdir, env=env) + out = proc.communicate()[0] + return out.splitlines() + finally: + shutil.rmtree(tmpdir) + +def activate_egg(eggpath): + """Activate a Scrapy egg file. This is meant to be used from egg runners + to activate a Scrapy egg file. Don't use it from other code as it may + leave unwanted side effects. + """ + d = pkg_resources.find_distributions(eggpath).next() + d.activate() + settings_module = d.get_entry_info('scrapy', 'settings').module_name + os.environ['SCRAPY_SETTINGS_MODULE'] = settings_module diff --git a/scrapyd/environ.py b/scrapyd/environ.py new file mode 100644 index 000000000..0cd184687 --- /dev/null +++ b/scrapyd/environ.py @@ -0,0 +1,24 @@ +import os + +from zope.interface import implements + +from .interfaces import IEnvironment + +class Environment(object): + + implements(IEnvironment) + + def __init__(self, config): + self.dbs_dir = config.get('dbs_dir', 'dbs') + self.logs_dir = config.get('logs_dir', 'logs') + + def get_environment(self, message, slot): + project = message['project'] + env = os.environ.copy() + env['SCRAPY_PROJECT'] = project + dbpath = os.path.join(self.dbs_dir, '%s.db' % project) + env['SCRAPY_SQLITE_DB'] = dbpath + logpath = os.path.join(self.logs_dir, 'slot%s.log' % slot) + env['SCRAPY_LOG_FILE'] = logpath + return env + diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py new file mode 100644 index 000000000..ae33ec75f --- /dev/null +++ b/scrapyd/interfaces.py @@ -0,0 +1,69 @@ +from zope.interface import Interface + +class IEggStorage(Interface): + """A component that handles storing and retrieving eggs""" + + def put(eggfile, project, version): + """Store the egg (passed in the file object) under the given project and + version""" + + def get(project, version=None): + """Return a tuple (version, file) with the the egg for the specified + project and version. If version is None, the latest version is + returned.""" + + def list(self, project): + """Return the list of versions which have eggs stored (for the given + project) in order (the latest version is the currently used).""" + + def delete(self, project, version=None): + """Delete the egg stored for the given project and version. If should + also delete the project if no versions are left""" + + +class IPoller(Interface): + """A component that polls for projects that need to run""" + + def poll(): + """Called periodically to poll for projects""" + + def next(): + """Return the next message. + + It should return a Deferred which will get fired when there is a new + project that needs to run, or already fired if there was a project + waiting to run already. + + The message is a dict containing (at least) the name of the project to + be run in the 'project' key. This message will be passed later to + IEnvironment.get_environment(). + """ + + def update_projects(): + """Called when projects may have changed, to refresh the available + projects""" + + +class ISpiderScheduler(Interface): + """A component to schedule spider runs""" + + def schedule(project, spider_name, **spider_args): + """Schedule a spider for the given project""" + + def list_projects(): + """Return the list of available projects""" + + def update_projects(): + """Called when projects may have changed, to refresh the available + projects""" + + +class IEnvironment(Interface): + """A component to generate the environment of crawler processes""" + + def get_environment(message, slot): + """Return the environment variables to use for running the process. + + `message` is the message received from the IPoller.next() + `slot` is the Launcher slot where the process will be running. + """ diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py new file mode 100644 index 000000000..a2fda7d16 --- /dev/null +++ b/scrapyd/launcher.py @@ -0,0 +1,81 @@ +import sys, os +from shutil import copyfileobj +from tempfile import mkstemp + +from twisted.internet import reactor, defer, protocol, error +from twisted.application.service import Service +from twisted.python import log + +from scrapy.utils.py26 import cpu_count +from .interfaces import IPoller, IEggStorage, IEnvironment + +class Launcher(Service): + + def __init__(self, config, app): + self.max_proc = config.getint('max_proc', 0) or cpu_count() + self.egg_runner = config.get('egg_runner', 'scrapyd.eggrunner') + self.app = app + + def startService(self): + for slot in range(self.max_proc): + self._wait_for_project(slot) + log.msg("Launcher started: max_proc=%r, egg_runner=%r" % \ + (self.max_proc, self.egg_runner), system="Launcher") + + def _wait_for_project(self, slot): + poller = self.app.getComponent(IPoller) + poller.next().addCallback(self._spawn_process, slot) + + def _get_eggpath(self, project): + eggstorage = self.app.getComponent(IEggStorage) + version, eggf = eggstorage.get(project) + prefix = '%s-%s-' % (project, version) + fd, eggpath = mkstemp(prefix=prefix, suffix='.egg') + lf = os.fdopen(fd, 'wb') + copyfileobj(eggf, lf) + lf.close() + return eggpath + + def _spawn_process(self, message, slot): + project = message['project'] + eggpath = self._get_eggpath(project) + args = [sys.executable, '-m', self.egg_runner, eggpath, 'crawl'] + e = self.app.getComponent(IEnvironment) + env = e.get_environment(message, slot) + pp = ScrapyProcessProtocol(eggpath, slot) + pp.deferred.addBoth(self._process_finished, eggpath, slot) + reactor.spawnProcess(pp, sys.executable, args=args, env=env) + + def _process_finished(self, _, eggpath, slot): + os.remove(eggpath) + self._wait_for_project(slot) + + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + + def __init__(self, eggfile, slot): + self.eggfile = eggfile + self.slot = slot + self.pid = None + self.deferred = defer.Deferred() + + def outReceived(self, data): + log.msg(data.rstrip(), system="Launcher,%d/stdout" % self.pid) + + def errReceived(self, data): + log.msg(data.rstrip(), system="Launcher,%d/stderr" % self.pid) + + def connectionMade(self): + self.pid = self.transport.pid + self.log("Process started: ") + + def processEnded(self, status): + if isinstance(status.value, error.ProcessDone): + self.log("Process finished: ") + else: + self.log("Process died: exitstatus=%r " % status.value.exitCode) + self.deferred.callback(self) + + def log(self, msg): + msg += "slot=%r pid=%r egg=%r" % (self.slot, self.pid, self.eggfile) + log.msg(msg, system="Launcher") diff --git a/scrapyd/poller.py b/scrapyd/poller.py new file mode 100644 index 000000000..37a5c6922 --- /dev/null +++ b/scrapyd/poller.py @@ -0,0 +1,31 @@ +from zope.interface import implements +from twisted.internet.defer import DeferredQueue + +from .utils import get_spider_queues +from .interfaces import IPoller + +class QueuePoller(object): + + implements(IPoller) + + def __init__(self, config): + self.eggs_dir = config.get('eggs_dir', 'eggs') + self.dbs_dir = config.get('dbs_dir', 'dbs') + self.update_projects() + self.dq = DeferredQueue(size=1) + + def poll(self): + if self.dq.pending: + return + for p, q in self.queues.iteritems(): + if q.count(): + return self.dq.put(self._message(p)) + + def next(self): + return self.dq.get() + + def update_projects(self): + self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir) + + def _message(self, project): + return {'project': str(project)} diff --git a/scrapyd/scheduler.py b/scrapyd/scheduler.py new file mode 100644 index 000000000..f96380d72 --- /dev/null +++ b/scrapyd/scheduler.py @@ -0,0 +1,23 @@ +from zope.interface import implements + +from .interfaces import ISpiderScheduler +from .utils import get_spider_queues + +class SpiderScheduler(object): + + implements(ISpiderScheduler) + + def __init__(self, config): + self.eggs_dir = config.get('eggs_dir', 'eggs') + self.dbs_dir = config.get('dbs_dir', 'dbs') + self.update_projects() + + def schedule(self, project, spider_name, **spider_args): + q = self.queues[project] + q.add(spider_name, **spider_args) + + def list_projects(self): + return self.queues.keys() + + def update_projects(self): + self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir) diff --git a/scrapyd/utils.py b/scrapyd/utils.py new file mode 100644 index 000000000..c8b29df31 --- /dev/null +++ b/scrapyd/utils.py @@ -0,0 +1,12 @@ +import os +import pkg_resources + +from scrapy.spiderqueue import SqliteSpiderQueue + +def get_spider_queues(eggsdir, dbsdir): + """Return a dict of Spider Quees keyed by project name""" + d = {} + for project in os.listdir(eggsdir): + dbpath = os.path.join(dbsdir, '%s.db' % project) + d[project] = SqliteSpiderQueue(dbpath) + return d diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py new file mode 100644 index 000000000..570aa70c3 --- /dev/null +++ b/scrapyd/webservice.py @@ -0,0 +1,111 @@ +import cgi +import traceback +from cStringIO import StringIO + +from twisted.web.resource import Resource + +from scrapy.utils.txweb import JsonResource +from .interfaces import IPoller, IEggStorage, ISpiderScheduler +from .eggutils import get_spider_list_from_eggfile + +class WsResource(JsonResource): + + def __init__(self, root): + JsonResource.__init__(self) + self.root = root + + def render(self, txrequest): + try: + return JsonResource.render(self, txrequest) + except Exception, e: + if self.root.debug: + return traceback.format_exc() + r = {"status": "error", "message": str(e)} + return self.render_object(r, txrequest) + +class Schedule(WsResource): + + def render_POST(self, txrequest): + args = dict((k, v[0]) for k, v in txrequest.args.items()) + project = args.pop('project') + spider = args.pop('spider') + sched = self.root.app.getComponent(ISpiderScheduler) + sched.schedule(project, spider, **args) + return {"status": "ok"} + +class AddVersion(WsResource): + + def render_POST(self, txrequest): + ct = txrequest.requestHeaders.getRawHeaders('Content-Type')[0] + boundary = ct.split('boundary=', 1)[1] + d = cgi.parse_multipart(txrequest.content, {'boundary': boundary}) + project = d['project'][0] + version = d['version'][0] + eggf = StringIO(d['egg'][0]) + spiders = get_spider_list_from_eggfile(eggf, project) + eggstorage = self.root.app.getComponent(IEggStorage) + eggstorage.put(eggf, project, version) + self.root.update_projects() + return {"status": "ok", "spiders": spiders} + +class ListProjects(WsResource): + + def render_GET(self, txrequest): + projects = self.root.app.getComponent(ISpiderScheduler).list_projects() + return {"status": "ok", "projects": projects} + +class ListVersions(WsResource): + + def render_GET(self, txrequest): + project = txrequest.args['project'][0] + eggstorage = self.root.app.getComponent(IEggStorage) + versions = eggstorage.list(project) + return {"status": "ok", "versions": versions} + +class ListSpiders(WsResource): + + def render_GET(self, txrequest): + project = txrequest.args['project'][0] + eggstorage = self.root.app.getComponent(IEggStorage) + _, eggf = eggstorage.get(project) + spiders = get_spider_list_from_eggfile(eggf, project) + return {"status": "ok", "spiders": spiders} + +class DeleteProject(WsResource): + + def render_POST(self, txrequest): + project = txrequest.args['project'][0] + self._delete_version(project) + return {"status": "ok"} + + def _delete_version(self, project, version=None): + eggstorage = self.root.app.getComponent(IEggStorage) + eggstorage.delete(project, version) + self.root.update_projects() + +class DeleteVersion(DeleteProject): + + def render_POST(self, txrequest): + project = txrequest.args['project'][0] + version = txrequest.args['version'][0] + self._delete_version(project, version) + return {"status": "ok"} + +class Root(Resource): + + def __init__(self, config, app): + Resource.__init__(self) + self.debug = config.getboolean('debug', False) + self.app = app + self.putChild('schedule.json', Schedule(self)) + self.putChild('addversion.json', AddVersion(self)) + self.putChild('listprojects.json', ListProjects(self)) + self.putChild('listversions.json', ListVersions(self)) + self.putChild('listspiders.json', ListSpiders(self)) + self.putChild('delproject.json', DeleteProject(self)) + self.putChild('delversion.json', DeleteVersion(self)) + self.update_projects() + + def update_projects(self): + self.app.getComponent(IPoller).update_projects() + self.app.getComponent(ISpiderScheduler).update_projects() diff --git a/setup.py b/setup.py index 9c2c3a96e..899136679 100644 --- a/setup.py +++ b/setup.py @@ -51,22 +51,22 @@ packages, data_files = [], [] root_dir = os.path.dirname(__file__) if root_dir != '': os.chdir(root_dir) -scrapy_dir = 'scrapy' def is_not_module(filename): return os.path.splitext(f)[1] not in ['.py', '.pyc', '.pyo'] -for dirpath, dirnames, filenames in os.walk(scrapy_dir): - # Ignore dirnames that start with '.' - for i, dirname in enumerate(dirnames): - if dirname.startswith('.'): del dirnames[i] - if '__init__.py' in filenames: - packages.append('.'.join(fullsplit(dirpath))) - data = [f for f in filenames if is_not_module(f)] - if data: - data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]]) - elif filenames: - data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]]) +for scrapy_dir in ['scrapy', 'scrapyd']: + for dirpath, dirnames, filenames in os.walk(scrapy_dir): + # Ignore dirnames that start with '.' + for i, dirname in enumerate(dirnames): + if dirname.startswith('.'): del dirnames[i] + if '__init__.py' in filenames: + packages.append('.'.join(fullsplit(dirpath))) + data = [f for f in filenames if is_not_module(f)] + if data: + data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]]) + elif filenames: + data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]]) # Small hack for working with bdist_wininst. # See http://mail.python.org/pipermail/distutils-sig/2004-August/004134.html