mirror of https://github.com/scrapy/scrapy.git
Added new Scrapy service with support for:
* multiple projects * uploading scrapy projects as Python eggs * scheduling spiders using a JSON API Documentation is added along with the code. Closes #218. --HG-- rename : debian/scrapy-service.default => debian/scrapyd.default rename : debian/scrapy-service.dirs => debian/scrapyd.dirs rename : debian/scrapy-service.install => debian/scrapyd.install rename : debian/scrapy-service.lintian-overrides => debian/scrapyd.lintian-overrides rename : debian/scrapy-service.postinst => debian/scrapyd.postinst rename : debian/scrapy-service.postrm => debian/scrapyd.postrm rename : debian/scrapy-service.upstart => debian/scrapyd.upstart rename : extras/scrapy.tac => extras/scrapyd.tac
This commit is contained in:
Pablo Hoffman
parent
1b766877f1
commit
37e9c5d78e
|
|
@ -3,8 +3,6 @@ include AUTHORS
|
|||
include INSTALL
|
||||
include LICENSE
|
||||
include MANIFEST.in
|
||||
include scrapy/core/downloader/responsetypes/mime.types
|
||||
include scrapy/xlib/pydispatch/license.txt
|
||||
recursive-include scrapy/templates *
|
||||
recursive-include scrapy/tests/sample_data *
|
||||
recursive-include docs *
|
||||
|
|
|
|||
|
|
@ -15,9 +15,11 @@ Description: Python web crawling and scraping framework
|
|||
It can be used for a wide range of purposes, from data mining to
|
||||
monitoring and automated testing.
|
||||
|
||||
Package: scrapy-service
|
||||
Package: scrapyd
|
||||
Architecture: all
|
||||
Depends: scrapy
|
||||
Depends: scrapy, python-setuptools
|
||||
Description: Scrapy Service
|
||||
This package provides support for running Scrapy as a system service,
|
||||
controlled through an upstart script.
|
||||
The Scrapy service allows you to deploy your Scrapy projects by building
|
||||
Python eggs of them and uploading them to the Scrapy service using a JSON API
|
||||
that you can also use for scheduling spider runs. It supports multiple
|
||||
projects also.
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
# Defaults for Scrapy service
|
||||
|
||||
export PYTHONPATH=/etc/scrapy
|
||||
export SCRAPY_SETTINGS_MODULE=service_conf
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
var/lib/scrapy
|
||||
var/log/scrapy
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
debian/service_conf.py etc/scrapy
|
||||
extras/scrapy.tac usr/share/scrapy
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
remove|upgrade|deconfigure)
|
||||
rm -f /etc/scrapy/service_conf.pyc
|
||||
;;
|
||||
|
||||
failed-upgrade)
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "prerm called with unknown argument \`$1'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# Scrapy service
|
||||
|
||||
start on runlevel [2345]
|
||||
stop on runlevel [06]
|
||||
|
||||
script
|
||||
[ -r /etc/default/scrapy-service ] && . /etc/default/scrapy-service
|
||||
exec twistd -ny /usr/share/scrapy/scrapy.tac \
|
||||
-u scrapy -g nogroup \
|
||||
-l /var/log/scrapy/service.log \
|
||||
--pidfile /var/run/scrapy/scrapy.pid
|
||||
end script
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
.TH SCRAPY 1 "October 17, 2009"
|
||||
.SH NAME
|
||||
scrapy \- Python Scrapy control script
|
||||
scrapy \- the Scrapy command-line tool
|
||||
.SH SYNOPSIS
|
||||
.B scrapy
|
||||
[\fIcommand\fR] [\fIOPTIONS\fR] ...
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
Scrapy is controlled through the \fBscrapy\fR control script. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options.
|
||||
Scrapy is controlled through the \fBscrapy\fR command-line tool. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options.
|
||||
.SH OPTIONS
|
||||
.SS fetch\fR [\fIOPTION\fR] \fIURL\fR
|
||||
.TP
|
||||
|
|
@ -50,8 +50,6 @@ Create new project with an initial project template
|
|||
|
||||
.SS --help, -h
|
||||
Print command help and options
|
||||
.SS --version
|
||||
Print Scrapy version and exit
|
||||
.SS --logfile=FILE
|
||||
Log file. if omitted stderr will be used
|
||||
.SS --loglevel=LEVEL, -L LEVEL
|
||||
|
|
@ -68,8 +66,6 @@ Write lsprof profiling stats to FILE
|
|||
Write process ID to FILE
|
||||
.SS --set=SET
|
||||
Set/override setting (may be repeated)
|
||||
.SS --settings=MODULE
|
||||
Python path to the Scrapy project settings
|
||||
|
||||
.SH AUTHOR
|
||||
Scrapy was written by the Scrapy Developers
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
debian/tmp/usr
|
||||
usr/lib/python*/*-packages/scrapy
|
||||
usr/bin
|
||||
extras/scrapy_bash_completion etc/bash_completion.d/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
[scrapyd]
|
||||
http_port = 6800
|
||||
debug = off
|
||||
#max_proc = 1
|
||||
eggs_dir = /var/lib/scrapyd/eggs
|
||||
dbs_dir = /var/lib/scrapyd/dbs
|
||||
logs_dir = /var/log/scrapyd
|
||||
|
|
@ -0,0 +1 @@
|
|||
# Defaults for Scrapy service
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
var/lib/scrapyd
|
||||
var/lib/scrapyd/eggs
|
||||
var/lib/scrapyd/dbs
|
||||
var/log/scrapyd
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
usr/lib/python*/*-packages/scrapyd
|
||||
debian/scrapyd.cfg etc
|
||||
extras/scrapyd.tac usr/share/scrapyd
|
||||
|
|
@ -1,2 +1,2 @@
|
|||
new-package-should-close-itp-bug
|
||||
script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapy-service
|
||||
script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapyd
|
||||
|
|
@ -6,16 +6,19 @@ case "$1" in
|
|||
configure)
|
||||
# Create user to run the service as
|
||||
if [ -z "`id -u scrapy 2> /dev/null`" ]; then
|
||||
adduser --system --home /var/lib/scrapy --gecos "scrapy" \
|
||||
adduser --system --home /var/lib/scrapyd --gecos "scrapy" \
|
||||
--no-create-home --disabled-password \
|
||||
--quiet scrapy || true
|
||||
fi
|
||||
if [ ! -d /var/run/scrapy ]; then
|
||||
mkdir /var/run/scrapy
|
||||
chown scrapy:nogroup /var/run/scrapy
|
||||
if [ ! -d /var/run/scrapyd ]; then
|
||||
mkdir /var/run/scrapyd
|
||||
chown scrapy:nogroup /var/run/scrapyd
|
||||
fi
|
||||
|
||||
chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy
|
||||
chown scrapy:nogroup /var/log/scrapyd /var/run/scrapyd \
|
||||
/var/lib/scrapyd /var/lib/scrapyd/eggs /var/lib/scrapyd/dbs
|
||||
|
||||
update-python-modules -p # so upstart restart uses the new code
|
||||
;;
|
||||
|
||||
abort-upgrade|abort-remove|abort-deconfigure)
|
||||
|
|
@ -8,7 +8,7 @@ if [ purge = "$1" ]; then
|
|||
else
|
||||
echo >&2 "not removing scrapy system account because deluser command was not found"
|
||||
fi
|
||||
rm -rf /var/run/scrapy
|
||||
rm -rf /var/run/scrapyd
|
||||
fi
|
||||
|
||||
#DEBHELPER#
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# Scrapy service
|
||||
|
||||
start on runlevel [2345]
|
||||
stop on runlevel [06]
|
||||
|
||||
script
|
||||
[ -r /etc/default/scrapyd ] && . /etc/default/scrapyd
|
||||
logdir=/var/log/scrapyd
|
||||
exec twistd -ny /usr/share/scrapyd/scrapyd.tac \
|
||||
-u scrapy -g nogroup \
|
||||
--pidfile /var/run/scrapyd/scrapy.pid \
|
||||
-l $logdir/scrapyd.log >$logdir/scrapyd.out 2>$logdir/scrapyd.err
|
||||
end script
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Scrapy service configuration
|
||||
|
||||
# Directory where logs will be stored (one per crawler)
|
||||
LOG_DIR = '/var/log/scrapy'
|
||||
|
||||
# A dict containing the Scrapy projects that will be run by this service
|
||||
# * Keys are paths to project settings modules
|
||||
# * Values are number of processes that should be started for each project, or
|
||||
# zero if you want to use the number of cores available.
|
||||
PROJECTS = {
|
||||
# 'mybot.settings': 0,
|
||||
}
|
||||
|
||||
|
|
@ -125,6 +125,7 @@ Solving specific problems
|
|||
topics/leaks
|
||||
topics/images
|
||||
topics/ubuntu
|
||||
topics/scrapyd
|
||||
|
||||
:doc:`faq`
|
||||
Get answers to most frequently asked questions.
|
||||
|
|
@ -144,6 +145,9 @@ Solving specific problems
|
|||
:doc:`topics/ubuntu`
|
||||
Install latest Scrapy packages easily on Ubuntu
|
||||
|
||||
:doc:`topics/scrapyd`
|
||||
Deploying your Scrapy project in production.
|
||||
|
||||
.. _extending-scrapy:
|
||||
|
||||
Extending Scrapy
|
||||
|
|
|
|||
|
|
@ -0,0 +1,379 @@
|
|||
.. _topics-scrapyd:
|
||||
|
||||
========================
|
||||
Scrapy Service (scrapyd)
|
||||
========================
|
||||
|
||||
Scrapy comes with a built-in service, called "Scrapyd", which allows you to
|
||||
deploy (aka. upload) your projects and control their spiders using a JSON web
|
||||
service.
|
||||
|
||||
Projects and versions
|
||||
=====================
|
||||
|
||||
Scrapyd can manage multiple projects and each project can have multiple
|
||||
versions uploaded, but only the latest one will be used for launching new
|
||||
spiders.
|
||||
|
||||
A common (and useful) convention to use for the version name is the revision
|
||||
number of the version control tool you're using to track your Scrapy project
|
||||
code. For example: ``r23``. The versions are not compared alphabetically but
|
||||
using a smarter algorithm (the same `distutils`_ uses) so ``r10`` compares
|
||||
greater to ``r9``, for example.
|
||||
|
||||
How Scrapyd works
|
||||
=================
|
||||
|
||||
Scrapyd is an application (typically run as a daemon) that continually polls
|
||||
for projects that need to run (ie. those projects that have spiders enqueued).
|
||||
|
||||
When a project needs to run, a Scrapy process is started for that project using
|
||||
something similar to the typical ``scrapy crawl``` command, and it continues to
|
||||
run until it finishes processing all spiders form the spider queue.
|
||||
|
||||
Scrapyd also runs multiple processes in parallel, allocating them in a fixed
|
||||
number of "slots", which defaults to the number of cpu processors available in
|
||||
the system, but this can be changed with the ``max_proc`` option. It also
|
||||
starts as many processes as possible to handle the load.
|
||||
|
||||
In addition to dispatching and managing processes, Scrapyd provides a
|
||||
:ref:`JSON web service <topics-scrapyd-jsonapi>` to upload new project versions
|
||||
(as eggs) and schedule spiders. This feature is optional and can be disabled if
|
||||
you want to implement your own custom Scrapyd. The components are pluggable and
|
||||
can be changed, if you're familiar with the `Twisted Application Framework`_
|
||||
which Scrapyd is implemented in.
|
||||
|
||||
Starting Scrapyd
|
||||
================
|
||||
|
||||
Scrapyd is implemented using the standard `Twisted Application Framework`_. To
|
||||
start the service, use the ``extras/scrapyd.tac`` file provided in the Scrapy
|
||||
distribution, like this::
|
||||
|
||||
twistd -ny extras/scrapyd.tac
|
||||
|
||||
That should get your Scrapyd started.
|
||||
|
||||
Installing Scrapyd
|
||||
==================
|
||||
|
||||
How to deploy Scrapyd on your servers depends on the platform your're using.
|
||||
Scrapy comes with Ubuntu packages for Scrapyd ready for deploying it as a
|
||||
system service, to ease the installation and administration, but you can create
|
||||
packages for other distribution or operating systems (including Windows). If
|
||||
you do so, and want to contribute them, send a message to
|
||||
scrapy-developers@googlegroups.com and say hi. The community will appreciate
|
||||
it.
|
||||
|
||||
.. _topics-scrapyd-ubuntu:
|
||||
|
||||
Installing Scrapyd in Ubuntu
|
||||
----------------------------
|
||||
|
||||
When deploying Scrapyd, it's very useful to have a version already packaged for
|
||||
your system. For this reason, Scrapyd comes with Ubuntu packages ready to use
|
||||
in your Ubuntu servers.
|
||||
|
||||
So, if you plan to deploy Scrapyd on a Ubuntu server, just add the Ubuntu
|
||||
repositories as described in :ref:`topics-ubuntu` and then run::
|
||||
|
||||
aptitude install scrapyd
|
||||
|
||||
This will install Scrapyd in your Ubuntu server creating a ``scrapy`` user
|
||||
which Scrapyd will run as. It will also create some directories and files that
|
||||
are listed below:
|
||||
|
||||
/etc/scrapyd.cfg
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Scrapyd configuration file. See :ref:`topics-scrapyd-config`.
|
||||
|
||||
/var/log/scrapyd/scrapyd.log
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Scrapyd main log file.
|
||||
|
||||
/var/log/scrapyd/scrapyd.out
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The standard output captured from Scrapyd process and any
|
||||
sub-process spawned from it.
|
||||
|
||||
/var/log/scrapyd/scrapyd.err
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The standard error captured from Scrapyd and any sub-process spawned
|
||||
from it. Remember to check this file if you're having problems, as the errors
|
||||
may not get logged to the ``scrapyd.log`` file.
|
||||
|
||||
/var/log/scrapyd/slotN.log
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
The log files of Scrapy processes started from Scrapyd, one per slot. These are
|
||||
standard :ref:`Scrapy log files <topics-logging>`.
|
||||
|
||||
/var/lib/scrapyd/
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
Directory used to store data files (uploaded eggs and spider queues).
|
||||
|
||||
.. _topics-scrapyd-config:
|
||||
|
||||
Scrapyd Configuration file
|
||||
==========================
|
||||
|
||||
The Scrapyd configuration file supports the following options:
|
||||
|
||||
http_port
|
||||
---------
|
||||
|
||||
The TCP port where the HTTP JSON API will listen. Defaults to ``6800``.
|
||||
|
||||
max_proc
|
||||
--------
|
||||
|
||||
The maximum number of concurrent Scrapy process that will be started. If unset
|
||||
or ``0`` it will use the number of cpus available in the system.
|
||||
|
||||
debug
|
||||
-----
|
||||
|
||||
Whether debug mode is enabled. Defaults to ``off``. When debug mode is enabled
|
||||
the full Python traceback will be returned (as plain text responses) when there
|
||||
is an error processing a JSON API call.
|
||||
|
||||
eggs_dir
|
||||
--------
|
||||
|
||||
The directory where the project eggs will be stored.
|
||||
|
||||
dbs_dir
|
||||
-------
|
||||
|
||||
The directory where the project databases will be stored (this includes the
|
||||
spider queues).
|
||||
|
||||
logs_dir
|
||||
--------
|
||||
|
||||
The directory where the Scrapy processes logs (``slotN.log``) will be stored.
|
||||
|
||||
Example configuration file
|
||||
--------------------------
|
||||
|
||||
Here is an example configuration file with all the defaults:
|
||||
|
||||
.. literalinclude:: ../../scrapyd/default_scrapyd.cfg
|
||||
|
||||
Eggifying your project
|
||||
======================
|
||||
|
||||
In order to upload your project to Scrapyd, you must first build a `Python
|
||||
egg`_ of it. This is called "eggifying" your project. You'll need to install
|
||||
`setuptools`_ for this.
|
||||
|
||||
To eggify your project add a `setup.py`_ file to the root directory of your
|
||||
project (where the ``scrapy.cfg`` resides) with the following contents::
|
||||
|
||||
#!/usr/bin/env python
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
name = 'myproject',
|
||||
version = '1.0',
|
||||
packages = find_packages(),
|
||||
entry_points = {'scrapy': ['settings = myproject.settings']},
|
||||
)
|
||||
|
||||
And then the run the following command::
|
||||
|
||||
python setup.py bdist_egg
|
||||
|
||||
This will generate an egg file and leave it in the ``dist`` directory, for
|
||||
example::
|
||||
|
||||
dist/myproject-1.0-py2.6.egg
|
||||
|
||||
Egg caveats
|
||||
-----------
|
||||
|
||||
There are some things to keep in mind when building eggs of your Scrapy
|
||||
project:
|
||||
|
||||
* make sure no local development settings are included in the egg when you
|
||||
build it. The ``find_packages`` function may be picking up your custom
|
||||
settings. In most cases you want to upload the egg with the default project
|
||||
settings.
|
||||
|
||||
* you shouldn't use ``__file__`` in your project code as it doesn't play well
|
||||
with eggs. Consider using `pkgutil.get_data()`_ instead.
|
||||
|
||||
* be careful when writing to disk in your project (in any spider, extension or
|
||||
middleware) as Scrapyd will probably run with a different user which may not
|
||||
have write access to certain directories. If you can, avoid writing to disk
|
||||
and always use `tempfile`_ for temporary files.
|
||||
|
||||
Uploading your project
|
||||
======================
|
||||
|
||||
In these examples we'll be using `curl`_ for the web service interaction
|
||||
examples, but you can use any command or library that speaks HTTP.
|
||||
|
||||
Once you've built the egg, you can upload your project to Scrapyd, like this::
|
||||
|
||||
$ curl http://localhost:6800/addversion.json -F project=myproject -F version=r23 -F egg=@dist/myproject-1.0-py2.6.egg
|
||||
{"status": "ok", "spiders": ["spider1", "spider2", "spider3"]}
|
||||
|
||||
You'll see that the JSON response contains the spiders found in your project.
|
||||
|
||||
Scheduling a spider run
|
||||
=======================
|
||||
|
||||
To schedule a spider run::
|
||||
|
||||
$ curl http://localhost:6800/schedule.json -d project=myproject -d spider=spider=spider2
|
||||
{"status": "ok"}
|
||||
|
||||
For more resources see: :ref:`topics-scrapyd-jsonapi` for more available resources.
|
||||
|
||||
.. _topics-scrapyd-jsonapi:
|
||||
|
||||
JSON API reference
|
||||
==================
|
||||
|
||||
The following section describes the available resources in Scrapyd JSON API.
|
||||
|
||||
addversion.json
|
||||
---------------
|
||||
|
||||
Add a version to a project, creating the project if it doesn't exist.
|
||||
|
||||
* Supported Request Methods: ``POST``
|
||||
* Parameters:
|
||||
* ``project`` (string, required) - the project name
|
||||
* ``version`` (string, required) - the project version
|
||||
* ``egg`` (file, required) - a Python egg containing the project's code
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/addversion.json -F project=myproject -F version=r23 -F egg=@myproject.egg
|
||||
|
||||
Example reponse::
|
||||
|
||||
{"status": "ok", "spiders": ["spider1", "spider2", "spider3"]}
|
||||
|
||||
schedule.json
|
||||
-------------
|
||||
|
||||
Schedule a spider run.
|
||||
|
||||
* Supported Request Methods: ``POST``
|
||||
* Parameters:
|
||||
* ``project`` (string, required) - the project name
|
||||
* ``spider`` (string, required) - the spider name
|
||||
* any other parameter is passed as spider argument
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/schedule.json -d project=myproject -d spider=somespider
|
||||
|
||||
Example response::
|
||||
|
||||
{"status": "ok"}
|
||||
|
||||
listprojects.json
|
||||
-----------------
|
||||
|
||||
Get the list of projects uploaded to this Scrapy server.
|
||||
|
||||
* Supported Request Methods: ``GET``
|
||||
* Parameters: none
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/listprojects.json
|
||||
|
||||
Example response::
|
||||
|
||||
{"status": "ok", "projects": ["myproject", "otherproject"]}
|
||||
|
||||
listversions.json
|
||||
-----------------
|
||||
|
||||
Get the list of versions available for some project. The versions are returned
|
||||
in order, the last one is the currently used version.
|
||||
|
||||
* Supported Request Methods: ``GET``
|
||||
* Parameters:
|
||||
* ``project`` (string, required) - the project name
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/listversions.json?project=myproject
|
||||
|
||||
Example response::
|
||||
|
||||
{"status": "ok", "versions": ["r99", "r156"]}
|
||||
|
||||
listspiders.json
|
||||
----------------
|
||||
|
||||
Get the list of spiders available in the last version of some project.
|
||||
|
||||
* Supported Request Methods: ``GET``
|
||||
* Parameters:
|
||||
* ``project`` (string, required) - the project name
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/listspiders.json?project=myproject
|
||||
|
||||
Example response::
|
||||
|
||||
{"status": "ok", "spiders": ["spider1", "spider2", "spider3"]}
|
||||
|
||||
delversion.json
|
||||
---------------
|
||||
|
||||
Delete a project version. If there are no more versions available for a given
|
||||
project, that project will be deleted too.
|
||||
|
||||
* Supported Request Methods: ``POST``
|
||||
* Parameters:
|
||||
* ``project`` (string, required) - the project name
|
||||
* ``version`` (string, required) - the project version
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/delversion.json -d project=myproject -d version=r99
|
||||
|
||||
Example response::
|
||||
|
||||
{"status": "ok"}
|
||||
|
||||
delproject.json
|
||||
---------------
|
||||
|
||||
Delete a project and all its uploaded versions.
|
||||
|
||||
* Supported Request Methods: ``POST``
|
||||
* Parameters:
|
||||
* ``project`` (string, required) - the project name
|
||||
|
||||
Example request::
|
||||
|
||||
$ curl http://localhost:6800/delproject.json?project=myproject
|
||||
|
||||
Example response::
|
||||
|
||||
{"status": "ok"}
|
||||
|
||||
.. _Python egg: http://peak.telecommunity.com/DevCenter/PythonEggs
|
||||
.. _setup.py: http://docs.python.org/distutils/setupscript.html
|
||||
.. _curl: http://en.wikipedia.org/wiki/CURL
|
||||
.. _setuptools: http://pypi.python.org/pypi/setuptools
|
||||
.. _pkgutil.get_data(): http://docs.python.org/library/pkgutil.html#pkgutil.get_data
|
||||
.. _tempfile: http://docs.python.org/library/tempfile.html
|
||||
.. _Twisted Application Framework: http://twistedmatrix.com/documents/current/core/howto/application.html
|
||||
.. _distutils: http://docs.python.org/library/distutils.html
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
from twisted.application.service import Application
|
||||
from scrapy.service import ScrapyService
|
||||
|
||||
application = Application("Scrapy")
|
||||
ScrapyService().setServiceParent(application)
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
from scrapyd.app import get_application
|
||||
application = get_application()
|
||||
|
|
@ -14,6 +14,11 @@ if sys.version_info < (2,5):
|
|||
# ignore noisy twisted deprecation warnings
|
||||
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
|
||||
|
||||
# prevents noisy (and innocent) dropin.cache errors when loading spiders from
|
||||
# egg files using the old Spider Manager. TODO: Remove for Scrapy 0.11
|
||||
from twisted.python.zippath import ZipPath
|
||||
ZipPath.setContent = lambda x, y: None
|
||||
|
||||
# monkey patches to fix external library issues
|
||||
from scrapy.xlib import twisted_250_monkeypatches
|
||||
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
import sys, os
|
||||
|
||||
from twisted.python import log
|
||||
from twisted.internet import reactor, defer, protocol, error
|
||||
from twisted.application.service import Service
|
||||
|
||||
from scrapy.utils.py26 import cpu_count
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class ScrapyService(Service):
|
||||
|
||||
def startService(self):
|
||||
reactor.callWhenRunning(self.spawn_processes)
|
||||
|
||||
def spawn_processes(self):
|
||||
for settings_module, count in settings['PROJECTS'].items():
|
||||
for i in range(count or cpu_count()):
|
||||
self.spawn_process(settings_module, i)
|
||||
|
||||
def spawn_process(self, settings_module, position):
|
||||
args = [sys.executable, '-m', 'scrapy.cmdline', 'runserver']
|
||||
env = os.environ.copy()
|
||||
botname = self.get_bot_name(settings_module)
|
||||
logfile = self.get_log_file(settings_module, position)
|
||||
env['SCRAPY_SETTINGS_MODULE'] = settings_module
|
||||
env['SCRAPY_LOG_FILE'] = logfile
|
||||
pp = ScrapyProcessProtocol(botname, settings_module, logfile)
|
||||
pp.deferred.addCallback(lambda _: reactor.callLater(5, \
|
||||
self.spawn_process, settings_module, position))
|
||||
reactor.spawnProcess(pp, sys.executable, args=args, env=env)
|
||||
|
||||
def get_log_file(self, settings_module, position):
|
||||
botname = self.get_bot_name(settings_module)
|
||||
basename = "%s-%s" % (botname, position) if position else botname
|
||||
return os.path.join(settings['LOG_DIR'], "%s.log" % basename)
|
||||
|
||||
def get_bot_name(self, settings_module):
|
||||
mod = __import__(settings_module, {}, {}, [''], -1)
|
||||
return mod.BOT_NAME
|
||||
|
||||
class ScrapyProcessProtocol(protocol.ProcessProtocol):
|
||||
|
||||
TAIL_LINES = 100
|
||||
|
||||
def __init__(self, botname, settings_module, logfile):
|
||||
self.botname = botname
|
||||
self.settings_module = settings_module
|
||||
self.logfile = logfile
|
||||
self.pid = None
|
||||
self.deferred = defer.Deferred()
|
||||
self.outdata = ''
|
||||
self.errdata = ''
|
||||
|
||||
def outReceived(self, data):
|
||||
self.outdata = self._tail(self.outdata + data)
|
||||
|
||||
def errReceived(self, data):
|
||||
self.errdata = self._tail(self.errdata + data)
|
||||
|
||||
def connectionMade(self):
|
||||
self.pid = self.transport.pid
|
||||
log.msg("Crawler %r started: pid=%r settings=%r log=%r" % \
|
||||
(self.botname, self.pid, self.settings_module, self.logfile))
|
||||
|
||||
def processEnded(self, status):
|
||||
if isinstance(status.value, error.ProcessDone):
|
||||
msg = "Crawler %r finished: pid=%r settings=%r log=%r" % \
|
||||
(self.botname, self.pid, self.settings_module, self.logfile)
|
||||
else:
|
||||
msg = "Crawler %r died: exitstatus=%r pid=%r settings=%r log=%r" % \
|
||||
(self.botname, status.value.exitCode, self.pid, self.settings_module, \
|
||||
self.logfile)
|
||||
self._log_ended(msg)
|
||||
self.deferred.callback(self)
|
||||
|
||||
def _log_ended(self, msg):
|
||||
tolog = [msg]
|
||||
if self.outdata:
|
||||
tolog += [">>> stdout (last %d lines) <<<" % self.TAIL_LINES]
|
||||
tolog += [self.outdata]
|
||||
if self.errdata:
|
||||
tolog += [">>> stderr (last %d lines) <<<" % self.TAIL_LINES]
|
||||
tolog += [self.errdata]
|
||||
log.msg(os.linesep.join(tolog))
|
||||
|
||||
def _tail(self, data, lines=TAIL_LINES):
|
||||
return os.linesep.join(data.split(os.linesep)[-lines:])
|
||||
|
|
@ -49,9 +49,13 @@ def init_env(project='default', set_syspath=True):
|
|||
|
||||
def get_config(use_closest=True):
|
||||
"""Get Scrapy config file as a SafeConfigParser"""
|
||||
sources = [os.path.expanduser('~/.scrapy.cfg'), '/etc/scrapy.cfg']
|
||||
if use_closest:
|
||||
sources.insert(0, closest_scrapy_cfg())
|
||||
sources = get_sources(use_closest)
|
||||
cfg = SafeConfigParser()
|
||||
cfg.read(sources)
|
||||
return cfg
|
||||
|
||||
def get_sources(use_closest=True):
|
||||
sources = [os.path.expanduser('~/.scrapy.cfg'), '/etc/scrapy.cfg']
|
||||
if use_closest:
|
||||
sources.insert(0, closest_scrapy_cfg())
|
||||
return sources
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
from twisted.web import resource
|
||||
|
||||
from scrapy.utils.py26 import json
|
||||
|
||||
class JsonResource(resource.Resource):
|
||||
|
||||
json_encoder = json.JSONEncoder()
|
||||
|
||||
def render(self, txrequest):
|
||||
r = resource.Resource.render(self, txrequest)
|
||||
return self.render_object(r, txrequest)
|
||||
|
||||
def render_object(self, obj, txrequest):
|
||||
r = self.json_encoder.encode(obj) + "\n"
|
||||
txrequest.setHeader('Content-Type', 'application/json')
|
||||
txrequest.setHeader('Content-Length', len(r))
|
||||
return r
|
||||
|
|
@ -5,7 +5,7 @@ See docs/topics/ws.rst
|
|||
"""
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.web import server, resource, error
|
||||
from twisted.web import server, error
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -13,23 +13,15 @@ from scrapy import signals
|
|||
from scrapy.utils.jsonrpc import jsonrpc_server_call
|
||||
from scrapy.utils.serialize import ScrapyJSONEncoder, ScrapyJSONDecoder
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.txweb import JsonResource as JsonResource_
|
||||
from scrapy.utils.conf import build_component_list
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class JsonResource(resource.Resource):
|
||||
class JsonResource(JsonResource_):
|
||||
|
||||
ws_name = None
|
||||
json_encoder = ScrapyJSONEncoder()
|
||||
|
||||
def render(self, txrequest):
|
||||
r = resource.Resource.render(self, txrequest)
|
||||
r = self.json_encoder.encode(r)
|
||||
txrequest.setHeader('Content-Type', 'application/json')
|
||||
txrequest.setHeader('Content-Length', len(r))
|
||||
return r
|
||||
|
||||
|
||||
class JsonRpcResource(JsonResource):
|
||||
|
||||
json_decoder = ScrapyJSONDecoder()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
import sys, os
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.application.service import Application, Service
|
||||
from twisted.application.internet import TimerService, TCPServer
|
||||
from twisted.web import server
|
||||
|
||||
from .interfaces import IEggStorage, IPoller, ISpiderScheduler, IEnvironment
|
||||
from .launcher import Launcher
|
||||
from .eggstorage import FilesystemEggStorage
|
||||
from .scheduler import SpiderScheduler
|
||||
from .poller import QueuePoller
|
||||
from .environ import Environment
|
||||
from .webservice import Root
|
||||
from .config import Config
|
||||
|
||||
def get_application():
|
||||
app = Application("Scrapy")
|
||||
config = Config()
|
||||
http_port = config.getint('http_port', 6800)
|
||||
|
||||
poller = QueuePoller(config)
|
||||
eggstorage = FilesystemEggStorage(config)
|
||||
scheduler = SpiderScheduler(config)
|
||||
environment = Environment(config)
|
||||
|
||||
app.setComponent(IPoller, poller)
|
||||
app.setComponent(IEggStorage, eggstorage)
|
||||
app.setComponent(ISpiderScheduler, scheduler)
|
||||
app.setComponent(IEnvironment, environment)
|
||||
|
||||
launcher = Launcher(config, app)
|
||||
timer = TimerService(5, poller.poll)
|
||||
webservice = TCPServer(http_port, server.Site(Root(config, app)))
|
||||
|
||||
launcher.setServiceParent(app)
|
||||
timer.setServiceParent(app)
|
||||
webservice.setServiceParent(app)
|
||||
|
||||
return app
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import pkgutil
|
||||
from cStringIO import StringIO
|
||||
from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from scrapy.utils.conf import get_sources
|
||||
|
||||
class Config(object):
|
||||
"""A ConfigParser wrapper to support defaults when calling instance
|
||||
methods, and also tied to a single section"""
|
||||
|
||||
SOURCES = ['scrapyd.cfg', '/etc/scrapyd.cfg']
|
||||
SECTION = 'scrapyd'
|
||||
|
||||
def __init__(self):
|
||||
sources = self.SOURCES + get_sources()
|
||||
default_config = pkgutil.get_data(__package__, 'default_scrapyd.cfg')
|
||||
self.cp = SafeConfigParser()
|
||||
self.cp.readfp(StringIO(default_config))
|
||||
self.cp.read(sources)
|
||||
|
||||
def _getany(self, method, option, default):
|
||||
try:
|
||||
return method(self.SECTION, option)
|
||||
except (NoSectionError, NoOptionError):
|
||||
if default is not None:
|
||||
return default
|
||||
raise
|
||||
|
||||
def get(self, option, default=None):
|
||||
return self._getany(self.cp.get, option, default)
|
||||
|
||||
def getint(self, option, default=None):
|
||||
return self._getany(self.cp.getint, option, default)
|
||||
|
||||
def getfloat(self, option, default=None):
|
||||
return self._getany(self.cp.getfloat, option, default)
|
||||
|
||||
def getboolean(self, option, default=None):
|
||||
return self._getany(self.cp.getboolean, option, default)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[scrapyd]
|
||||
eggs_dir = eggs
|
||||
logs_dir = logs
|
||||
dbs_dir = dbs
|
||||
max_proc = 0
|
||||
http_port = 6800
|
||||
debug = off
|
||||
egg_runner = scrapyd.eggrunner
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""
|
||||
This module can be used to run a Scrapy project contained in an egg file
|
||||
|
||||
To see all spiders in a project:
|
||||
|
||||
python -m scrapyd.eggrunner myproject.egg list
|
||||
|
||||
To crawl a spider:
|
||||
|
||||
python -m scrapyd.eggrunner myproject.egg crawl somespider
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
from scrapyd.eggutils import activate_egg
|
||||
|
||||
def main(eggpath, args):
|
||||
"""Run scrapy for the settings module name passed"""
|
||||
activate_egg(eggpath)
|
||||
from scrapy.cmdline import execute
|
||||
execute(['scrapy'] + list(args))
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) < 2:
|
||||
print "usage: %s <eggfile> [scrapy_command args ...]" % sys.argv[0]
|
||||
sys.exit(1)
|
||||
main(sys.argv[1], sys.argv[2:])
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
from glob import glob
|
||||
from os import path, makedirs, remove
|
||||
from shutil import copyfileobj, rmtree
|
||||
from distutils.version import LooseVersion
|
||||
|
||||
from zope.interface import implements
|
||||
from twisted.application.service import Service
|
||||
|
||||
from .interfaces import IEggStorage
|
||||
|
||||
class FilesystemEggStorage(Service):
|
||||
|
||||
implements(IEggStorage)
|
||||
|
||||
def __init__(self, config):
|
||||
self.basedir = config.get('eggs_dir', 'eggs')
|
||||
|
||||
def put(self, eggfile, project, version):
|
||||
eggpath = self._eggpath(project, version)
|
||||
eggdir = path.dirname(eggpath)
|
||||
if not path.exists(eggdir):
|
||||
makedirs(eggdir)
|
||||
with open(eggpath, 'wb') as f:
|
||||
copyfileobj(eggfile, f)
|
||||
|
||||
def get(self, project, version=None):
|
||||
if version is None:
|
||||
version = self.list(project)[-1]
|
||||
return version, open(self._eggpath(project, version), 'rb')
|
||||
|
||||
def list(self, project):
|
||||
eggdir = path.join(self.basedir, project)
|
||||
versions = [path.splitext(path.basename(x))[0] \
|
||||
for x in glob("%s/*.egg" % eggdir)]
|
||||
return sorted(versions, key=LooseVersion)
|
||||
|
||||
def delete(self, project, version=None):
|
||||
if version is None:
|
||||
rmtree(path.join(self.basedir, project))
|
||||
else:
|
||||
remove(self._eggpath(project, version))
|
||||
if not self.list(project): # remove project if no versions left
|
||||
self.delete(project)
|
||||
|
||||
def _eggpath(self, project, version):
|
||||
x = path.join(self.basedir, project, "%s.egg" % version)
|
||||
return x
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import os, sys, shutil, pkg_resources
|
||||
from subprocess import Popen, PIPE
|
||||
from tempfile import NamedTemporaryFile, mkdtemp
|
||||
|
||||
def get_spider_list_from_eggfile(eggfile, project):
|
||||
# FIXME: we use a temporary directory here to avoid permissions problems
|
||||
# when running as system service, as "scrapy list" command tries to write
|
||||
# the scrapy.db sqlite database in current directory
|
||||
tmpdir = mkdtemp()
|
||||
try:
|
||||
with NamedTemporaryFile(suffix='.egg', dir=tmpdir) as f:
|
||||
shutil.copyfileobj(eggfile, f)
|
||||
f.flush()
|
||||
eggfile.seek(0)
|
||||
pargs = [sys.executable, '-m', 'scrapyd.eggrunner', f.name, 'list']
|
||||
env = os.environ.copy()
|
||||
env['SCRAPY_PROJECT'] = project
|
||||
proc = Popen(pargs, stdout=PIPE, cwd=tmpdir, env=env)
|
||||
out = proc.communicate()[0]
|
||||
return out.splitlines()
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def activate_egg(eggpath):
|
||||
"""Activate a Scrapy egg file. This is meant to be used from egg runners
|
||||
to activate a Scrapy egg file. Don't use it from other code as it may
|
||||
leave unwanted side effects.
|
||||
"""
|
||||
d = pkg_resources.find_distributions(eggpath).next()
|
||||
d.activate()
|
||||
settings_module = d.get_entry_info('scrapy', 'settings').module_name
|
||||
os.environ['SCRAPY_SETTINGS_MODULE'] = settings_module
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import os
|
||||
|
||||
from zope.interface import implements
|
||||
|
||||
from .interfaces import IEnvironment
|
||||
|
||||
class Environment(object):
|
||||
|
||||
implements(IEnvironment)
|
||||
|
||||
def __init__(self, config):
|
||||
self.dbs_dir = config.get('dbs_dir', 'dbs')
|
||||
self.logs_dir = config.get('logs_dir', 'logs')
|
||||
|
||||
def get_environment(self, message, slot):
|
||||
project = message['project']
|
||||
env = os.environ.copy()
|
||||
env['SCRAPY_PROJECT'] = project
|
||||
dbpath = os.path.join(self.dbs_dir, '%s.db' % project)
|
||||
env['SCRAPY_SQLITE_DB'] = dbpath
|
||||
logpath = os.path.join(self.logs_dir, 'slot%s.log' % slot)
|
||||
env['SCRAPY_LOG_FILE'] = logpath
|
||||
return env
|
||||
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
from zope.interface import Interface
|
||||
|
||||
class IEggStorage(Interface):
|
||||
"""A component that handles storing and retrieving eggs"""
|
||||
|
||||
def put(eggfile, project, version):
|
||||
"""Store the egg (passed in the file object) under the given project and
|
||||
version"""
|
||||
|
||||
def get(project, version=None):
|
||||
"""Return a tuple (version, file) with the the egg for the specified
|
||||
project and version. If version is None, the latest version is
|
||||
returned."""
|
||||
|
||||
def list(self, project):
|
||||
"""Return the list of versions which have eggs stored (for the given
|
||||
project) in order (the latest version is the currently used)."""
|
||||
|
||||
def delete(self, project, version=None):
|
||||
"""Delete the egg stored for the given project and version. If should
|
||||
also delete the project if no versions are left"""
|
||||
|
||||
|
||||
class IPoller(Interface):
|
||||
"""A component that polls for projects that need to run"""
|
||||
|
||||
def poll():
|
||||
"""Called periodically to poll for projects"""
|
||||
|
||||
def next():
|
||||
"""Return the next message.
|
||||
|
||||
It should return a Deferred which will get fired when there is a new
|
||||
project that needs to run, or already fired if there was a project
|
||||
waiting to run already.
|
||||
|
||||
The message is a dict containing (at least) the name of the project to
|
||||
be run in the 'project' key. This message will be passed later to
|
||||
IEnvironment.get_environment().
|
||||
"""
|
||||
|
||||
def update_projects():
|
||||
"""Called when projects may have changed, to refresh the available
|
||||
projects"""
|
||||
|
||||
|
||||
class ISpiderScheduler(Interface):
|
||||
"""A component to schedule spider runs"""
|
||||
|
||||
def schedule(project, spider_name, **spider_args):
|
||||
"""Schedule a spider for the given project"""
|
||||
|
||||
def list_projects():
|
||||
"""Return the list of available projects"""
|
||||
|
||||
def update_projects():
|
||||
"""Called when projects may have changed, to refresh the available
|
||||
projects"""
|
||||
|
||||
|
||||
class IEnvironment(Interface):
|
||||
"""A component to generate the environment of crawler processes"""
|
||||
|
||||
def get_environment(message, slot):
|
||||
"""Return the environment variables to use for running the process.
|
||||
|
||||
`message` is the message received from the IPoller.next()
|
||||
`slot` is the Launcher slot where the process will be running.
|
||||
"""
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
import sys, os
|
||||
from shutil import copyfileobj
|
||||
from tempfile import mkstemp
|
||||
|
||||
from twisted.internet import reactor, defer, protocol, error
|
||||
from twisted.application.service import Service
|
||||
from twisted.python import log
|
||||
|
||||
from scrapy.utils.py26 import cpu_count
|
||||
from .interfaces import IPoller, IEggStorage, IEnvironment
|
||||
|
||||
class Launcher(Service):
|
||||
|
||||
def __init__(self, config, app):
|
||||
self.max_proc = config.getint('max_proc', 0) or cpu_count()
|
||||
self.egg_runner = config.get('egg_runner', 'scrapyd.eggrunner')
|
||||
self.app = app
|
||||
|
||||
def startService(self):
|
||||
for slot in range(self.max_proc):
|
||||
self._wait_for_project(slot)
|
||||
log.msg("Launcher started: max_proc=%r, egg_runner=%r" % \
|
||||
(self.max_proc, self.egg_runner), system="Launcher")
|
||||
|
||||
def _wait_for_project(self, slot):
|
||||
poller = self.app.getComponent(IPoller)
|
||||
poller.next().addCallback(self._spawn_process, slot)
|
||||
|
||||
def _get_eggpath(self, project):
|
||||
eggstorage = self.app.getComponent(IEggStorage)
|
||||
version, eggf = eggstorage.get(project)
|
||||
prefix = '%s-%s-' % (project, version)
|
||||
fd, eggpath = mkstemp(prefix=prefix, suffix='.egg')
|
||||
lf = os.fdopen(fd, 'wb')
|
||||
copyfileobj(eggf, lf)
|
||||
lf.close()
|
||||
return eggpath
|
||||
|
||||
def _spawn_process(self, message, slot):
|
||||
project = message['project']
|
||||
eggpath = self._get_eggpath(project)
|
||||
args = [sys.executable, '-m', self.egg_runner, eggpath, 'crawl']
|
||||
e = self.app.getComponent(IEnvironment)
|
||||
env = e.get_environment(message, slot)
|
||||
pp = ScrapyProcessProtocol(eggpath, slot)
|
||||
pp.deferred.addBoth(self._process_finished, eggpath, slot)
|
||||
reactor.spawnProcess(pp, sys.executable, args=args, env=env)
|
||||
|
||||
def _process_finished(self, _, eggpath, slot):
|
||||
os.remove(eggpath)
|
||||
self._wait_for_project(slot)
|
||||
|
||||
|
||||
class ScrapyProcessProtocol(protocol.ProcessProtocol):
|
||||
|
||||
def __init__(self, eggfile, slot):
|
||||
self.eggfile = eggfile
|
||||
self.slot = slot
|
||||
self.pid = None
|
||||
self.deferred = defer.Deferred()
|
||||
|
||||
def outReceived(self, data):
|
||||
log.msg(data.rstrip(), system="Launcher,%d/stdout" % self.pid)
|
||||
|
||||
def errReceived(self, data):
|
||||
log.msg(data.rstrip(), system="Launcher,%d/stderr" % self.pid)
|
||||
|
||||
def connectionMade(self):
|
||||
self.pid = self.transport.pid
|
||||
self.log("Process started: ")
|
||||
|
||||
def processEnded(self, status):
|
||||
if isinstance(status.value, error.ProcessDone):
|
||||
self.log("Process finished: ")
|
||||
else:
|
||||
self.log("Process died: exitstatus=%r " % status.value.exitCode)
|
||||
self.deferred.callback(self)
|
||||
|
||||
def log(self, msg):
|
||||
msg += "slot=%r pid=%r egg=%r" % (self.slot, self.pid, self.eggfile)
|
||||
log.msg(msg, system="Launcher")
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
from zope.interface import implements
|
||||
from twisted.internet.defer import DeferredQueue
|
||||
|
||||
from .utils import get_spider_queues
|
||||
from .interfaces import IPoller
|
||||
|
||||
class QueuePoller(object):
|
||||
|
||||
implements(IPoller)
|
||||
|
||||
def __init__(self, config):
|
||||
self.eggs_dir = config.get('eggs_dir', 'eggs')
|
||||
self.dbs_dir = config.get('dbs_dir', 'dbs')
|
||||
self.update_projects()
|
||||
self.dq = DeferredQueue(size=1)
|
||||
|
||||
def poll(self):
|
||||
if self.dq.pending:
|
||||
return
|
||||
for p, q in self.queues.iteritems():
|
||||
if q.count():
|
||||
return self.dq.put(self._message(p))
|
||||
|
||||
def next(self):
|
||||
return self.dq.get()
|
||||
|
||||
def update_projects(self):
|
||||
self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir)
|
||||
|
||||
def _message(self, project):
|
||||
return {'project': str(project)}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
from zope.interface import implements
|
||||
|
||||
from .interfaces import ISpiderScheduler
|
||||
from .utils import get_spider_queues
|
||||
|
||||
class SpiderScheduler(object):
|
||||
|
||||
implements(ISpiderScheduler)
|
||||
|
||||
def __init__(self, config):
|
||||
self.eggs_dir = config.get('eggs_dir', 'eggs')
|
||||
self.dbs_dir = config.get('dbs_dir', 'dbs')
|
||||
self.update_projects()
|
||||
|
||||
def schedule(self, project, spider_name, **spider_args):
|
||||
q = self.queues[project]
|
||||
q.add(spider_name, **spider_args)
|
||||
|
||||
def list_projects(self):
|
||||
return self.queues.keys()
|
||||
|
||||
def update_projects(self):
|
||||
self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir)
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
import os
|
||||
import pkg_resources
|
||||
|
||||
from scrapy.spiderqueue import SqliteSpiderQueue
|
||||
|
||||
def get_spider_queues(eggsdir, dbsdir):
|
||||
"""Return a dict of Spider Quees keyed by project name"""
|
||||
d = {}
|
||||
for project in os.listdir(eggsdir):
|
||||
dbpath = os.path.join(dbsdir, '%s.db' % project)
|
||||
d[project] = SqliteSpiderQueue(dbpath)
|
||||
return d
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
import cgi
|
||||
import traceback
|
||||
from cStringIO import StringIO
|
||||
|
||||
from twisted.web.resource import Resource
|
||||
|
||||
from scrapy.utils.txweb import JsonResource
|
||||
from .interfaces import IPoller, IEggStorage, ISpiderScheduler
|
||||
from .eggutils import get_spider_list_from_eggfile
|
||||
|
||||
class WsResource(JsonResource):
|
||||
|
||||
def __init__(self, root):
|
||||
JsonResource.__init__(self)
|
||||
self.root = root
|
||||
|
||||
def render(self, txrequest):
|
||||
try:
|
||||
return JsonResource.render(self, txrequest)
|
||||
except Exception, e:
|
||||
if self.root.debug:
|
||||
return traceback.format_exc()
|
||||
r = {"status": "error", "message": str(e)}
|
||||
return self.render_object(r, txrequest)
|
||||
|
||||
class Schedule(WsResource):
|
||||
|
||||
def render_POST(self, txrequest):
|
||||
args = dict((k, v[0]) for k, v in txrequest.args.items())
|
||||
project = args.pop('project')
|
||||
spider = args.pop('spider')
|
||||
sched = self.root.app.getComponent(ISpiderScheduler)
|
||||
sched.schedule(project, spider, **args)
|
||||
return {"status": "ok"}
|
||||
|
||||
class AddVersion(WsResource):
|
||||
|
||||
def render_POST(self, txrequest):
|
||||
ct = txrequest.requestHeaders.getRawHeaders('Content-Type')[0]
|
||||
boundary = ct.split('boundary=', 1)[1]
|
||||
d = cgi.parse_multipart(txrequest.content, {'boundary': boundary})
|
||||
project = d['project'][0]
|
||||
version = d['version'][0]
|
||||
eggf = StringIO(d['egg'][0])
|
||||
spiders = get_spider_list_from_eggfile(eggf, project)
|
||||
eggstorage = self.root.app.getComponent(IEggStorage)
|
||||
eggstorage.put(eggf, project, version)
|
||||
self.root.update_projects()
|
||||
return {"status": "ok", "spiders": spiders}
|
||||
|
||||
class ListProjects(WsResource):
|
||||
|
||||
def render_GET(self, txrequest):
|
||||
projects = self.root.app.getComponent(ISpiderScheduler).list_projects()
|
||||
return {"status": "ok", "projects": projects}
|
||||
|
||||
class ListVersions(WsResource):
|
||||
|
||||
def render_GET(self, txrequest):
|
||||
project = txrequest.args['project'][0]
|
||||
eggstorage = self.root.app.getComponent(IEggStorage)
|
||||
versions = eggstorage.list(project)
|
||||
return {"status": "ok", "versions": versions}
|
||||
|
||||
class ListSpiders(WsResource):
|
||||
|
||||
def render_GET(self, txrequest):
|
||||
project = txrequest.args['project'][0]
|
||||
eggstorage = self.root.app.getComponent(IEggStorage)
|
||||
_, eggf = eggstorage.get(project)
|
||||
spiders = get_spider_list_from_eggfile(eggf, project)
|
||||
return {"status": "ok", "spiders": spiders}
|
||||
|
||||
class DeleteProject(WsResource):
|
||||
|
||||
def render_POST(self, txrequest):
|
||||
project = txrequest.args['project'][0]
|
||||
self._delete_version(project)
|
||||
return {"status": "ok"}
|
||||
|
||||
def _delete_version(self, project, version=None):
|
||||
eggstorage = self.root.app.getComponent(IEggStorage)
|
||||
eggstorage.delete(project, version)
|
||||
self.root.update_projects()
|
||||
|
||||
class DeleteVersion(DeleteProject):
|
||||
|
||||
def render_POST(self, txrequest):
|
||||
project = txrequest.args['project'][0]
|
||||
version = txrequest.args['version'][0]
|
||||
self._delete_version(project, version)
|
||||
return {"status": "ok"}
|
||||
|
||||
class Root(Resource):
|
||||
|
||||
def __init__(self, config, app):
|
||||
Resource.__init__(self)
|
||||
self.debug = config.getboolean('debug', False)
|
||||
self.app = app
|
||||
self.putChild('schedule.json', Schedule(self))
|
||||
self.putChild('addversion.json', AddVersion(self))
|
||||
self.putChild('listprojects.json', ListProjects(self))
|
||||
self.putChild('listversions.json', ListVersions(self))
|
||||
self.putChild('listspiders.json', ListSpiders(self))
|
||||
self.putChild('delproject.json', DeleteProject(self))
|
||||
self.putChild('delversion.json', DeleteVersion(self))
|
||||
self.update_projects()
|
||||
|
||||
def update_projects(self):
|
||||
self.app.getComponent(IPoller).update_projects()
|
||||
self.app.getComponent(ISpiderScheduler).update_projects()
|
||||
24
setup.py
24
setup.py
|
|
@ -51,22 +51,22 @@ packages, data_files = [], []
|
|||
root_dir = os.path.dirname(__file__)
|
||||
if root_dir != '':
|
||||
os.chdir(root_dir)
|
||||
scrapy_dir = 'scrapy'
|
||||
|
||||
def is_not_module(filename):
|
||||
return os.path.splitext(f)[1] not in ['.py', '.pyc', '.pyo']
|
||||
|
||||
for dirpath, dirnames, filenames in os.walk(scrapy_dir):
|
||||
# Ignore dirnames that start with '.'
|
||||
for i, dirname in enumerate(dirnames):
|
||||
if dirname.startswith('.'): del dirnames[i]
|
||||
if '__init__.py' in filenames:
|
||||
packages.append('.'.join(fullsplit(dirpath)))
|
||||
data = [f for f in filenames if is_not_module(f)]
|
||||
if data:
|
||||
data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]])
|
||||
elif filenames:
|
||||
data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]])
|
||||
for scrapy_dir in ['scrapy', 'scrapyd']:
|
||||
for dirpath, dirnames, filenames in os.walk(scrapy_dir):
|
||||
# Ignore dirnames that start with '.'
|
||||
for i, dirname in enumerate(dirnames):
|
||||
if dirname.startswith('.'): del dirnames[i]
|
||||
if '__init__.py' in filenames:
|
||||
packages.append('.'.join(fullsplit(dirpath)))
|
||||
data = [f for f in filenames if is_not_module(f)]
|
||||
if data:
|
||||
data_files.append([dirpath, [os.path.join(dirpath, f) for f in data]])
|
||||
elif filenames:
|
||||
data_files.append([dirpath, [os.path.join(dirpath, f) for f in filenames]])
|
||||
|
||||
# Small hack for working with bdist_wininst.
|
||||
# See http://mail.python.org/pipermail/distutils-sig/2004-August/004134.html
|
||||
|
|
|
|||
Loading…
Reference in New Issue