From 5db45b382522be9f9b77009892ec304fb2d83018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 6 Feb 2013 05:05:06 +0000 Subject: [PATCH] remove scrapyd, it was migrated to its own repository --- .travis.yml | 2 +- MANIFEST.in | 2 - Makefile.buildbot | 18 ++- bin/runtests.sh | 2 +- bin/scrapyd | 14 -- debian/control | 11 -- debian/rules | 3 - debian/scrapy.install | 2 - debian/scrapyd-files/000-default | 8 -- debian/scrapyd.default | 1 - debian/scrapyd.dirs | 5 - debian/scrapyd.install | 2 - debian/scrapyd.lintian-overrides | 2 - debian/scrapyd.postinst | 34 ----- debian/scrapyd.postrm | 17 --- debian/scrapyd.scrapyd.upstart | 13 -- docs/topics/commands.rst | 18 --- extras/makedeb.py | 5 +- extras/scrapyd.tac | 2 - extras/test-scrapyd.sh | 122 ---------------- scrapy/commands/server.py | 22 --- scrapyd/__init__.py | 9 -- scrapyd/app.py | 44 ------ scrapyd/config.py | 62 -------- scrapyd/default_scrapyd.conf | 25 ---- scrapyd/eggstorage.py | 49 ------- scrapyd/eggutils.py | 14 -- scrapyd/environ.py | 46 ------ scrapyd/interfaces.py | 109 -------------- scrapyd/launcher.py | 102 ------------- scrapyd/poller.py | 36 ----- scrapyd/runner.py | 39 ----- scrapyd/scheduler.py | 22 --- scrapyd/script.py | 42 ------ scrapyd/spiderqueue.py | 33 ----- scrapyd/sqlite.py | 172 ---------------------- scrapyd/tests/__init__.py | 0 scrapyd/tests/mybot.egg | Bin 5129 -> 0 bytes scrapyd/tests/test_dont_load_settings.py | 22 --- scrapyd/tests/test_eggstorage.py | 43 ------ scrapyd/tests/test_environ.py | 45 ------ scrapyd/tests/test_poller.py | 41 ------ scrapyd/tests/test_scheduler.py | 44 ------ scrapyd/tests/test_spiderqueue.py | 66 --------- scrapyd/tests/test_sqlite.py | 173 ----------------------- scrapyd/tests/test_utils.py | 51 ------- scrapyd/utils.py | 67 --------- scrapyd/webservice.py | 122 ---------------- scrapyd/website.py | 131 ----------------- setup.py | 6 +- tox.ini | 2 +- 51 files changed, 22 insertions(+), 1900 deletions(-) delete mode 100755 bin/scrapyd delete mode 100644 debian/scrapyd-files/000-default delete mode 100644 debian/scrapyd.default delete mode 100644 debian/scrapyd.dirs delete mode 100644 debian/scrapyd.install delete mode 100644 debian/scrapyd.lintian-overrides delete mode 100644 debian/scrapyd.postinst delete mode 100644 debian/scrapyd.postrm delete mode 100644 debian/scrapyd.scrapyd.upstart delete mode 100644 extras/scrapyd.tac delete mode 100755 extras/test-scrapyd.sh delete mode 100644 scrapy/commands/server.py delete mode 100644 scrapyd/__init__.py delete mode 100644 scrapyd/app.py delete mode 100644 scrapyd/config.py delete mode 100644 scrapyd/default_scrapyd.conf delete mode 100644 scrapyd/eggstorage.py delete mode 100644 scrapyd/eggutils.py delete mode 100644 scrapyd/environ.py delete mode 100644 scrapyd/interfaces.py delete mode 100644 scrapyd/launcher.py delete mode 100644 scrapyd/poller.py delete mode 100644 scrapyd/runner.py delete mode 100644 scrapyd/scheduler.py delete mode 100644 scrapyd/script.py delete mode 100644 scrapyd/spiderqueue.py delete mode 100644 scrapyd/sqlite.py delete mode 100644 scrapyd/tests/__init__.py delete mode 100644 scrapyd/tests/mybot.egg delete mode 100644 scrapyd/tests/test_dont_load_settings.py delete mode 100644 scrapyd/tests/test_eggstorage.py delete mode 100644 scrapyd/tests/test_environ.py delete mode 100644 scrapyd/tests/test_poller.py delete mode 100644 scrapyd/tests/test_scheduler.py delete mode 100644 scrapyd/tests/test_spiderqueue.py delete mode 100644 scrapyd/tests/test_sqlite.py delete mode 100644 scrapyd/tests/test_utils.py delete mode 100644 scrapyd/utils.py delete mode 100644 scrapyd/webservice.py delete mode 100644 scrapyd/website.py diff --git a/.travis.yml b/.travis.yml index 2e0a82b27..dd93fba70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ env: install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt script: - - trial scrapy scrapyd + - trial scrapy notifications: - irc: - "irc.freenode.org#scrapy" diff --git a/MANIFEST.in b/MANIFEST.in index e49beddea..dfa3d15df 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,8 +4,6 @@ include INSTALL include LICENSE include MANIFEST.in include scrapy/mime.types -include scrapyd/default_scrapyd.conf -recursive-include scrapyd/tests *.egg recursive-include scrapy/templates * recursive-include scrapy/tests/sample_data * recursive-include scrapy license.txt diff --git a/Makefile.buildbot b/Makefile.buildbot index 61e40b2b0..611260b2a 100644 --- a/Makefile.buildbot +++ b/Makefile.buildbot @@ -1,8 +1,18 @@ TRIAL := $(shell which trial) +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +ifeq ($(BRANCH),master) +export SCRAPY_VERSION_FROM_GIT=1 +endif +export PYTHONPATH=$(PWD) -deb: +test: + coverage run --branch $(TRIAL) --reporter=text scrapy.tests + rm -rf htmlcov && coverage html + -s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/ + +build: python extras/makedeb.py build -deb-clean: - debclean - python extras/makedeb.py clean +clean: + git checkout debian scrapy/__init__.py + git clean -dfq diff --git a/bin/runtests.sh b/bin/runtests.sh index 0f6a309f7..0e7a318b9 100755 --- a/bin/runtests.sh +++ b/bin/runtests.sh @@ -37,7 +37,7 @@ fi find . -name '*.py[co]' -delete if [ $# -eq 0 ]; then - $trial --reporter=text scrapy scrapyd + $trial --reporter=text scrapy else $trial "$@" fi diff --git a/bin/scrapyd b/bin/scrapyd deleted file mode 100755 index fe440cbb5..000000000 --- a/bin/scrapyd +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -repotac=$(cd $(dirname $0)/../extras; pwd)/scrapyd.tac - -if [ -f "$repotac" ]; then - tacfile="$repotac" -elif [ -f "/usr/share/scrapyd/scrapyd.tac" ]; then - tacfile="/usr/share/scrapyd/scrapyd.tac" -else - echo "Unable to find scrapy.tac file" - exit 1 -fi - -twistd -ny "$tacfile" diff --git a/debian/control b/debian/control index 43d48859f..fcf2b5b93 100644 --- a/debian/control +++ b/debian/control @@ -17,14 +17,3 @@ Description: Python web crawling and scraping framework used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. - -Package: scrapyd-SUFFIX -Architecture: all -Depends: scrapy-SUFFIX (>= ${source:Version}), python-setuptools -Conflicts: scrapyd, scrapyd-0.11 -Provides: scrapyd -Description: Scrapy Service - The Scrapy service allows you to deploy your Scrapy projects by building - Python eggs of them and uploading them to the Scrapy service using a JSON API - that you can also use for scheduling spider runs. It supports multiple - projects also. diff --git a/debian/rules b/debian/rules index 3026acd0e..b8796e6e3 100755 --- a/debian/rules +++ b/debian/rules @@ -3,6 +3,3 @@ %: dh $@ - -override_dh_installinit: - dh_installinit --name scrapyd diff --git a/debian/scrapy.install b/debian/scrapy.install index 2ccb5ce19..5977d5f43 100644 --- a/debian/scrapy.install +++ b/debian/scrapy.install @@ -1,3 +1 @@ -usr/lib/python*/*-packages/scrapy* -usr/bin extras/scrapy_bash_completion etc/bash_completion.d/ diff --git a/debian/scrapyd-files/000-default b/debian/scrapyd-files/000-default deleted file mode 100644 index e3afc8f54..000000000 --- a/debian/scrapyd-files/000-default +++ /dev/null @@ -1,8 +0,0 @@ -[scrapyd] -http_port = 6800 -debug = off -#max_proc = 1 -eggs_dir = /var/lib/scrapyd/eggs -dbs_dir = /var/lib/scrapyd/dbs -items_dir = /var/lib/scrapyd/items -logs_dir = /var/log/scrapyd diff --git a/debian/scrapyd.default b/debian/scrapyd.default deleted file mode 100644 index f71fdb569..000000000 --- a/debian/scrapyd.default +++ /dev/null @@ -1 +0,0 @@ -# Defaults for Scrapy service diff --git a/debian/scrapyd.dirs b/debian/scrapyd.dirs deleted file mode 100644 index 31ae26fdb..000000000 --- a/debian/scrapyd.dirs +++ /dev/null @@ -1,5 +0,0 @@ -var/lib/scrapyd -var/lib/scrapyd/eggs -var/lib/scrapyd/dbs -var/lib/scrapyd/items -var/log/scrapyd diff --git a/debian/scrapyd.install b/debian/scrapyd.install deleted file mode 100644 index f46d4e3bd..000000000 --- a/debian/scrapyd.install +++ /dev/null @@ -1,2 +0,0 @@ -debian/scrapyd-files/000-default etc/scrapyd/conf.d -extras/scrapyd.tac usr/share/scrapyd diff --git a/debian/scrapyd.lintian-overrides b/debian/scrapyd.lintian-overrides deleted file mode 100644 index 52f5e4cb7..000000000 --- a/debian/scrapyd.lintian-overrides +++ /dev/null @@ -1,2 +0,0 @@ -new-package-should-close-itp-bug -script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapyd diff --git a/debian/scrapyd.postinst b/debian/scrapyd.postinst deleted file mode 100644 index db0ba9589..000000000 --- a/debian/scrapyd.postinst +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -set -e - -case "$1" in - configure) - # Create user to run the service as - if [ -z "`id -u scrapy 2> /dev/null`" ]; then - adduser --system --home /var/lib/scrapyd --gecos "scrapy" \ - --no-create-home --disabled-password \ - --quiet scrapy || true - fi - - chown scrapy:nogroup /var/log/scrapyd \ - /var/lib/scrapyd /var/lib/scrapyd/eggs /var/lib/scrapyd/dbs \ - /var/lib/scrapyd/items - - update-python-modules -p # so upstart restart uses the new code - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -#DEBHELPER# - -exit 0 - - diff --git a/debian/scrapyd.postrm b/debian/scrapyd.postrm deleted file mode 100644 index 5f114be46..000000000 --- a/debian/scrapyd.postrm +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -set -e - -if [ purge = "$1" ]; then - if [ -x "$(command -v deluser)" ]; then - deluser --quiet --system scrapy > /dev/null || true - else - echo >&2 "not removing scrapy system account because deluser command was not found" - fi -fi - -#DEBHELPER# - -exit 0 - - diff --git a/debian/scrapyd.scrapyd.upstart b/debian/scrapyd.scrapyd.upstart deleted file mode 100644 index 5a53d7086..000000000 --- a/debian/scrapyd.scrapyd.upstart +++ /dev/null @@ -1,13 +0,0 @@ -# Scrapy service - -start on runlevel [2345] -stop on runlevel [06] - -script - [ -r /etc/default/scrapyd ] && . /etc/default/scrapyd - logdir=/var/log/scrapyd - exec twistd -ny /usr/share/scrapyd/scrapyd.tac \ - -u scrapy -g nogroup \ - --pidfile /var/run/scrapyd.pid \ - -l $logdir/scrapyd.log >$logdir/scrapyd.out 2>$logdir/scrapyd.err -end script diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index c4941da08..bb95720a7 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -249,24 +249,6 @@ Usage examples:: [FAILED] first_spider:parse >>> Returned 92 requests, expected 0..4 -.. command:: server - -server ------- - -* Syntax: ``scrapy server`` -* Requires project: *yes* - -Start Scrapyd server for this project, which can be referred from the JSON API -with the project name ``default``. For more info see: :ref:`topics-scrapyd`. - -Usage example:: - - $ scrapy server - [ ... scrapyd starts and stays idle waiting for spiders to get scheduled ... ] - -To schedule spiders, use the Scrapyd JSON API. - .. command:: list list diff --git a/extras/makedeb.py b/extras/makedeb.py index 08a02645f..14ed4dcd6 100644 --- a/extras/makedeb.py +++ b/extras/makedeb.py @@ -2,7 +2,7 @@ import sys, os, glob, shutil from subprocess import check_call def build(suffix): - for ifn in glob.glob("debian/scrapy.*") + glob.glob("debian/scrapyd.*"): + for ifn in glob.glob("debian/scrapy.*"): s = open(ifn).read() s = s.replace('SUFFIX', suffix) pre, suf = ifn.split('.', 1) @@ -21,8 +21,7 @@ def build(suffix): check_call('debuild -us -uc -b', shell=True) def clean(suffix): - for f in glob.glob("debian/python-scrapy%s*" % suffix) + \ - glob.glob("debian/scrapyd%s*" % suffix): + for f in glob.glob("debian/python-scrapy%s*" % suffix): if os.path.isdir(f): shutil.rmtree(f) else: diff --git a/extras/scrapyd.tac b/extras/scrapyd.tac deleted file mode 100644 index 6abce3086..000000000 --- a/extras/scrapyd.tac +++ /dev/null @@ -1,2 +0,0 @@ -from scrapyd import get_application -application = get_application() diff --git a/extras/test-scrapyd.sh b/extras/test-scrapyd.sh deleted file mode 100755 index 5e9080250..000000000 --- a/extras/test-scrapyd.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# -# This script is a quick system test for Scrapyd that: -# -# 1. runs scrapyd -# 2. creates a new project and deploys it on scrapyd -# 3. schedules a spider on scrapyd and waits for it to finish -# 4. check the spider scraped the expected data -# - -set -e - -export PATH=$PATH:$(pwd)/bin -export PYTHONPATH=$PYTHONPATH:$(pwd) - -scrapyd_dir=$(mktemp /tmp/test-scrapyd.XXXXXXX -d) -scrapyd_log=$scrapyd_dir/scrapyd.log -scrapy_dir=$(mktemp /tmp/test-scrapy.XXXXXXX -d) - -echo "scrapyd dir: $scrapyd_dir" -echo "scrapy dir : $scrapy_dir" - -twistd -ny extras/scrapyd.tac -d $scrapyd_dir -l $scrapyd_log & - -cd $scrapy_dir -scrapy startproject testproj -cd testproj -cat > testproj/spiders/insophia.py < scrapy.cfg <~i?^_%i!C{7il@!msf$n}>cq#w=s0T&Z)7sGC!`a5EjxIv$!u&_` zwU*NaQcu%y1~T7a?(M(lae}!ppfm#2Sj~) zJ`tKR^xe}!iB(W)0t z>S!QmKN0yyO{7fck%TL|J#}P@31QI8nJ0;9I?RDw%1%wu;T~Lz#rJa?XI6Q_!!n*5 zPi=_yiB@U&tlosPPSR)*5d9+7EN4mv-3pR%W4De*)Z}~LRx71ngPvPNPOXb$W$%qy z67Sr4WE6)}Y8WYPN!S?LMRtr-QjGrOTz~^kc>#<4r(<4dJl@6Kc`Nj02^ig071$LK z9ga%WZdI)DEFT9QViP7QdzW_c;2VvDK{OsOVcN<#xu-U5Onw6Q-!WvBn0eSAcO8XAIY5g!j#^Lo@xO-|xXPD%n{c%{j39kR-z&{-Z~{ z_un!`-FflwGoiUY^L1Evpa-V-fhcmSq);b&u5(7MFeEErD8V_y@kVqkCjEIt`jK5& z;j=-vHXABmuGn4Uwc1_R$M#|_?ZdY(p+oBK7ZTOA4nBy`uAo^C%~5byh!Y|xwAp%@ zg&TT|qKMRF2T6xyRtc()2Aqr7;h?Rpi@OWn)|ONs>Q|%EdU{JoMKY)vxk6r1N5-59 z)?kFFm(gpt89{`uo`TO)N97IlX2Cb|l6&8=H)i!V3^2Z7WUBC!=GYiq!f%Xr=dH7E ztgiMp^d_zPHHLPxvu1PN7+p^u$U3c%KxpR=qo`OiiS}dkz^SFBp`zjd4Es^Bz^Xfz z7^xbzO2DiO%vAO$;-=^x8rvGTQ|A;nsznT#29|3#lw3>dDCR9nsU_@Nr4oN9))0J7 z;p48teBQvudg%(~=2pVe_xYmmPfLdn!9QJ5f$%@F3g%p3SfDPmHc5IUFsY<0N&WWMk`cl=qC%$4|ozoZk1b^&n1_LGweA6~(+mTUX_1mUUvk;{Bx z{L&ksSdtE_pEK6f2#}2%kd1T={g4fm@MZ~bT*D;b?PEj>>%;9V(eC#n6l-O9)rL$) zKH4*v^NcVPRXuwan#YFzh4F`ec|RjR<3ywRQFJ71ZzSroVr56joG(4$1QBuZJcnF} zaFLDZ$YU!9WyncwHMB^!9FJI$BG;R*c}&I~f-u+p<~-QG`kU8yo|$*Dsz=tl=doZu z(VH2nIL_T>=h1}VMW!9Y?0I1yPe#uwEjfOCT6;OXqx-ei-k62p;(bAnd6%Z4-uXqs zPb~kWcbhU{CNNq#!`GP^T7CJLALpiII^-)gGof4M*OR%PHDK^Ja~qtjGGohUE}1%c zbWbpZeqjkS37DA|cwC`+nVFBKBf921;Uzj#yjsjAVbi}w9g7ysVr#POP~J4L=n-q> zyu^*QxFI#XP=8EpPG{nKxw`oj>d) zXA**GpCTYQ6(Ic9LslP@Nf6K}M*!g}iOibU0RC> zV%_~_hqt=EK`~^Pz77J_)r;s77d64@9yMg4kHL8e&j@ku6;ts$#RKIw5JMvA_`87K z!mP~9_-eXcULPH3XHwELC}I)Iz>X!P!}2#fJC?+6^#+>FEQmeqX7RhtE(+vudYhQ- zacvVglMVGvx>`?Ah4f1$pG+lcQS;A)Z zDN*A&e3iyCBqR7K`zgDF*$S_y#~#h-tNA9<9y?zn!V{8RICuS;sdLi6#fqYUn;2@Bnrrta-K%~W zCxtn;XWGXMd#n1splm=~uP65&gX&U&evh=vT&V`ViV`n$lFrJ1Rb*a0-=x!XXz6Xb zWXZRSv&*-!o35YV)kN&_rzgjiB$m=H1{4MmDHd2fzla5j??0%NH~vwjd>7iAJecc1 zQ8CSbR!q8feh4V_Nk+0)_nlY)jQhKYN|LHMIieAaQN^8+37OC$&2vDz$5XB?hu9}Z zKt9M+3FnBqHzJLrHXOXWL?glRe3`~_dF9%MB6fX4dVj-#s~&-DOir}&fCWh zp@+rKwMX42#AgM;@vGyR8b#h#T=`a^sPSu zh78r$fzo7!jJW2>Y56LXJ0T*(3v+F1k^c6lbML4bBUW4%KlGwyj?U*<=iW69YNs_A zS)eo4qo#pF{`>R_c(?WBg^*r9M@Y(9mXZcMg^^0=_LCPvs*yiwwr5vLIOXU{hDU+n zph1?BfpUZ-GpGZl@}~hi21-f}uoI+Fa{I{(2?BHcZm^`JqMR$qRE=P&?J1KIPC59I z;b(!1ied}|gMT*`k--7L2ZfzrFqTw}C?^jx)|7S|3u;Ro!T8^G-DG?k-8LTF4TABc z{VAJTGTsmPIPk-EP_FIRpMLWJ_?2ac6eJqT>yu2A$wWy5 z^63{I${UW%qs&an^TVvI`wr~Y?@B+J2E+2-X}~PME8}FAnO|5yP8sXbkv>iV-__t$ N$AFXEK>FT5^=}bQL+1bh diff --git a/scrapyd/tests/test_dont_load_settings.py b/scrapyd/tests/test_dont_load_settings.py deleted file mode 100644 index cefa987db..000000000 --- a/scrapyd/tests/test_dont_load_settings.py +++ /dev/null @@ -1,22 +0,0 @@ -import sys -import unittest - -class SettingsSafeModulesTest(unittest.TestCase): - - # these modules must not load scrapy.conf - SETTINGS_SAFE_MODULES = [ - 'scrapy.utils.project', - 'scrapy.utils.conf', - 'scrapyd.interfaces', - 'scrapyd.eggutils', - ] - - def test_modules_that_shouldnt_load_settings(self): - sys.modules.pop('scrapy.conf', None) - for m in self.SETTINGS_SAFE_MODULES: - __import__(m) - assert 'scrapy.conf' not in sys.modules, \ - "Module %r must not cause the scrapy.conf module to be loaded" % m - -if __name__ == "__main__": - unittest.main() diff --git a/scrapyd/tests/test_eggstorage.py b/scrapyd/tests/test_eggstorage.py deleted file mode 100644 index 311f32fee..000000000 --- a/scrapyd/tests/test_eggstorage.py +++ /dev/null @@ -1,43 +0,0 @@ -from cStringIO import StringIO - -from twisted.trial import unittest - -from zope.interface.verify import verifyObject - -from scrapyd.interfaces import IEggStorage -from scrapyd.config import Config -from scrapyd.eggstorage import FilesystemEggStorage - -class EggStorageTest(unittest.TestCase): - - def setUp(self): - d = self.mktemp() - config = Config(values={'eggs_dir': d}) - self.eggst = FilesystemEggStorage(config) - - def test_interface(self): - verifyObject(IEggStorage, self.eggst) - - def test_put_get_list_delete(self): - self.eggst.put(StringIO("egg01"), 'mybot', '01') - self.eggst.put(StringIO("egg03"), 'mybot', '03') - self.eggst.put(StringIO("egg02"), 'mybot', '02') - - self.assertEqual(self.eggst.list('mybot'), ['01', '02', '03']) - self.assertEqual(self.eggst.list('mybot2'), []) - - v, f = self.eggst.get('mybot') - self.assertEqual(v, "03") - self.assertEqual(f.read(), "egg03") - f.close() - - v, f = self.eggst.get('mybot', '02') - self.assertEqual(v, "02") - self.assertEqual(f.read(), "egg02") - f.close() - - self.eggst.delete('mybot', '02') - self.assertEqual(self.eggst.list('mybot'), ['01', '03']) - - self.eggst.delete('mybot') - self.assertEqual(self.eggst.list('mybot'), []) diff --git a/scrapyd/tests/test_environ.py b/scrapyd/tests/test_environ.py deleted file mode 100644 index 25d81c0b0..000000000 --- a/scrapyd/tests/test_environ.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -from twisted.trial import unittest - -from zope.interface.verify import verifyObject - -from scrapyd.interfaces import IEnvironment -from scrapyd.config import Config -from scrapyd.environ import Environment - -class EnvironmentTest(unittest.TestCase): - - def setUp(self): - d = self.mktemp() - os.mkdir(d) - config = Config(values={'eggs_dir': d, 'logs_dir': d}) - config.cp.add_section('settings') - config.cp.set('settings', 'newbot', 'newbot.settings') - self.environ = Environment(config, initenv={}) - - def test_interface(self): - verifyObject(IEnvironment, self.environ) - - def test_get_environment_with_eggfile(self): - msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'} - slot = 3 - env = self.environ.get_environment(msg, slot) - self.assertEqual(env['SCRAPY_PROJECT'], 'mybot') - self.assertEqual(env['SCRAPY_SLOT'], '3') - self.assertEqual(env['SCRAPY_SPIDER'], 'myspider') - self.assertEqual(env['SCRAPY_JOB'], 'ID') - self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log'))) - self.assert_(env['SCRAPY_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl'))) - self.failIf('SCRAPY_SETTINGS_MODULE' in env) - - def test_get_environment_with_no_items_dir(self): - config = Config(values={'items_dir': '', 'logs_dir': ''}) - config.cp.add_section('settings') - config.cp.set('settings', 'newbot', 'newbot.settings') - msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'} - slot = 3 - environ = Environment(config, initenv={}) - env = environ.get_environment(msg, slot) - self.failUnless('SCRAPY_FEED_URI' not in env) - self.failUnless('SCRAPY_LOG_FILE' not in env) diff --git a/scrapyd/tests/test_poller.py b/scrapyd/tests/test_poller.py deleted file mode 100644 index 32291d3e8..000000000 --- a/scrapyd/tests/test_poller.py +++ /dev/null @@ -1,41 +0,0 @@ -import os - -from twisted.trial import unittest -from twisted.internet.defer import Deferred - -from zope.interface.verify import verifyObject - -from scrapyd.interfaces import IPoller -from scrapyd.config import Config -from scrapyd.poller import QueuePoller -from scrapyd.utils import get_spider_queues - -class QueuePollerTest(unittest.TestCase): - - def setUp(self): - d = self.mktemp() - eggs_dir = os.path.join(d, 'eggs') - dbs_dir = os.path.join(d, 'dbs') - os.makedirs(eggs_dir) - os.makedirs(dbs_dir) - os.makedirs(os.path.join(eggs_dir, 'mybot1')) - os.makedirs(os.path.join(eggs_dir, 'mybot2')) - config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir}) - self.queues = get_spider_queues(config) - self.poller = QueuePoller(config) - - def test_interface(self): - verifyObject(IPoller, self.poller) - - def test_poll_next(self): - self.queues['mybot1'].add('spider1') - self.queues['mybot2'].add('spider2') - d1 = self.poller.next() - d2 = self.poller.next() - self.failUnless(isinstance(d1, Deferred)) - self.failIf(hasattr(d1, 'result')) - self.poller.poll() - self.queues['mybot1'].pop() - self.poller.poll() - self.failUnlessEqual(d1.result, {'_project': 'mybot1', '_spider': 'spider1'}) - self.failUnlessEqual(d2.result, {'_project': 'mybot2', '_spider': 'spider2'}) diff --git a/scrapyd/tests/test_scheduler.py b/scrapyd/tests/test_scheduler.py deleted file mode 100644 index ce655966d..000000000 --- a/scrapyd/tests/test_scheduler.py +++ /dev/null @@ -1,44 +0,0 @@ -import os - -from twisted.trial import unittest - -from zope.interface.verify import verifyObject - -from scrapyd.interfaces import ISpiderScheduler -from scrapyd.config import Config -from scrapyd.scheduler import SpiderScheduler -from scrapyd.utils import get_spider_queues - -class SpiderSchedulerTest(unittest.TestCase): - - def setUp(self): - d = self.mktemp() - eggs_dir = self.eggs_dir = os.path.join(d, 'eggs') - dbs_dir = os.path.join(d, 'dbs') - os.mkdir(d) - os.makedirs(eggs_dir) - os.makedirs(dbs_dir) - os.makedirs(os.path.join(eggs_dir, 'mybot1')) - os.makedirs(os.path.join(eggs_dir, 'mybot2')) - config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir}) - self.queues = get_spider_queues(config) - self.sched = SpiderScheduler(config) - - def test_interface(self): - verifyObject(ISpiderScheduler, self.sched) - - def test_list_update_projects(self): - self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2'])) - os.makedirs(os.path.join(self.eggs_dir, 'mybot3')) - self.sched.update_projects() - self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2', 'mybot3'])) - - def test_schedule(self): - q = self.queues['mybot1'] - self.failIf(q.count()) - self.sched.schedule('mybot1', 'myspider1', a='b') - self.sched.schedule('mybot2', 'myspider2', c='d') - self.assertEqual(q.pop(), {'name': 'myspider1', 'a': 'b'}) - q = self.queues['mybot2'] - self.assertEqual(q.pop(), {'name': 'myspider2', 'c': 'd'}) - diff --git a/scrapyd/tests/test_spiderqueue.py b/scrapyd/tests/test_spiderqueue.py deleted file mode 100644 index 8a400a0e9..000000000 --- a/scrapyd/tests/test_spiderqueue.py +++ /dev/null @@ -1,66 +0,0 @@ -from twisted.internet.defer import inlineCallbacks, maybeDeferred -from twisted.trial import unittest - -from zope.interface.verify import verifyObject - -from scrapyd.interfaces import ISpiderQueue -from scrapyd.spiderqueue import SqliteSpiderQueue - -class SpiderQueueTest(unittest.TestCase): - """This test case can be used easily for testing other SpiderQueue's by - just changing the _get_queue() method. It also supports queues with - deferred methods. - """ - - def setUp(self): - self.q = self._get_queue() - self.name = 'spider1' - self.args = {'arg1': 'val1', 'arg2': 2} - self.msg = self.args.copy() - self.msg['name'] = self.name - - def _get_queue(self): - return SqliteSpiderQueue(':memory:') - - def test_interface(self): - verifyObject(ISpiderQueue, self.q) - - @inlineCallbacks - def test_add_pop_count(self): - c = yield maybeDeferred(self.q.count) - self.assertEqual(c, 0) - - yield maybeDeferred(self.q.add, self.name, **self.args) - - c = yield maybeDeferred(self.q.count) - self.assertEqual(c, 1) - - m = yield maybeDeferred(self.q.pop) - self.assertEqual(m, self.msg) - - c = yield maybeDeferred(self.q.count) - self.assertEqual(c, 0) - - @inlineCallbacks - def test_list(self): - l = yield maybeDeferred(self.q.list) - self.assertEqual(l, []) - - yield maybeDeferred(self.q.add, self.name, **self.args) - yield maybeDeferred(self.q.add, self.name, **self.args) - - l = yield maybeDeferred(self.q.list) - self.assertEqual(l, [self.msg, self.msg]) - - @inlineCallbacks - def test_clear(self): - yield maybeDeferred(self.q.add, self.name, **self.args) - yield maybeDeferred(self.q.add, self.name, **self.args) - - c = yield maybeDeferred(self.q.count) - self.assertEqual(c, 2) - - yield maybeDeferred(self.q.clear) - - c = yield maybeDeferred(self.q.count) - self.assertEqual(c, 0) diff --git a/scrapyd/tests/test_sqlite.py b/scrapyd/tests/test_sqlite.py deleted file mode 100644 index 51d62048a..000000000 --- a/scrapyd/tests/test_sqlite.py +++ /dev/null @@ -1,173 +0,0 @@ -import unittest -from datetime import datetime -from decimal import Decimal - -from scrapy.http import Request -from scrapyd.sqlite import SqlitePriorityQueue, JsonSqlitePriorityQueue, \ - PickleSqlitePriorityQueue, SqliteDict, JsonSqliteDict, PickleSqliteDict - - -class SqliteDictTest(unittest.TestCase): - - dict_class = SqliteDict - test_dict = {'hello': 'world', 'int': 1, 'float': 1.5} - - def test_basic_types(self): - test = self.test_dict - d = self.dict_class() - d.update(test) - self.failUnlessEqual(d.items(), test.items()) - d.clear() - self.failIf(d.items()) - - def test_in(self): - d = self.dict_class() - self.assertFalse('test' in d) - d['test'] = 123 - self.assertTrue('test' in d) - - def test_keyerror(self): - d = self.dict_class() - self.assertRaises(KeyError, d.__getitem__, 'test') - - def test_replace(self): - d = self.dict_class() - self.assertEqual(d.get('test'), None) - d['test'] = 123 - self.assertEqual(d.get('test'), 123) - d['test'] = 456 - self.assertEqual(d.get('test'), 456) - - -class JsonSqliteDictTest(SqliteDictTest): - - dict_class = JsonSqliteDict - test_dict = SqliteDictTest.test_dict.copy() - test_dict.update({'list': ['a', 'world'], 'dict': {'some': 'dict'}}) - - -class PickleSqliteDictTest(JsonSqliteDictTest): - - dict_class = PickleSqliteDict - test_dict = JsonSqliteDictTest.test_dict.copy() - test_dict.update({'decimal': Decimal("10"), 'datetime': datetime.now()}) - - def test_request_persistance(self): - r1 = Request("http://www.example.com", body="some") - d = self.dict_class() - d['request'] = r1 - r2 = d['request'] - self.failUnless(isinstance(r2, Request)) - self.failUnlessEqual(r1.url, r2.url) - self.failUnlessEqual(r1.body, r2.body) - - -class SqlitePriorityQueueTest(unittest.TestCase): - - queue_class = SqlitePriorityQueue - - supported_values = ["bytes", u"\xa3", 123, 1.2, True] - - def setUp(self): - self.q = self.queue_class() - - def test_empty(self): - self.failUnless(self.q.pop() is None) - - def test_one(self): - msg = "a message" - self.q.put(msg) - self.failIf("_id" in msg) - self.failUnlessEqual(self.q.pop(), msg) - self.failUnless(self.q.pop() is None) - - def test_multiple(self): - msg1 = "first message" - msg2 = "second message" - self.q.put(msg1) - self.q.put(msg2) - out = [] - out.append(self.q.pop()) - out.append(self.q.pop()) - self.failUnless(msg1 in out) - self.failUnless(msg2 in out) - self.failUnless(self.q.pop() is None) - - def test_priority(self): - msg1 = "message 1" - msg2 = "message 2" - msg3 = "message 3" - msg4 = "message 4" - self.q.put(msg1, priority=1.0) - self.q.put(msg2, priority=5.0) - self.q.put(msg3, priority=3.0) - self.q.put(msg4, priority=2.0) - self.failUnlessEqual(self.q.pop(), msg2) - self.failUnlessEqual(self.q.pop(), msg3) - self.failUnlessEqual(self.q.pop(), msg4) - self.failUnlessEqual(self.q.pop(), msg1) - - def test_iter_len_clear(self): - self.failUnlessEqual(len(self.q), 0) - self.failUnlessEqual(list(self.q), []) - msg1 = "message 1" - msg2 = "message 2" - msg3 = "message 3" - msg4 = "message 4" - self.q.put(msg1, priority=1.0) - self.q.put(msg2, priority=5.0) - self.q.put(msg3, priority=3.0) - self.q.put(msg4, priority=2.0) - self.failUnlessEqual(len(self.q), 4) - self.failUnlessEqual(list(self.q), \ - [(msg2, 5.0), (msg3, 3.0), (msg4, 2.0), (msg1, 1.0)]) - self.q.clear() - self.failUnlessEqual(len(self.q), 0) - self.failUnlessEqual(list(self.q), []) - - def test_remove(self): - self.failUnlessEqual(len(self.q), 0) - self.failUnlessEqual(list(self.q), []) - msg1 = "good message 1" - msg2 = "bad message 2" - msg3 = "good message 3" - msg4 = "bad message 4" - self.q.put(msg1) - self.q.put(msg2) - self.q.put(msg3) - self.q.put(msg4) - self.q.remove(lambda x: x.startswith("bad")) - self.failUnlessEqual(list(self.q), [(msg1, 0.0), (msg3, 0.0)]) - - def test_types(self): - for x in self.supported_values: - self.q.put(x) - self.failUnlessEqual(self.q.pop(), x) - - -class JsonSqlitePriorityQueueTest(SqlitePriorityQueueTest): - - queue_class = JsonSqlitePriorityQueue - - supported_values = SqlitePriorityQueueTest.supported_values + [ - ["a", "list", 1], - {"a": "dict"}, - ] - - -class PickleSqlitePriorityQueueTest(JsonSqlitePriorityQueueTest): - - queue_class = PickleSqlitePriorityQueue - - supported_values = JsonSqlitePriorityQueueTest.supported_values + [ - Decimal("10"), - datetime.now(), - ] - - def test_request_persistance(self): - r1 = Request("http://www.example.com", body="some") - self.q.put(r1) - r2 = self.q.pop() - self.failUnless(isinstance(r2, Request)) - self.failUnlessEqual(r1.url, r2.url) - self.failUnlessEqual(r1.body, r2.body) diff --git a/scrapyd/tests/test_utils.py b/scrapyd/tests/test_utils.py deleted file mode 100644 index a73c58d2d..000000000 --- a/scrapyd/tests/test_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -from pkgutil import get_data -from cStringIO import StringIO - -from twisted.trial import unittest - -from scrapy.utils.test import get_pythonpath -from scrapyd.interfaces import IEggStorage -from scrapyd.utils import get_crawl_args, get_spider_list -from scrapyd import get_application - -class UtilsTest(unittest.TestCase): - - def test_get_crawl_args(self): - msg = {'_project': 'lolo', '_spider': 'lala'} - self.assertEqual(get_crawl_args(msg), ['lala']) - msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1'} - cargs = get_crawl_args(msg) - self.assertEqual(cargs, ['lala', '-a', 'arg1=val1']) - assert all(isinstance(x, str) for x in cargs), cargs - - def test_get_crawl_args_with_settings(self): - msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1', 'settings': {'ONE': 'two'}} - cargs = get_crawl_args(msg) - self.assertEqual(cargs, ['lala', '-a', 'arg1=val1', '-s', 'ONE=two']) - assert all(isinstance(x, str) for x in cargs), cargs - -class GetSpiderListTest(unittest.TestCase): - - def test_get_spider_list(self): - path = os.path.abspath(self.mktemp()) - j = os.path.join - eggs_dir = j(path, 'eggs') - os.makedirs(eggs_dir) - dbs_dir = j(path, 'dbs') - os.makedirs(dbs_dir) - logs_dir = j(path, 'logs') - os.makedirs(logs_dir) - os.chdir(path) - with open('scrapyd.conf', 'w') as f: - f.write("[scrapyd]\n") - f.write("eggs_dir = %s\n" % eggs_dir) - f.write("dbs_dir = %s\n" % dbs_dir) - f.write("logs_dir = %s\n" % logs_dir) - app = get_application() - eggstorage = app.getComponent(IEggStorage) - eggfile = StringIO(get_data(__package__, 'mybot.egg')) - eggstorage.put(eggfile, 'mybot', 'r1') - spiders = get_spider_list('mybot', pythonpath=get_pythonpath()) - self.assertEqual(sorted(spiders), ['spider1', 'spider2']) - diff --git a/scrapyd/utils.py b/scrapyd/utils.py deleted file mode 100644 index a299ba6a9..000000000 --- a/scrapyd/utils.py +++ /dev/null @@ -1,67 +0,0 @@ -import sys -import os -from subprocess import Popen, PIPE -from ConfigParser import NoSectionError - -from scrapyd.spiderqueue import SqliteSpiderQueue -from scrapy.utils.python import stringify_dict, unicode_to_str -from scrapyd.config import Config - -def get_spider_queues(config): - """Return a dict of Spider Quees keyed by project name""" - dbsdir = config.get('dbs_dir', 'dbs') - if not os.path.exists(dbsdir): - os.makedirs(dbsdir) - d = {} - for project in get_project_list(config): - dbpath = os.path.join(dbsdir, '%s.db' % project) - d[project] = SqliteSpiderQueue(dbpath) - return d - -def get_project_list(config): - """Get list of projects by inspecting the eggs dir and the ones defined in - the scrapyd.conf [settings] section - """ - eggs_dir = config.get('eggs_dir', 'eggs') - if os.path.exists(eggs_dir): - projects = os.listdir(eggs_dir) - else: - projects = [] - try: - projects += [x[0] for x in config.cp.items('settings')] - except NoSectionError: - pass - return projects - -def get_crawl_args(message): - """Return the command-line arguments to use for the scrapy crawl process - that will be started for this message - """ - msg = message.copy() - args = [unicode_to_str(msg['_spider'])] - del msg['_project'], msg['_spider'] - settings = msg.pop('settings', {}) - for k, v in stringify_dict(msg, keys_only=False).items(): - args += ['-a'] - args += ['%s=%s' % (k, v)] - for k, v in stringify_dict(settings, keys_only=False).items(): - args += ['-s'] - args += ['%s=%s' % (k, v)] - return args - -def get_spider_list(project, runner=None, pythonpath=None): - """Return the spider list from the given project, using the given runner""" - if runner is None: - runner = Config().get('runner') - env = os.environ.copy() - env['SCRAPY_PROJECT'] = project - if pythonpath: - env['PYTHONPATH'] = pythonpath - pargs = [sys.executable, '-m', runner, 'list'] - proc = Popen(pargs, stdout=PIPE, stderr=PIPE, env=env) - out, err = proc.communicate() - if proc.returncode: - msg = err or out or 'unknown error' - raise RuntimeError(msg.splitlines()[-1]) - return out.splitlines() - diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py deleted file mode 100644 index 3df47f1f1..000000000 --- a/scrapyd/webservice.py +++ /dev/null @@ -1,122 +0,0 @@ -import traceback -import uuid -from cStringIO import StringIO - -from twisted.python import log - -from scrapy.utils.txweb import JsonResource -from .utils import get_spider_list - -class WsResource(JsonResource): - - def __init__(self, root): - JsonResource.__init__(self) - self.root = root - - def render(self, txrequest): - try: - return JsonResource.render(self, txrequest) - except Exception, e: - if self.root.debug: - return traceback.format_exc() - log.err() - r = {"status": "error", "message": str(e)} - return self.render_object(r, txrequest) - -class Schedule(WsResource): - - def render_POST(self, txrequest): - settings = txrequest.args.pop('setting', []) - settings = dict(x.split('=', 1) for x in settings) - args = dict((k, v[0]) for k, v in txrequest.args.items()) - project = args.pop('project') - spider = args.pop('spider') - args['settings'] = settings - jobid = uuid.uuid1().hex - args['_job'] = jobid - self.root.scheduler.schedule(project, spider, **args) - return {"status": "ok", "jobid": jobid} - -class Cancel(WsResource): - - def render_POST(self, txrequest): - args = dict((k, v[0]) for k, v in txrequest.args.items()) - project = args['project'] - jobid = args['job'] - signal = args.get('signal', 'TERM') - prevstate = None - queue = self.root.poller.queues[project] - c = queue.remove(lambda x: x["_job"] == jobid) - if c: - prevstate = "pending" - spiders = self.root.launcher.processes.values() - for s in spiders: - if s.job == jobid: - s.transport.signalProcess(signal) - prevstate = "running" - return {"status": "ok", "prevstate": prevstate} - -class AddVersion(WsResource): - - def render_POST(self, txrequest): - project = txrequest.args['project'][0] - version = txrequest.args['version'][0] - eggf = StringIO(txrequest.args['egg'][0]) - self.root.eggstorage.put(eggf, project, version) - spiders = get_spider_list(project) - self.root.update_projects() - return {"status": "ok", "project": project, "version": version, \ - "spiders": len(spiders)} - -class ListProjects(WsResource): - - def render_GET(self, txrequest): - projects = self.root.scheduler.list_projects() - return {"status": "ok", "projects": projects} - -class ListVersions(WsResource): - - def render_GET(self, txrequest): - project = txrequest.args['project'][0] - versions = self.root.eggstorage.list(project) - return {"status": "ok", "versions": versions} - -class ListSpiders(WsResource): - - def render_GET(self, txrequest): - project = txrequest.args['project'][0] - spiders = get_spider_list(project, runner=self.root.runner) - return {"status": "ok", "spiders": spiders} - -class ListJobs(WsResource): - - def render_GET(self, txrequest): - project = txrequest.args['project'][0] - spiders = self.root.launcher.processes.values() - running = [{"id": s.job, "spider": s.spider} for s in spiders if s.project == project] - queue = self.root.poller.queues[project] - pending = [{"id": x["_job"], "spider": x["name"]} for x in queue.list()] - finished = [{"id": s.job, "spider": s.spider, - "start_time": s.start_time.isoformat(' '), - "end_time": s.end_time.isoformat(' ')} for s in self.root.launcher.finished - if s.project == project] - return {"status":"ok", "pending": pending, "running": running, "finished": finished} - -class DeleteProject(WsResource): - - def render_POST(self, txrequest): - project = txrequest.args['project'][0] - self._delete_version(project) - return {"status": "ok"} - - def _delete_version(self, project, version=None): - self.root.eggstorage.delete(project, version) - self.root.update_projects() - -class DeleteVersion(DeleteProject): - - def render_POST(self, txrequest): - project = txrequest.args['project'][0] - version = txrequest.args['version'][0] - self._delete_version(project, version) - return {"status": "ok"} diff --git a/scrapyd/website.py b/scrapyd/website.py deleted file mode 100644 index 779c857f2..000000000 --- a/scrapyd/website.py +++ /dev/null @@ -1,131 +0,0 @@ -from datetime import datetime - -from twisted.web import resource, static -from twisted.application.service import IServiceCollection - -from scrapy.utils.misc import load_object - -from .interfaces import IPoller, IEggStorage, ISpiderScheduler - -class Root(resource.Resource): - - def __init__(self, config, app): - resource.Resource.__init__(self) - self.debug = config.getboolean('debug', False) - self.runner = config.get('runner') - logsdir = config.get('logs_dir') - itemsdir = config.get('items_dir') - self.app = app - self.putChild('', Home(self)) - self.putChild('logs', static.File(logsdir, 'text/plain')) - self.putChild('items', static.File(itemsdir, 'text/plain')) - self.putChild('jobs', Jobs(self)) - services = config.items('services', ()) - for servName, servClsName in services: - servCls = load_object(servClsName) - self.putChild(servName, servCls(self)) - self.update_projects() - - def update_projects(self): - self.poller.update_projects() - self.scheduler.update_projects() - - @property - def launcher(self): - app = IServiceCollection(self.app, self.app) - return app.getServiceNamed('launcher') - - @property - def scheduler(self): - return self.app.getComponent(ISpiderScheduler) - - @property - def eggstorage(self): - return self.app.getComponent(IEggStorage) - - @property - def poller(self): - return self.app.getComponent(IPoller) - - -class Home(resource.Resource): - - def __init__(self, root): - resource.Resource.__init__(self) - self.root = root - - def render_GET(self, txrequest): - vars = { - 'projects': ', '.join(self.root.scheduler.list_projects()), - } - return """ - -Scrapyd - -

Scrapyd

-

Available projects: %(projects)s

- - -

How to schedule a spider?

- -

To schedule a spider you need to use the API (this web UI is only for -monitoring)

- -

Example using curl:

-

curl http://localhost:6800/schedule.json -d project=default -d spider=somespider

- -

For more information about the API, see the Scrapyd documentation

- - -""" % vars - - -class Jobs(resource.Resource): - - def __init__(self, root): - resource.Resource.__init__(self) - self.root = root - - def render(self, txrequest): - s = "Scrapyd" - s += "" - s += "

Jobs

" - s += "

Go back

" - s += "" - s += "" - s += "" - for project, queue in self.root.poller.queues.items(): - for m in queue.list(): - s += "" - s += "" % project - s += "" % str(m['name']) - s += "" % str(m['_job']) - s += "" - s += "" - for p in self.root.launcher.processes.values(): - s += "" - for a in ['project', 'spider', 'job', 'pid']: - s += "" % getattr(p, a) - s += "" % (datetime.now() - p.start_time) - s += "" % (p.project, p.spider, p.job) - s += "" % (p.project, p.spider, p.job) - s += "" - s += "" - for p in self.root.launcher.finished: - s += "" - for a in ['project', 'spider', 'job']: - s += "" % getattr(p, a) - s += "" - s += "" % (p.end_time - p.start_time) - s += "" % (p.project, p.spider, p.job) - s += "" % (p.project, p.spider, p.job) - s += "" - s += "
ProjectSpiderJobPIDRuntimeLogItems
Pending
%s%s%s
Running
%s%sLogItems
Finished
%s%sLogItems
" - s += "" - s += "" - return s diff --git a/setup.py b/setup.py index 551bb5208..1c2a610d8 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Scrapy setup.py script +# Scrapy setup.py script # # It doesn't depend on setuptools, but if setuptools is available it'll use # some of its features, like package dependencies. @@ -57,7 +57,7 @@ if root_dir != '': def is_not_module(filename): return os.path.splitext(f)[1] not in ['.py', '.pyc', '.pyo'] -for scrapy_dir in ['scrapy', 'scrapyd']: +for scrapy_dir in ['scrapy']: for dirpath, dirnames, filenames in os.walk(scrapy_dir): # Ignore dirnames that start with '.' for i, dirname in enumerate(dirnames): @@ -121,5 +121,5 @@ except ImportError: from distutils.core import setup else: setup_args['install_requires'] = ['Twisted>=8.0', 'w3lib>=1.2', 'lxml', 'pyOpenSSL'] - + setup(**setup_args) diff --git a/tox.ini b/tox.ini index e563fb3c7..157c0c217 100644 --- a/tox.ini +++ b/tox.ini @@ -9,4 +9,4 @@ envlist = py26, py27 [testenv] commands = pip install --use-mirrors -r .travis/requirements-latest.txt - trial scrapy scrapyd + trial scrapy