diff --git a/Makefile.buildbot b/Makefile.buildbot new file mode 100644 index 000000000..94c57bee2 --- /dev/null +++ b/Makefile.buildbot @@ -0,0 +1,6 @@ +deb: + python extras/makedeb.py build + +deb-clean: + debclean + python extras/makedeb.py clean diff --git a/debian/changelog b/debian/changelog index 46d5d3be1..a9c54e63c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -scrapy (0.11) unstable; urgency=low +scrapy-SUFFIX (0.11) unstable; urgency=low * Initial release. diff --git a/debian/control b/debian/control index 217e520ab..1bbec3e55 100644 --- a/debian/control +++ b/debian/control @@ -1,4 +1,4 @@ -Source: scrapy +Source: scrapy-SUFFIX Section: python Priority: optional Maintainer: Insophia Team @@ -6,7 +6,7 @@ Build-Depends: debhelper (>= 7.0.50), python (>=2.5), python-twisted Standards-Version: 3.8.4 Homepage: http://scrapy.org/ -Package: scrapy +Package: scrapy-SUFFIX Architecture: all Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl Conflicts: python-scrapy @@ -17,7 +17,7 @@ Description: Python web crawling and scraping framework It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -Package: scrapyd +Package: scrapyd-SUFFIX Architecture: all Depends: scrapy, python-setuptools Description: Scrapy Service diff --git a/debian/rules b/debian/rules index b8796e6e3..3026acd0e 100755 --- a/debian/rules +++ b/debian/rules @@ -3,3 +3,6 @@ %: dh $@ + +override_dh_installinit: + dh_installinit --name scrapyd diff --git a/debian/scrapy.1 b/debian/scrapy-files/scrapy.1 similarity index 100% rename from debian/scrapy.1 rename to debian/scrapy-files/scrapy.1 diff --git a/debian/scrapy.manpages b/debian/scrapy.manpages index acb5c60ef..baa38f044 100644 --- a/debian/scrapy.manpages +++ b/debian/scrapy.manpages @@ -1 +1 @@ -debian/scrapy.1 +debian/scrapy-files/scrapy.1 diff --git a/debian/000-default b/debian/scrapyd-files/000-default similarity index 100% rename from debian/000-default rename to debian/scrapyd-files/000-default diff --git a/debian/scrapyd.install b/debian/scrapyd.install index 8e0042970..d83e70b48 100644 --- a/debian/scrapyd.install +++ b/debian/scrapyd.install @@ -1,3 +1,3 @@ usr/lib/python*/*-packages/scrapyd -debian/000-default etc/scrapyd/conf.d +debian/scrapyd-files/000-default etc/scrapyd/conf.d extras/scrapyd.tac usr/share/scrapyd diff --git a/debian/scrapyd.upstart b/debian/scrapyd.scrapyd.upstart similarity index 100% rename from debian/scrapyd.upstart rename to debian/scrapyd.scrapyd.upstart diff --git a/extras/makedeb.py b/extras/makedeb.py new file mode 100644 index 000000000..054204cf2 --- /dev/null +++ b/extras/makedeb.py @@ -0,0 +1,36 @@ +import sys, os, glob +from subprocess import check_call + +def build(suffix): + for ifn in glob.glob("debian/scrapy.*") + glob.glob("debian/scrapyd.*"): + s = open(ifn).read() + s = s.replace('SUFFIX', suffix) + pre, suf = ifn.split('.', 1) + ofn = "%s-%s.%s" % (pre, suffix, suf) + with open(ofn, 'w') as of: + of.write(s) + + for ifn in ['debian/control', 'debian/changelog']: + s = open(ifn).read() + s = s.replace('SUFFIX', suffix) + with open(ifn, 'w') as of: + of.write(s) + + check_call('debchange -m -D unstable --force-distribution -v $(python setup.py --version)-r$(hg tip --template "{rev}")+$(date +%s) "Automatic build"', shell=True) + check_call('debuild -us -uc -b', shell=True) + +def clean(suffix): + for f in glob.glob("debian/scrapy-%s.*" % suffix) + \ + glob.glob("debian/scrapyd-%s.*" % suffix): + os.remove(f) + +def main(): + cmd = sys.argv[1] + suffix = '%s.%s' % __import__('scrapy').version_info[:2] + if cmd == 'build': + build(suffix) + elif cmd == 'clean': + clean(suffix) + +if __name__ == '__main__': + main() diff --git a/extras/scrapy.1 b/extras/scrapy.1 new file mode 100644 index 000000000..e73ff5459 --- /dev/null +++ b/extras/scrapy.1 @@ -0,0 +1,75 @@ +.TH SCRAPY 1 "October 17, 2009" +.SH NAME +scrapy \- the Scrapy command-line tool +.SH SYNOPSIS +.B scrapy +[\fIcommand\fR] [\fIOPTIONS\fR] ... +.SH DESCRIPTION +.PP +Scrapy is controlled through the \fBscrapy\fR command-line tool. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options. +.SH OPTIONS +.SS fetch\fR [\fIOPTION\fR] \fIURL\fR +.TP +Fetch a URL using the Scrapy downloader +.TP +.I --headers +Print response HTTP headers instead of body + +.SS runspider\fR [\fIOPTION\fR] \fIspiderfile\fR +Run a spider +.TP +.I --output=FILE +Store scraped items to FILE in XML format + +.SS settings [\fIOPTION\fR] +Query Scrapy settings +.TP +.I --get=SETTING +Print raw setting value +.TP +.I --getbool=SETTING +Print setting value, intepreted as a boolean +.TP +.I --getint=SETTING +Print setting value, intepreted as an integer +.TP +.I --getfloat=SETTING +Print setting value, intepreted as an float +.TP +.I --getlist=SETTING +Print setting value, intepreted as an float +.TP +.I --init +Print initial setting value (before loading extensions and spiders) + +.SS shell\fR \fIURL\fR | \fIfile\fR +Launch the interactive scraping console + +.SS startproject\fR \fIprojectname\fR +Create new project with an initial project template + +.SS --help, -h +Print command help and options +.SS --logfile=FILE +Log file. if omitted stderr will be used +.SS --loglevel=LEVEL, -L LEVEL +Log level (default: None) +.SS --nolog +Disable logging completely +.SS --spider=SPIDER +Always use this spider when arguments are urls +.SS --profile=FILE +Write python cProfile stats to FILE +.SS --lsprof=FILE +Write lsprof profiling stats to FILE +.SS --pidfile=FILE +Write process ID to FILE +.SS --set=SET +Set/override setting (may be repeated) + +.SH AUTHOR +Scrapy was written by the Scrapy Developers +. +.PP +This manual page was written by Ignace Mouzannar , +for the Debian project (but may be used by others).