diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..c720c089b --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +VERSION ?= $(shell python -c "import scrapy; print '.'.join(map(str, scrapy.version_info[:2]))") +DEBREV ?= 1 +HGREV ?= $(shell hg log -l1 --template '{rev}') +FULL_VERSION ?= $(VERSION)~r$(HGREV) +BUILDBASE ?= $(shell pwd)/build/deb +BUILDDIR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig +BUILDTAR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig.tar.gz + +help: + @echo 'Commonly used make targets:' + @echo ' deb-binary - build debian binary package' + @echo ' deb-source - build debian source package' + @echo ' deb-all - build source and binary debian packages' + @echo ' tarball - build source tarball' + +deb-binary: deb-prepare + cd $(BUILDDIR); debuild -i -us -uc -b + +deb-source: deb-prepare + cd $(BUILDDIR); debuild -i -us -uc -S + +deb-all: deb-prepare + cd $(BUILDDIR); debuild -i -us -uc + +deb-prepare: + @if [ -d $(BUILDBASE) ]; then \ + rm -rf $(BUILDBASE); \ + fi; + mkdir -p $(BUILDBASE) + hg archive -t tgz -X .hgtags $(BUILDTAR) + tar zxf $(BUILDTAR) -C $(BUILDBASE) + cp -r debian $(BUILDDIR) + cd $(BUILDDIR); debchange -m -D unstable --force-distribution -v $(FULL_VERSION)-$(DEBREV) "Automatic build" + +tarball: + python setup.py sdist diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 000000000..b1f6b3d94 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +scrapy (0.9~r2000-1) unstable; urgency=low + + * Initial release. + + -- Insophia Team Thu, 10 Jun 2010 17:24:02 -0300 diff --git a/debian/compat b/debian/compat new file mode 100644 index 000000000..7f8f011eb --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/conf/environment b/debian/conf/environment new file mode 100644 index 000000000..bea783073 --- /dev/null +++ b/debian/conf/environment @@ -0,0 +1,3 @@ +# Environment variables to use in Scrapy service + +#export PYTHONPATH=/var/lib/scrapy/myproject diff --git a/debian/conf/service_conf.py b/debian/conf/service_conf.py new file mode 100644 index 000000000..54aa00a82 --- /dev/null +++ b/debian/conf/service_conf.py @@ -0,0 +1,13 @@ +# Scrapy service configuration + +# Directory where logs will be stored (one per crawler) +LOG_DIR = '/var/log/scrapy' + +# A dict containing the Scrapy projects that will be run by this service +# * Keys are paths to project settings modules +# * Values are number of processes that should be started for each project, or +# zero if you want to use the number of cores available. +PROJECTS = { +# 'mybot.settings': 0, +} + diff --git a/debian/control b/debian/control new file mode 100644 index 000000000..54cf6a236 --- /dev/null +++ b/debian/control @@ -0,0 +1,18 @@ +Source: scrapy +Section: python +Priority: optional +Maintainer: Insophia Team +Build-Depends: debhelper (>= 7.0.50), python-twisted +Build-Depends-Indep: python-support, python, python-sphinx, libjs-jquery +Standards-Version: 3.8.4 +Homepage: http://scrapy.org/ + +Package: scrapy +Architecture: all +Depends: ${misc:Depends}, ${python:Depends}, python-libxml2, python-twisted, python-openssl +Provides: ${python:Provides} +Description: Python web scraping and crawling framework + Scrapy is a fast high-level screen scraping and web crawling framework, + used to crawl websites and extract structured data from their pages. + It can be used for a wide range of purposes, from data mining to + monitoring and automated testing. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 000000000..452544013 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,40 @@ +This package was debianized by Insophia . + +It was downloaded from http://scrapy.org + +Upstream Author: Scrapy Developers + +Copyright: 2007-2010 Scrapy Developers + +License: bsd + +Copyright (c) Scrapy developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Scrapy nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The Debian packaging is (C) 2010, Insophia and +is licensed under the BSD, see `/usr/share/common-licenses/BSD'. diff --git a/debian/pycompat b/debian/pycompat new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/debian/pycompat @@ -0,0 +1 @@ +2 diff --git a/debian/pyversions b/debian/pyversions new file mode 100644 index 000000000..3ec13be1a --- /dev/null +++ b/debian/pyversions @@ -0,0 +1 @@ +2.5-2.6 diff --git a/debian/rules b/debian/rules new file mode 100755 index 000000000..ae5f7608e --- /dev/null +++ b/debian/rules @@ -0,0 +1,11 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +export DH_ALWAYS_EXCLUDE=license.txt:_sources/:.buildinfo + +%: + dh $@ + +override_dh_auto_install: + dh_auto_install + mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl diff --git a/debian/scrapy-ctl.1 b/debian/scrapy-ctl.1 new file mode 100644 index 000000000..21168bde1 --- /dev/null +++ b/debian/scrapy-ctl.1 @@ -0,0 +1,79 @@ +.TH SCRAPY-CTL 1 "October 17, 2009" +.SH NAME +scrapy-ctl \- Python Scrapy control script +.SH SYNOPSIS +.B scrapy-ctl +[\fIcommand\fR] [\fIOPTIONS\fR] ... +.SH DESCRIPTION +.PP +Scrapy is controlled through the \fBscrapy-ctl\fR control script. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options. +.SH OPTIONS +.SS fetch\fR [\fIOPTION\fR] \fIURL\fR +.TP +Fetch a URL using the Scrapy downloader +.TP +.I --headers +Print response HTTP headers instead of body + +.SS runspider\fR [\fIOPTION\fR] \fIspiderfile\fR +Run a spider +.TP +.I --output=FILE +Store scraped items to FILE in XML format + +.SS settings [\fIOPTION\fR] +Query Scrapy settings +.TP +.I --get=SETTING +Print raw setting value +.TP +.I --getbool=SETTING +Print setting value, intepreted as a boolean +.TP +.I --getint=SETTING +Print setting value, intepreted as an integer +.TP +.I --getfloat=SETTING +Print setting value, intepreted as an float +.TP +.I --getlist=SETTING +Print setting value, intepreted as an float +.TP +.I --init +Print initial setting value (before loading extensions and spiders) + +.SS shell\fR \fIURL\fR | \fIfile\fR +Launch the interactive scraping console + +.SS startproject\fR \fIprojectname\fR +Create new project with an initial project template + +.SS --help, -h +Print command help and options +.SS --version +Print Scrapy version and exit +.SS --logfile=FILE +Log file. if omitted stderr will be used +.SS --loglevel=LEVEL, -L LEVEL +Log level (default: None) +.SS --nolog +Disable logging completely +.SS --spider=SPIDER +Always use this spider when arguments are urls +.SS --profile=FILE +Write python cProfile stats to FILE +.SS --lsprof=FILE +Write lsprof profiling stats to FILE +.SS --pidfile=FILE +Write process ID to FILE +.SS --set=SET +Set/override setting (may be repeated) +.SS --settings=MODULE +Python path to the Scrapy project settings + +.SH AUTHOR +Scrapy-ctl was written by the Scrapy Developers +. +.PP +This manual page was written by Ignace Mouzannar , +for the Debian project (but may be used by others). diff --git a/debian/scrapy.dirs b/debian/scrapy.dirs new file mode 100644 index 000000000..b3c079630 --- /dev/null +++ b/debian/scrapy.dirs @@ -0,0 +1,4 @@ +usr/bin +usr/share/scrapy +var/lib/scrapy +var/log/scrapy diff --git a/debian/scrapy.docs b/debian/scrapy.docs new file mode 100644 index 000000000..55bc0a63d --- /dev/null +++ b/debian/scrapy.docs @@ -0,0 +1,2 @@ +README +AUTHORS diff --git a/debian/scrapy.examples b/debian/scrapy.examples new file mode 100644 index 000000000..e39721e20 --- /dev/null +++ b/debian/scrapy.examples @@ -0,0 +1 @@ +examples/* diff --git a/debian/scrapy.install b/debian/scrapy.install new file mode 100644 index 000000000..d3d8c0d98 --- /dev/null +++ b/debian/scrapy.install @@ -0,0 +1,3 @@ +debian/conf/service_conf.py etc/scrapy +debian/conf/environment etc/scrapy +bin/scrapy.tac usr/share/scrapy diff --git a/debian/scrapy.manpages b/debian/scrapy.manpages new file mode 100644 index 000000000..a5edfdd8a --- /dev/null +++ b/debian/scrapy.manpages @@ -0,0 +1 @@ +debian/scrapy-ctl.1 diff --git a/debian/scrapy.postinst b/debian/scrapy.postinst new file mode 100644 index 000000000..cbc63b20c --- /dev/null +++ b/debian/scrapy.postinst @@ -0,0 +1,60 @@ +#!/bin/sh +# postinst script for scrapy +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + +case "$1" in + configure) + # Fix for old packages not managing pyc files + for d in /usr/lib/python*/site-packages/scrapy ; do + if [ -d "$d" ] ; then + echo "Removing old python byte-compiled files in $d" + rm -rf "$d" + fi + done + + # Create user to run the service as + if [ -z "`id -u scrapy 2> /dev/null`" ]; then + adduser --system --home /var/lib/scrapy --gecos "scrapy" \ + --no-create-home --disabled-password \ + --quiet scrapy || true + fi + if [ ! -d /var/run/scrapy ]; then + mkdir /var/run/scrapy + chown scrapy:nogroup /var/run/scrapy + fi + + chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 + + diff --git a/debian/scrapy.postrm b/debian/scrapy.postrm new file mode 100644 index 000000000..b6a3bf880 --- /dev/null +++ b/debian/scrapy.postrm @@ -0,0 +1,20 @@ +#!/bin/sh +set -e + +if [ purge = "$1" ]; then + if [ -x "$(command -v deluser)" ]; then + deluser --quiet --system scrapy > /dev/null || true + else + echo >&2 "not removing scrapy system account because deluser command was not found" + fi + rm -rf /var/run/scrapy +fi + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 + + diff --git a/debian/scrapy.upstart b/debian/scrapy.upstart new file mode 100644 index 000000000..36f0f570e --- /dev/null +++ b/debian/scrapy.upstart @@ -0,0 +1,16 @@ +# Scrapy service + +description "Scrapy web crawler" +author "Pablo Hoffman " + +start on runlevel [2345] +stop on runlevel [!2345] + +script + [ -r /etc/scrapy/environment ] && . /etc/scrapy/environment + export PYTHONPATH=/etc/scrapy:$PYTHONPATH + export SCRAPY_SETTINGS_MODULE=service_conf + twistd -ny /usr/share/scrapy/scrapy.tac \ + -u scrapy -g nogroup \ + -l /var/log/scrapy/service.log +end script diff --git a/debian/watch b/debian/watch new file mode 100644 index 000000000..a62198e8b --- /dev/null +++ b/debian/watch @@ -0,0 +1,2 @@ +version=3 +http://scrapy.org/releases/(\d\.\d)/Scrapy-([\d\.]+)\.tar\.gz