Added sources and Makefile for building Debian package

Makefile

@@ -0,0 +1,36 @@
+VERSION ?= $(shell python -c "import scrapy; print '.'.join(map(str, scrapy.version_info[:2]))")
+DEBREV ?= 1
+HGREV ?= $(shell hg log -l1 --template '{rev}')
+FULL_VERSION ?= $(VERSION)~r$(HGREV)
+BUILDBASE ?= $(shell pwd)/build/deb
+BUILDDIR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig
+BUILDTAR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig.tar.gz
+
+help:
+	@echo 'Commonly used make targets:'
+	@echo '  deb-binary   - build debian binary package'
+	@echo '  deb-source   - build debian source package'
+	@echo '  deb-all      - build source and binary debian packages'
+	@echo '  tarball      - build source tarball'
+
+deb-binary: deb-prepare
+	cd $(BUILDDIR); debuild -i -us -uc -b
+
+deb-source: deb-prepare
+	cd $(BUILDDIR); debuild -i -us -uc -S
+
+deb-all: deb-prepare
+	cd $(BUILDDIR); debuild -i -us -uc
+
+deb-prepare:
+	@if [ -d $(BUILDBASE) ]; then \
+		rm -rf $(BUILDBASE); \
+	fi;
+	mkdir -p $(BUILDBASE)
+	hg archive -t tgz -X .hgtags $(BUILDTAR)
+	tar zxf $(BUILDTAR) -C $(BUILDBASE)
+	cp -r debian $(BUILDDIR)
+	cd $(BUILDDIR); debchange -m -D unstable --force-distribution -v $(FULL_VERSION)-$(DEBREV) "Automatic build"
+
+tarball:
+	python setup.py sdist

debian/changelog

@@ -0,0 +1,5 @@
+scrapy (0.9~r2000-1) unstable; urgency=low
+
+  * Initial release.
+
+ -- Insophia Team <info@insophia.com>  Thu, 10 Jun 2010 17:24:02 -0300

debian/compat

@@ -0,0 +1 @@
+7

debian/conf/environment

@@ -0,0 +1,3 @@
+# Environment variables to use in Scrapy service
+
+#export PYTHONPATH=/var/lib/scrapy/myproject

debian/conf/service_conf.py

@@ -0,0 +1,13 @@
+# Scrapy service configuration
+
+# Directory where logs will be stored (one per crawler)
+LOG_DIR = '/var/log/scrapy'
+
+# A dict containing the Scrapy projects that will be run by this service
+# * Keys are paths to project settings modules
+# * Values are number of processes that should be started for each project, or
+#   zero if you want to use the number of cores available.
+PROJECTS = {
+#        'mybot.settings': 0,
+}
+

debian/control

@@ -0,0 +1,18 @@
+Source: scrapy
+Section: python
+Priority: optional
+Maintainer: Insophia Team <info@insophia.com> 
+Build-Depends: debhelper (>= 7.0.50), python-twisted
+Build-Depends-Indep: python-support, python, python-sphinx, libjs-jquery
+Standards-Version: 3.8.4
+Homepage: http://scrapy.org/
+
+Package: scrapy
+Architecture: all
+Depends: ${misc:Depends}, ${python:Depends}, python-libxml2, python-twisted, python-openssl
+Provides: ${python:Provides}
+Description: Python web scraping and crawling framework
+ Scrapy is a fast high-level screen scraping and web crawling framework, 
+ used to crawl websites and extract structured data from their pages. 
+ It can be used for a wide range of purposes, from data mining to 
+ monitoring and automated testing.

debian/copyright

@@ -0,0 +1,40 @@
+This package was debianized by Insophia <info@insophia.com>.
+
+It was downloaded from http://scrapy.org
+
+Upstream Author: Scrapy Developers
+
+Copyright: 2007-2010 Scrapy Developers
+
+License: bsd
+
+Copyright (c) Scrapy developers.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    1. Redistributions of source code must retain the above copyright notice, 
+       this list of conditions and the following disclaimer.
+    
+    2. Redistributions in binary form must reproduce the above copyright 
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+
+    3. Neither the name of Scrapy nor the names of its contributors may be used
+       to endorse or promote products derived from this software without
+       specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The Debian packaging is (C) 2010, Insophia <info@insophia.com> and
+is licensed under the BSD, see `/usr/share/common-licenses/BSD'.

debian/pycompat

@@ -0,0 +1 @@
+2

debian/pyversions

@@ -0,0 +1 @@
+2.5-2.6

debian/rules

@@ -0,0 +1,11 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+
+export DH_ALWAYS_EXCLUDE=license.txt:_sources/:.buildinfo
+
+%:
+	dh $@
+
+override_dh_auto_install:
+	dh_auto_install
+	mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl

debian/scrapy-ctl.1

@@ -0,0 +1,79 @@
+.TH SCRAPY-CTL 1 "October 17, 2009"
+.SH NAME
+scrapy-ctl \- Python Scrapy control script
+.SH SYNOPSIS
+.B scrapy-ctl
+[\fIcommand\fR] [\fIOPTIONS\fR] ...
+.SH DESCRIPTION
+.PP
+Scrapy is controlled through the \fBscrapy-ctl\fR control script. The script provides several commands, for different purposes. Each command supports its own particular syntax. In other words, each command supports a different set of arguments and options.
+.SH OPTIONS
+.SS fetch\fR [\fIOPTION\fR]  \fIURL\fR
+.TP
+Fetch a URL using the Scrapy downloader
+.TP
+.I --headers
+Print response HTTP headers instead of body
+
+.SS runspider\fR [\fIOPTION\fR]  \fIspiderfile\fR
+Run a spider
+.TP
+.I --output=FILE
+Store scraped items to FILE in XML format
+
+.SS settings [\fIOPTION\fR]
+Query Scrapy settings
+.TP
+.I --get=SETTING
+Print raw setting value
+.TP
+.I --getbool=SETTING
+Print setting value, intepreted as a boolean
+.TP
+.I --getint=SETTING
+Print setting value, intepreted as an integer
+.TP
+.I --getfloat=SETTING
+Print setting value, intepreted as an float
+.TP
+.I --getlist=SETTING
+Print setting value, intepreted as an float
+.TP
+.I --init
+Print initial setting value (before loading extensions and spiders)
+
+.SS shell\fR  \fIURL\fR | \fIfile\fR
+Launch the interactive scraping console
+
+.SS startproject\fR \fIprojectname\fR
+Create new project with an initial project template
+
+.SS --help, -h
+Print command help and options
+.SS --version
+Print Scrapy version and exit
+.SS --logfile=FILE
+Log file. if omitted stderr will be used
+.SS --loglevel=LEVEL, -L LEVEL
+Log level (default: None)
+.SS --nolog
+Disable logging completely
+.SS --spider=SPIDER
+Always use this spider when arguments are urls
+.SS --profile=FILE
+Write python cProfile stats to FILE
+.SS --lsprof=FILE
+Write lsprof profiling stats to FILE
+.SS --pidfile=FILE
+Write process ID to FILE
+.SS --set=SET
+Set/override setting (may be repeated)
+.SS --settings=MODULE
+Python path to the Scrapy project settings
+
+.SH AUTHOR
+Scrapy-ctl was written by the Scrapy Developers 
+<scrapy-developers@googlegroups.com>.
+.PP
+This manual page was written by Ignace Mouzannar <mouzannar@gmail.com>,
+for the Debian project (but may be used by others).

debian/scrapy.dirs

@@ -0,0 +1,4 @@
+usr/bin
+usr/share/scrapy
+var/lib/scrapy
+var/log/scrapy

debian/scrapy.docs

@@ -0,0 +1,2 @@
+README
+AUTHORS

debian/scrapy.examples

@@ -0,0 +1 @@
+examples/*

debian/scrapy.install

@@ -0,0 +1,3 @@
+debian/conf/service_conf.py etc/scrapy
+debian/conf/environment etc/scrapy
+bin/scrapy.tac usr/share/scrapy

debian/scrapy.manpages

@@ -0,0 +1 @@
+debian/scrapy-ctl.1

debian/scrapy.postinst

@@ -0,0 +1,60 @@
+#!/bin/sh
+# postinst script for scrapy
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# summary of how this script can be called:
+#        * <postinst> `configure' <most-recently-configured-version>
+#        * <old-postinst> `abort-upgrade' <new version>
+#        * <conflictor's-postinst> `abort-remove' `in-favour' <package>
+#          <new-version>
+#        * <postinst> `abort-remove'
+#        * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
+#          <failed-install-package> <version> `removing'
+#          <conflicting-package> <version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+case "$1" in
+    configure)
+        # Fix for old packages not managing pyc files
+        for d in /usr/lib/python*/site-packages/scrapy ; do
+	    if [ -d "$d" ] ; then
+	        echo "Removing old python byte-compiled files in $d"
+	        rm -rf "$d"
+	    fi
+	done
+
+        # Create user to run the service as
+        if [ -z "`id -u scrapy 2> /dev/null`" ]; then
+          adduser --system --home /var/lib/scrapy --gecos "scrapy" \
+                  --no-create-home --disabled-password \
+                  --quiet scrapy || true
+        fi
+        if [ ! -d /var/run/scrapy ]; then
+          mkdir /var/run/scrapy
+          chown scrapy:nogroup /var/run/scrapy
+        fi
+
+        chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+

debian/scrapy.postrm

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+
+if [ purge = "$1" ]; then
+   if [ -x "$(command -v deluser)" ]; then
+     deluser --quiet --system scrapy > /dev/null || true
+  else
+     echo >&2 "not removing scrapy system account because deluser command was not found"
+  fi
+  rm -rf /var/run/scrapy
+fi
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0
+
+

debian/scrapy.upstart

@@ -0,0 +1,16 @@
+# Scrapy service
+
+description	"Scrapy web crawler"
+author          "Pablo Hoffman <pablo@pablohoffman.com>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+script
+    [ -r /etc/scrapy/environment ] && . /etc/scrapy/environment
+    export PYTHONPATH=/etc/scrapy:$PYTHONPATH
+    export SCRAPY_SETTINGS_MODULE=service_conf
+    twistd -ny /usr/share/scrapy/scrapy.tac \
+        -u scrapy -g nogroup \
+        -l /var/log/scrapy/service.log
+end script

debian/watch

@@ -0,0 +1,2 @@
+version=3
+http://scrapy.org/releases/(\d\.\d)/Scrapy-([\d\.]+)\.tar\.gz