Merge pull request #330 from alexcepoi/contracts_fix

.travis.yml

@@ -1,19 +1,26 @@
 language: python
-python:
-    - 2.6
-    - 2.7
 env:
-    - BUILDENV=lucid
-    - BUILDENV=precise
-    - BUILDENV=latest
+    TRAVISBUG="#1027"
+matrix:
+    exclude:
+        - env: TRAVISBUG="#1027"
+    include:
+        - python: "2.6"
+          env: BUILDENV=lucid
+        - python: "2.7"
+          env: BUILDENV=precise
+        - python: "2.7"
+          env: BUILDENV=latest
+        - python: "pypy"
+          env: BUILDENV=latest
+    allow_failures:
+        - python: "pypy"
+          env: BUILDENV=latest
 install:
     - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt
+    - pip install --use-mirrors .
 script:
     - trial scrapy
-branches:
-    only:
-        - master
-        - /^[0-9].*$/
 notifications:
     irc:
         channels:

.travis/requirements-latest.txt

@@ -3,4 +3,4 @@ lxml
 twisted
 boto
 Pillow
-.
+django

.travis/requirements-lucid.txt

@@ -4,4 +4,3 @@ lxml==2.2.4
 twisted==10.0.0
 boto==1.9b
 Pillow<2.0
-.

.travis/requirements-precise.txt

@@ -4,4 +4,4 @@ lxml==2.3.2
 twisted==11.1.0
 boto==2.2.2
 Pillow<2.0
-.
+django==1.3.1

docs/faq.rst

@@ -84,8 +84,8 @@ How can I simulate a user login in my spider?
 
 See :ref:`topics-request-response-ref-request-userlogin`.
 
-Does Scrapy crawl in breath-first or depth-first order?
--------------------------------------------------------
+Does Scrapy crawl in breadth-first or depth-first order?
+--------------------------------------------------------
 
 By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
 basically means that it crawls in `DFO order`_. This order is more convenient

docs/intro/overview.rst

@@ -125,7 +125,7 @@ tag with ``id=specifications``::
 
 .. highlight:: none
 
-An XPath expression to select the description could be::
+An XPath expression to select the file size could be::
 
    //div[@id='specifications']/p[2]/text()[2]
 

docs/news.rst

@@ -3,17 +3,122 @@
 Release notes
 =============
 
-0.18 (unreleased)
------------------
+0.20 (not released yet)
+-----------------------
 
-- :ref:`benchmarking`
-- moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
-- add scrapy commands using external libraries (:issue:`260`)
-- added ``--pdb`` option to ``scrapy`` command line tool
-- added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
-- several improvements to spider contracts
+- Request/Response url/body attributes are now immutable (modifying them had
+  been deprecated for a long time)
+
+0.18.1 (released 2013-08-27)
+----------------------------
+
+- remove extra import added by cherry picked changes (:commit:`d20304e`)
+- fix crawling tests under twisted pre 11.0.0 (:commit:`1994f38`)
+- py26 can not format zero length fields {} (:commit:`abf756f`)
+- test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`)
+- Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`)
+- do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`)
+- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
+- limit travis-ci build matrix (:commit:`3b01bb8`)
+- Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`)
+- Fixed so it refers to the correct folder (:commit:`3283809`)
+- added quantal & raring to support ubuntu releases (:commit:`1411923`)
+- fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`)
+- fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`)
+- minor updates to 0.18 release notes (:commit:`c45e5f1`)
+- fix contributters list format (:commit:`0b60031`)
+
+0.18.0 (released 2013-08-09)
+----------------------------
+
+- Lot of improvements to testsuite run using Tox, including a way to test on pypi
+- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
+- Use lxml recover option to parse sitemaps (:issue:`347`)
+- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
+- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`)
+- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`)
+- Support `dont_cache` request meta flag (:issue:`19`)
+- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`)
+- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`)
+- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`)
+- Support nested items in xml exporter (:issue:`66`)
+- Improve cookies handling performance (:issue:`77`)
+- Log dupe filtered requests once (:issue:`105`)
+- Split redirection middleware into status and meta based middlewares (:issue:`78`)
+- Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`)
+- Support xpath form selection on `FormRequest.from_response` (:issue:`185`)
+- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`)
+- Bugfix signal dispatching on pypi interpreter (:issue:`205`)
+- Improve request delay and concurrency handling (:issue:`206`)
+- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`)
+- Allow customization of messages logged by engine (:issue:`214`)
+- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`)
+- Extend Scrapy commands using setuptools entry points (:issue:`260`)
+- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`)
+- Support `settings.getdict` (:issue:`269`)
+- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`)
+- Added `Item.copy` (:issue:`290`)
+- Collect idle downloader slots (:issue:`297`)
+- Add `ftp://` scheme downloader handler (:issue:`329`)
+- Added downloader benchmark webserver and spider tools :ref:`benchmarking`
+- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
+- Add scrapy commands using external libraries (:issue:`260`)
+- Added ``--pdb`` option to ``scrapy`` command line tool
+- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
+- Several improvements to spider contracts
 - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
-  MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
+- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
+- added from_crawler method to spiders
+- added system tests with mock server
+- more improvements to Mac OS compatibility (thanks Alex Cepoi)
+- several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez)
+- support custom download slots
+- added --spider option to "shell" command.
+- log overridden settings when scrapy starts
+
+Thanks to everyone who contribute to this release. Here is a list of
+contributors sorted by number of commits::
+
+    130 Pablo Hoffman <pablo@...>
+     97 Daniel Graña <dangra@...>
+     20 Nicolás Ramírez <nramirez.uy@...>
+     13 Mikhail Korobov <kmike84@...>
+     12 Pedro Faustino <pedrobandim@...>
+     11 Steven Almeroth <sroth77@...>
+      5 Rolando Espinoza La fuente <darkrho@...>
+      4 Michal Danilak <mimino.coder@...>
+      4 Alex Cepoi <alex.cepoi@...>
+      4 Alexandr N Zamaraev (aka tonal) <tonal@...>
+      3 paul <paul.tremberth@...>
+      3 Martin Olveyra <molveyra@...>
+      3 Jordi Llonch <llonchj@...>
+      3 arijitchakraborty <myself.arijit@...>
+      2 Shane Evans <shane.evans@...>
+      2 joehillen <joehillen@...>
+      2 Hart <HartSimha@...>
+      2 Dan <ellisd23@...>
+      1 Zuhao Wan <wanzuhao@...>
+      1 whodatninja <blake@...>
+      1 vkrest <v.krestiannykov@...>
+      1 tpeng <pengtaoo@...>
+      1 Tom Mortimer-Jones <tom@...>
+      1 Rocio Aramberri <roschegel@...>
+      1 Pedro <pedro@...>
+      1 notsobad <wangxiaohugg@...>
+      1 Natan L <kuyanatan.nlao@...>
+      1 Mark Grey <mark.grey@...>
+      1 Luan <luanpab@...>
+      1 Libor Nenadál <libor.nenadal@...>
+      1 Juan M Uys <opyate@...>
+      1 Jonas Brunsgaard <jonas.brunsgaard@...>
+      1 Ilya Baryshev <baryshev@...>
+      1 Hasnain Lakhani <m.hasnain.lakhani@...>
+      1 Emanuel Schorsch <emschorsch@...>
+      1 Chris Tilden <chris.tilden@...>
+      1 Capi Etheriel <barraponto@...>
+      1 cacovsky <amarquesferraz@...>
+      1 Berend Iwema <berend@...>
+
 
 0.16.5 (released 2013-05-30)
 ----------------------------

docs/topics/downloader-middleware.rst

@@ -533,6 +533,19 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
+HttpCompressionMiddleware Settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: COMPRESSION_ENABLED
+
+COMPRESSION_ENABLED
+^^^^^^^^^^^^^^^^^^^
+
+Default: ``True``
+
+Whether the Compression middleware will be enabled.
+
+
 ChunkedTransferMiddleware
 -------------------------
 

docs/topics/email.rst

@@ -39,11 +39,11 @@ MailSender class reference
 ==========================
 
 MailSender is the preferred class to use for sending emails from Scrapy, as it
-uses `Twisted non-blocking IO`_, like the rest of the framework. 
+uses `Twisted non-blocking IO`_, like the rest of the framework.
 
 .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
 
-    :param smtphost: the SMTP host to use for sending the emails. If omitted, the 
+    :param smtphost: the SMTP host to use for sending the emails. If omitted, the
       :setting:`MAIL_HOST` setting will be used.
     :type smtphost: str
 
@@ -62,6 +62,12 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
     :param smtpport: the SMTP port to connect to
     :type smtpport: int
 
+    :param smtptls: enforce using SMTP STARTTLS
+    :type smtpport: boolean
+
+    :param smtpssl: enforce using a secure SSL connection
+    :type smtpport: boolean
+
     .. classmethod:: from_settings(settings)
 
         Instantiate using a Scrapy settings object, which will respect
@@ -148,3 +154,21 @@ MAIL_PASS
 Default: ``None``
 
 Password to use for SMTP authentication, along with :setting:`MAIL_USER`.
+
+.. setting:: MAIL_TLS
+
+MAIL_TLS
+---------
+
+Default: ``False``
+
+Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS.
+
+.. setting:: MAIL_SSL
+
+MAIL_SSL
+---------
+
+Default: ``False``
+
+Enforce connecting using an SSL encrypted connection

docs/topics/images.rst

@@ -24,10 +24,13 @@ being scheduled for download, and connects those items that arrive containing
 the same image, to that queue. This avoids downloading the same image more than
 once when it's shared by several items.
 
-The `Python Imaging Library`_ is used for thumbnailing and normalizing images
-to JPEG/RGB format, so you need to install that library in order to use the
-images pipeline.
+`Pillow`_ is used for thumbnailing and normalizing images to JPEG/RGB format,
+so you need to install this library in order to use the images pipeline.
+`Python Imaging Library`_ (PIL) should also work in most cases, but it
+is known to cause troubles in some setups, so we recommend to use `Pillow`_
+instead of `PIL <Python Imaging Library>`_.
 
+.. _Pillow: https://github.com/python-imaging/Pillow
 .. _Python Imaging Library: http://www.pythonware.com/products/pil/
 
 Using the Images Pipeline
@@ -107,7 +110,7 @@ File system storage
 -------------------
 
 The images are stored in files (one per image), using a `SHA1 hash`_ of their
-URLs for the file names. 
+URLs for the file names.
 
 For example, the following image URL::
 
@@ -167,7 +170,7 @@ When you use this feature, the Images Pipeline will create thumbnails of the
 each specified size with this format::
 
     <IMAGES_STORE>/thumbs/<size_name>/<image_id>.jpg
-  
+
 Where:
 
 * ``<size_name>`` is the one specified in the :setting:`IMAGES_THUMBS`

docs/topics/request-response.rst

@@ -318,7 +318,7 @@ key-value fields, you can return a :class:`FormRequest` object (from your
 spider) like this::
 
    return [FormRequest(url="http://www.example.com/post/action",
-                       formdata={'name': 'John Doe', age: '27'},
+                       formdata={'name': 'John Doe', 'age': '27'},
                        callback=self.after_post)]
 
 .. _topics-request-response-ref-request-userlogin:

docs/topics/spiders.rst

@@ -61,7 +61,8 @@ Spiders receive arguments in their constructors::
     class MySpider(BaseSpider):
         name = 'myspider'
 
-        def __init__(self, category=None):
+        def __init__(self, category=None, *args, **kwargs):
+            super(MySpider, self).__init__(*args, **kwargs)
             self.start_urls = ['http://www.example.com/categories/%s' % category]
             # ...
 

docs/topics/ubuntu.rst

@@ -23,7 +23,15 @@ with command::
     lsb_release -cs
 
 Supported Ubuntu releases are: ``karmic``, ``lucid``, ``maverick``, ``natty``,
-``oneiric``, ``precise``.
+``oneiric``, ``precise``, ``quantal``, ``raring``.
+
+For Ubuntu Raring (13.04)::
+
+    deb http://archive.scrapy.org/ubuntu raring main
+
+For Ubuntu Quantal (12.10)::
+
+    deb http://archive.scrapy.org/ubuntu quantal main
 
 For Ubuntu Precise (12.04)::
 

requirements.txt

@@ -0,0 +1,5 @@
+Twisted>=8.0
+w3lib>=1.2
+queuelib
+lxml
+pyOpenSSL

scrapy/VERSION

@@ -1 +1 @@
-0.17.0
+0.19.0

scrapy/__init__.py

@@ -7,7 +7,7 @@ version_info = tuple(__version__.split('.')[:3])
 
 import sys, os, warnings
 
-if sys.version_info < (2,6):
+if sys.version_info < (2, 6):
     print "Scrapy %s requires Python 2.6 or above" % __version__
     sys.exit(1)
 

scrapy/commands/crawl.py

@@ -34,7 +34,7 @@ class Command(ScrapyCommand):
                 self.settings.overrides['FEED_URI'] = opts.output
             valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
             if opts.output_format not in valid_output_formats:
-                raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats))
+                raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats))
             self.settings.overrides['FEED_FORMAT'] = opts.output_format
 
     def run(self, args, opts):

scrapy/contrib/downloadermiddleware/httpcompression.py

@@ -3,12 +3,19 @@ import zlib
 from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
+from scrapy.exceptions import NotConfigured
 
 
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-
+    
+    @classmethod
+    def from_crawler(cls, crawler):
+        if not crawler.settings.getbool('COMPRESSION_ENABLED'):
+            raise NotConfigured
+        return cls()
+    
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding', 'x-gzip,gzip,deflate')
 

scrapy/contrib/downloadermiddleware/retry.py

@@ -18,14 +18,16 @@ About HTTP errors to consider:
   indicate server overload, which would be something we want to retry
 """
 
-from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
-                                   ConnectionRefusedError, ConnectionDone, ConnectError, \
-                                   ConnectionLost, TCPTimedOutError
 from twisted.internet.defer import TimeoutError as UserTimeoutError
+from twisted.internet.error import TimeoutError as ServerTimeoutError, \
+        DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \
+        ConnectionLost, TCPTimedOutError
 
 from scrapy import log
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
+from scrapy.xlib.tx import ResponseFailed
+
 
 class RetryMiddleware(object):
 
@@ -33,7 +35,7 @@ class RetryMiddleware(object):
     # decompress an empty response
     EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
                            ConnectionRefusedError, ConnectionDone, ConnectError,
-                           ConnectionLost, TCPTimedOutError,
+                           ConnectionLost, TCPTimedOutError, ResponseFailed,
                            IOError)
 
     def __init__(self, settings):

scrapy/contrib/exporter/__init__.py

@@ -3,13 +3,14 @@ Item Exporters are used to export/serialize items into different formats.
 """
 
 import csv
+import sys
 import pprint
 import marshal
 import json
 import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 from scrapy.utils.serialize import ScrapyJSONEncoder
-
+from scrapy.item import BaseItem
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \
     'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \
@@ -141,9 +142,23 @@ class XmlItemExporter(BaseItemExporter):
             for value in serialized_value:
                 self._export_xml_field('value', value)
         else:
-            self.xg.characters(serialized_value)
+            self._xg_characters(serialized_value)
         self.xg.endElement(name)
 
+    # Workaround for http://bugs.python.org/issue17606
+    # Before Python 2.7.4 xml.sax.saxutils required bytes;
+    # since 2.7.4 it requires unicode. The bug is likely to be
+    # fixed in 2.7.6, but 2.7.6 will still support unicode,
+    # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
+    if sys.version_info[:3] >= (2, 7, 4):
+        def _xg_characters(self, serialized_value):
+            if not isinstance(serialized_value, unicode):
+                serialized_value = serialized_value.decode(self.encoding)
+            return self.xg.characters(serialized_value)
+    else:
+        def _xg_characters(self, serialized_value):
+            return self.xg.characters(serialized_value)
+
 
 class CsvItemExporter(BaseItemExporter):
 
@@ -183,7 +198,7 @@ class PickleItemExporter(BaseItemExporter):
 
     def __init__(self, file, protocol=2, **kwargs):
         self._configure(kwargs)
-        self.file =file
+        self.file = file
         self.protocol = protocol
 
     def export_item(self, item):
@@ -200,7 +215,6 @@ class MarshalItemExporter(BaseItemExporter):
     def export_item(self, item):
         marshal.dump(dict(self._get_serialized_fields(item)), self.file)
 
-
 class PprintItemExporter(BaseItemExporter):
 
     def __init__(self, file, **kwargs):
@@ -210,3 +224,30 @@ class PprintItemExporter(BaseItemExporter):
     def export_item(self, item):
         itemdict = dict(self._get_serialized_fields(item))
         self.file.write(pprint.pformat(itemdict) + '\n')
+
+class PythonItemExporter(BaseItemExporter):
+    """The idea behind this exporter is to have a mechanism to serialize items
+    to built-in python types so any serialization library (like
+    json, msgpack, binc, etc) can be used on top of it. Its main goal is to
+    seamless support what BaseItemExporter does plus nested items.
+    """
+
+    def serialize_field(self, field, name, value):
+        serializer = field.get('serializer', self._serialize_value)
+        return serializer(value)
+
+    def _serialize_value(self, value):
+        if isinstance(value, BaseItem):
+            return self.export_item(value)
+        if isinstance(value, dict):
+            return dict(self._serialize_dict(value))
+        if hasattr(value, '__iter__'):
+            return [self._serialize_value(v) for v in value]
+        return self._to_str_if_unicode(value)
+
+    def _serialize_dict(self, value):
+        for key, val in value.iteritems():
+            yield key, self._serialize_value(val)
+    
+    def export_item(self, item):
+        return dict(self._get_serialized_fields(item))

scrapy/contrib/pipeline/files.py

@@ -0,0 +1,268 @@
+"""
+Files Pipeline
+"""
+
+import hashlib
+import os
+import os.path
+import rfc822
+import time
+import urlparse
+from collections import defaultdict
+from cStringIO import StringIO
+
+from twisted.internet import defer, threads
+
+from scrapy import log
+from scrapy.contrib.pipeline.media import MediaPipeline
+from scrapy.exceptions import NotConfigured, IgnoreRequest
+from scrapy.http import Request
+from scrapy.utils.misc import md5sum
+
+
+class FileException(Exception):
+    """General media error exception"""
+
+
+class FSFilesStore(object):
+
+    def __init__(self, basedir):
+        if '://' in basedir:
+            basedir = basedir.split('://', 1)[1]
+        self.basedir = basedir
+        self._mkdir(self.basedir)
+        self.created_directories = defaultdict(set)
+
+    def persist_file(self, key, buf, info, meta=None, headers=None):
+        absolute_path = self._get_filesystem_path(key)
+        self._mkdir(os.path.dirname(absolute_path), info)
+        with open(absolute_path, 'wb') as f:
+            f.write(buf.getvalue())
+
+    def stat_file(self, key, info):
+        absolute_path = self._get_filesystem_path(key)
+        try:
+            last_modified = os.path.getmtime(absolute_path)
+        except:  # FIXME: catching everything!
+            return {}
+
+        with open(absolute_path, 'rb') as f:
+            checksum = md5sum(f)
+
+        return {'last_modified': last_modified, 'checksum': checksum}
+
+    def _get_filesystem_path(self, key):
+        path_comps = key.split('/')
+        return os.path.join(self.basedir, *path_comps)
+
+    def _mkdir(self, dirname, domain=None):
+        seen = self.created_directories[domain] if domain else set()
+        if dirname not in seen:
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+            seen.add(dirname)
+
+
+class S3FilesStore(object):
+
+    AWS_ACCESS_KEY_ID = None
+    AWS_SECRET_ACCESS_KEY = None
+
+    POLICY = 'public-read'
+    HEADERS = {
+        'Cache-Control': 'max-age=172800',
+    }
+
+    def __init__(self, uri):
+        assert uri.startswith('s3://')
+        self.bucket, self.prefix = uri[5:].split('/', 1)
+
+    def stat_file(self, key, info):
+        def _onsuccess(boto_key):
+            checksum = boto_key.etag.strip('"')
+            last_modified = boto_key.last_modified
+            modified_tuple = rfc822.parsedate_tz(last_modified)
+            modified_stamp = int(rfc822.mktime_tz(modified_tuple))
+            return {'checksum': checksum, 'last_modified': modified_stamp}
+
+        return self._get_boto_key(key).addCallback(_onsuccess)
+
+    def _get_boto_bucket(self):
+        from boto.s3.connection import S3Connection
+        # disable ssl (is_secure=False) because of this python bug:
+        # http://bugs.python.org/issue5103
+        c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
+        return c.get_bucket(self.bucket, validate=False)
+
+    def _get_boto_key(self, key):
+        b = self._get_boto_bucket()
+        key_name = '%s%s' % (self.prefix, key)
+        return threads.deferToThread(b.get_key, key_name)
+
+    def persist_file(self, key, buf, info, meta=None, headers=None):
+        """Upload file to S3 storage"""
+        b = self._get_boto_bucket()
+        key_name = '%s%s' % (self.prefix, key)
+        k = b.new_key(key_name)
+        if meta:
+            for metakey, metavalue in meta.iteritems():
+                k.set_metadata(metakey, str(metavalue))
+        h = self.HEADERS.copy()
+        if headers:
+            h.update(headers)
+        buf.seek(0)
+        return threads.deferToThread(k.set_contents_from_string, buf.getvalue(),
+                                     headers=h, policy=self.POLICY)
+
+
+class FilesPipeline(MediaPipeline):
+    """Abstract pipeline that implement the file downloading
+
+    This pipeline tries to minimize network transfers and file processing,
+    doing stat of the files and determining if file is new, uptodate or
+    expired.
+
+    `new` files are those that pipeline never processed and needs to be
+        downloaded from supplier site the first time.
+
+    `uptodate` files are the ones that the pipeline processed and are still
+        valid files.
+
+    `expired` files are those that pipeline already processed but the last
+        modification was made long time ago, so a reprocessing is recommended to
+        refresh it in case of change.
+
+    """
+
+    MEDIA_NAME = "file"
+    EXPIRES = 90
+    STORE_SCHEMES = {
+        '': FSFilesStore,
+        'file': FSFilesStore,
+        's3': S3FilesStore,
+    }
+
+    def __init__(self, store_uri, download_func=None):
+        if not store_uri:
+            raise NotConfigured
+        self.store = self._get_store(store_uri)
+        super(FilesPipeline, self).__init__(download_func=download_func)
+
+    @classmethod
+    def from_settings(cls, settings):
+        s3store = cls.STORE_SCHEMES['s3']
+        s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
+        s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+
+        cls.EXPIRES = settings.getint('FILES_EXPIRES', 90)
+        store_uri = settings['FILES_STORE']
+        return cls(store_uri)
+
+    def _get_store(self, uri):
+        if os.path.isabs(uri):  # to support win32 paths like: C:\\some\dir
+            scheme = 'file'
+        else:
+            scheme = urlparse.urlparse(uri).scheme
+        store_cls = self.STORE_SCHEMES[scheme]
+        return store_cls(uri)
+
+    def media_to_download(self, request, info):
+        def _onsuccess(result):
+            if not result:
+                return  # returning None force download
+
+            last_modified = result.get('last_modified', None)
+            if not last_modified:
+                return  # returning None force download
+
+            age_seconds = time.time() - last_modified
+            age_days = age_seconds / 60 / 60 / 24
+            if age_days > self.EXPIRES:
+                return  # returning None force download
+
+            referer = request.headers.get('Referer')
+            log.msg(format='File (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>',
+                    level=log.DEBUG, spider=info.spider,
+                    medianame=self.MEDIA_NAME, request=request, referer=referer)
+            self.inc_stats(info.spider, 'uptodate')
+
+            checksum = result.get('checksum', None)
+            return {'url': request.url, 'path': key, 'checksum': checksum}
+
+        key = self.file_key(request.url)
+        dfd = defer.maybeDeferred(self.store.stat_file, key, info)
+        dfd.addCallbacks(_onsuccess, lambda _: None)
+        dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_file')
+        return dfd
+
+    def media_failed(self, failure, request, info):
+        if not isinstance(failure.value, IgnoreRequest):
+            referer = request.headers.get('Referer')
+            log.msg(format='File (unknown-error): Error downloading '
+                           '%(medianame)s from %(request)s referred in '
+                           '<%(referer)s>: %(exception)s',
+                    level=log.WARNING, spider=info.spider, exception=failure.value,
+                    medianame=self.MEDIA_NAME, request=request, referer=referer)
+
+        raise FileException
+
+    def media_downloaded(self, response, request, info):
+        referer = request.headers.get('Referer')
+
+        if response.status != 200:
+            log.msg(format='File (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>',
+                    level=log.WARNING, spider=info.spider,
+                    status=response.status, request=request, referer=referer)
+            raise FileException('download-error')
+
+        if not response.body:
+            log.msg(format='File (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content',
+                    level=log.WARNING, spider=info.spider,
+                    request=request, referer=referer)
+            raise FileException('empty-content')
+
+        status = 'cached' if 'cached' in response.flags else 'downloaded'
+        log.msg(format='File (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>',
+                level=log.DEBUG, spider=info.spider,
+                status=status, request=request, referer=referer)
+        self.inc_stats(info.spider, status)
+
+        try:
+            key = self.file_key(request.url)
+            checksum = self.file_downloaded(response, request, info)
+        except FileException as exc:
+            whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s'
+            log.msg(format=whyfmt, level=log.WARNING, spider=info.spider,
+                    request=request, referer=referer, errormsg=str(exc))
+            raise
+        except Exception as exc:
+            whyfmt = 'File (unknown-error): Error processing image from %(request)s referred in <%(referer)s>'
+            log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider)
+            raise FileException(str(exc))
+
+        return {'url': request.url, 'path': key, 'checksum': checksum}
+
+    def inc_stats(self, spider, status):
+        spider.crawler.stats.inc_value('file_count', spider=spider)
+        spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)
+
+    ### Overridable Interface
+    def get_media_requests(self, item, info):
+        return [Request(x) for x in item.get('file_urls', [])]
+
+    def file_key(self, url):
+        media_guid = hashlib.sha1(url).hexdigest()
+        media_ext = os.path.splitext(url)[1]
+        return 'full/%s%s' % (media_guid, media_ext)
+
+    def file_downloaded(self, response, request, info):
+        key = self.file_key(request.url)
+        buf = StringIO(response.body)
+        self.store.persist_file(key, buf, info)
+        checksum = md5sum(buf)
+        return checksum
+
+    def item_completed(self, results, item, info):
+        if 'files' in item.fields:
+            item['files'] = [x for ok, x in results if ok]
+        return item

scrapy/contrib/pipeline/images.py

@@ -4,155 +4,35 @@ Images Pipeline
 See documentation in topics/images.rst
 """
 
-import os
-import time
 import hashlib
-import urlparse
-import rfc822
 from cStringIO import StringIO
-from collections import defaultdict
 
-from twisted.internet import defer, threads
 from PIL import Image
 
-from scrapy import log
 from scrapy.utils.misc import md5sum
 from scrapy.http import Request
-from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest
-from scrapy.contrib.pipeline.media import MediaPipeline
+from scrapy.exceptions import DropItem
+#TODO: from scrapy.contrib.pipeline.media import MediaPipeline
+from scrapy.contrib.pipeline.files import FileException, FilesPipeline
 
 
 class NoimagesDrop(DropItem):
     """Product with no images exception"""
 
 
-class ImageException(Exception):
+class ImageException(FileException):
     """General image error exception"""
 
 
-class FSImagesStore(object):
-
-    def __init__(self, basedir):
-        if '://' in basedir:
-            basedir = basedir.split('://', 1)[1]
-        self.basedir = basedir
-        self._mkdir(self.basedir)
-        self.created_directories = defaultdict(set)
-
-    def persist_image(self, key, image, buf, info):
-        absolute_path = self._get_filesystem_path(key)
-        self._mkdir(os.path.dirname(absolute_path), info)
-        image.save(absolute_path)
-
-    def stat_image(self, key, info):
-        absolute_path = self._get_filesystem_path(key)
-        try:
-            last_modified = os.path.getmtime(absolute_path)
-        except:  # FIXME: catching everything!
-            return {}
-
-        with open(absolute_path, 'rb') as imagefile:
-            checksum = md5sum(imagefile)
-
-        return {'last_modified': last_modified, 'checksum': checksum}
-
-    def _get_filesystem_path(self, key):
-        path_comps = key.split('/')
-        return os.path.join(self.basedir, *path_comps)
-
-    def _mkdir(self, dirname, domain=None):
-        seen = self.created_directories[domain] if domain else set()
-        if dirname not in seen:
-            if not os.path.exists(dirname):
-                os.makedirs(dirname)
-            seen.add(dirname)
-
-
-class S3ImagesStore(object):
-
-    AWS_ACCESS_KEY_ID = None
-    AWS_SECRET_ACCESS_KEY = None
-
-    POLICY = 'public-read'
-    HEADERS = {
-        'Cache-Control': 'max-age=172800',
-        'Content-Type': 'image/jpeg',
-    }
-
-    def __init__(self, uri):
-        assert uri.startswith('s3://')
-        self.bucket, self.prefix = uri[5:].split('/', 1)
-
-    def stat_image(self, key, info):
-        def _onsuccess(boto_key):
-            checksum = boto_key.etag.strip('"')
-            last_modified = boto_key.last_modified
-            modified_tuple = rfc822.parsedate_tz(last_modified)
-            modified_stamp = int(rfc822.mktime_tz(modified_tuple))
-            return {'checksum': checksum, 'last_modified': modified_stamp}
-
-        return self._get_boto_key(key).addCallback(_onsuccess)
-
-    def _get_boto_bucket(self):
-        from boto.s3.connection import S3Connection
-        # disable ssl (is_secure=False) because of this python bug:
-        # http://bugs.python.org/issue5103
-        c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
-        return c.get_bucket(self.bucket, validate=False)
-
-    def _get_boto_key(self, key):
-        b = self._get_boto_bucket()
-        key_name = '%s%s' % (self.prefix, key)
-        return threads.deferToThread(b.get_key, key_name)
-
-    def persist_image(self, key, image, buf, info):
-        """Upload image to S3 storage"""
-        width, height = image.size
-        b = self._get_boto_bucket()
-        key_name = '%s%s' % (self.prefix, key)
-        k = b.new_key(key_name)
-        k.set_metadata('width', str(width))
-        k.set_metadata('height', str(height))
-        buf.seek(0)
-        return threads.deferToThread(k.set_contents_from_file, buf,
-                                     headers=self.HEADERS, policy=self.POLICY)
-
-
-class ImagesPipeline(MediaPipeline):
-    """Abstract pipeline that implement the image downloading and thumbnail generation logic
-
-    This pipeline tries to minimize network transfers and image processing,
-    doing stat of the images and determining if image is new, uptodate or
-    expired.
-
-    `new` images are those that pipeline never processed and needs to be
-        downloaded from supplier site the first time.
-
-    `uptodate` images are the ones that the pipeline processed and are still
-        valid images.
-
-    `expired` images are those that pipeline already processed but the last
-        modification was made long time ago, so a reprocessing is recommended to
-        refresh it in case of change.
+class ImagesPipeline(FilesPipeline):
+    """Abstract pipeline that implement the image thumbnail generation logic
 
     """
 
     MEDIA_NAME = 'image'
     MIN_WIDTH = 0
     MIN_HEIGHT = 0
-    EXPIRES = 90
     THUMBS = {}
-    STORE_SCHEMES = {
-        '': FSImagesStore,
-        'file': FSImagesStore,
-        's3': S3ImagesStore,
-    }
-
-    def __init__(self, store_uri, download_func=None):
-        if not store_uri:
-            raise NotConfigured
-        self.store = self._get_store(store_uri)
-        super(ImagesPipeline, self).__init__(download_func=download_func)
 
     @classmethod
     def from_settings(cls, settings):
@@ -166,89 +46,11 @@ class ImagesPipeline(MediaPipeline):
         store_uri = settings['IMAGES_STORE']
         return cls(store_uri)
 
-    def _get_store(self, uri):
-        if os.path.isabs(uri):  # to support win32 paths like: C:\\some\dir
-            scheme = 'file'
-        else:
-            scheme = urlparse.urlparse(uri).scheme
-        store_cls = self.STORE_SCHEMES[scheme]
-        return store_cls(uri)
+    def file_key(self, url):
+        return self.image_key(url)
 
-    def media_downloaded(self, response, request, info):
-        referer = request.headers.get('Referer')
-
-        if response.status != 200:
-            log.msg(format='Image (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>',
-                    level=log.WARNING, spider=info.spider,
-                    status=response.status, request=request, referer=referer)
-            raise ImageException('download-error')
-
-        if not response.body:
-            log.msg(format='Image (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content',
-                    level=log.WARNING, spider=info.spider,
-                    request=request, referer=referer)
-            raise ImageException('empty-content')
-
-        status = 'cached' if 'cached' in response.flags else 'downloaded'
-        log.msg(format='Image (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>',
-                level=log.DEBUG, spider=info.spider,
-                status=status, request=request, referer=referer)
-        self.inc_stats(info.spider, status)
-
-        try:
-            key = self.image_key(request.url)
-            checksum = self.image_downloaded(response, request, info)
-        except ImageException as exc:
-            whyfmt = 'Image (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s'
-            log.msg(format=whyfmt, level=log.WARNING, spider=info.spider,
-                    request=request, referer=referer, errormsg=str(exc))
-            raise
-        except Exception as exc:
-            whyfmt = 'Image (unknown-error): Error processing image from %(request)s referred in <%(referer)s>'
-            log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider)
-            raise ImageException(str(exc))
-
-        return {'url': request.url, 'path': key, 'checksum': checksum}
-
-    def media_failed(self, failure, request, info):
-        if not isinstance(failure.value, IgnoreRequest):
-            referer = request.headers.get('Referer')
-            log.msg(format='Image (unknown-error): Error downloading '
-                           '%(medianame)s from %(request)s referred in '
-                           '<%(referer)s>: %(exception)s',
-                    level=log.WARNING, spider=info.spider, exception=failure.value,
-                    medianame=self.MEDIA_NAME, request=request, referer=referer)
-
-        raise ImageException
-
-    def media_to_download(self, request, info):
-        def _onsuccess(result):
-            if not result:
-                return  # returning None force download
-
-            last_modified = result.get('last_modified', None)
-            if not last_modified:
-                return  # returning None force download
-
-            age_seconds = time.time() - last_modified
-            age_days = age_seconds / 60 / 60 / 24
-            if age_days > self.EXPIRES:
-                return  # returning None force download
-
-            referer = request.headers.get('Referer')
-            log.msg(format='Image (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>',
-                    level=log.DEBUG, spider=info.spider,
-                    medianame=self.MEDIA_NAME, request=request, referer=referer)
-            self.inc_stats(info.spider, 'uptodate')
-
-            checksum = result.get('checksum', None)
-            return {'url': request.url, 'path': key, 'checksum': checksum}
-
-        key = self.image_key(request.url)
-        dfd = defer.maybeDeferred(self.store.stat_image, key, info)
-        dfd.addCallbacks(_onsuccess, lambda _: None)
-        dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image')
-        return dfd
+    def file_downloaded(self, response, request, info):
+        return self.image_downloaded(response, request, info)
 
     def image_downloaded(self, response, request, info):
         checksum = None
@@ -256,11 +58,15 @@ class ImagesPipeline(MediaPipeline):
             if checksum is None:
                 buf.seek(0)
                 checksum = md5sum(buf)
-            self.store.persist_image(key, image, buf, info)
+            width, height = image.size
+            self.store.persist_file(
+                key, buf, info,
+                meta={'width': width, 'height': height},
+                headers={'Content-Type': 'image/jpeg'})
         return checksum
 
     def get_images(self, response, request, info):
-        key = self.image_key(request.url)
+        key = self.file_key(request.url)
         orig_image = Image.open(StringIO(response.body))
 
         width, height = orig_image.size
@@ -276,10 +82,6 @@ class ImagesPipeline(MediaPipeline):
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_key, thumb_image, thumb_buf
 
-    def inc_stats(self, spider, status):
-        spider.crawler.stats.inc_value('image_count', spider=spider)
-        spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider)
-
     def convert_image(self, image, size=None):
         if image.format == 'PNG' and image.mode == 'RGBA':
             background = Image.new('RGBA', image.size, (255, 255, 255))
@@ -296,10 +98,6 @@ class ImagesPipeline(MediaPipeline):
         image.save(buf, 'JPEG')
         return image, buf
 
-    def image_key(self, url):
-        image_guid = hashlib.sha1(url).hexdigest()
-        return 'full/%s.jpg' % (image_guid)
-
     def thumb_key(self, url, thumb_id):
         image_guid = hashlib.sha1(url).hexdigest()
         return 'thumbs/%s/%s.jpg' % (thumb_id, image_guid)
@@ -307,6 +105,11 @@ class ImagesPipeline(MediaPipeline):
     def get_media_requests(self, item, info):
         return [Request(x) for x in item.get('image_urls', [])]
 
+    # backwards compatibility
+    def image_key(self, url):
+        media_guid = hashlib.sha1(url).hexdigest()
+        return 'full/%s.jpg' % (media_guid)
+
     def item_completed(self, results, item, info):
         if 'images' in item.fields:
             item['images'] = [x for ok, x in results if ok]

scrapy/contrib/pipeline/media.py

@@ -7,6 +7,7 @@ from scrapy import log
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
+
 class MediaPipeline(object):
 
     LOG_FAILED_RESULTS = True
@@ -65,7 +66,7 @@ class MediaPipeline(object):
         dfd.addCallback(self._check_media_to_download, request, info)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
         dfd.addErrback(log.err, spider=info.spider)
-        return dfd.addBoth(lambda _: wad) # it must return wad at last
+        return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
     def _check_media_to_download(self, result, request, info):
         if result is not None:
@@ -91,11 +92,11 @@ class MediaPipeline(object):
             result.frames = []
             result.stack = None
         info.downloading.remove(fp)
-        info.downloaded[fp] = result # cache result
+        info.downloaded[fp] = result  # cache result
         for wad in info.waiting.pop(fp):
             defer_result(result).chainDeferred(wad)
 
-    ### Overradiable Interface
+    ### Overridable Interface
     def media_to_download(self, request, info):
         """Check request before starting download"""
         pass

scrapy/contrib_exp/downloadermiddleware/decompression.py

@@ -10,7 +10,6 @@ from cStringIO import StringIO
 from tempfile import mktemp
 
 from scrapy import log
-from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 
 

scrapy/core/downloader/handlers/ftp.py

@@ -0,0 +1,104 @@
+"""
+An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
+
+FTP connection parameters are passed using the request meta field:
+- ftp_user (required)
+- ftp_password (required)
+- ftp_passive (by default, enabled) sets FTP connection passive mode
+- ftp_local_filename
+        - If not given, file data will come in the response.body, as a normal scrapy Response,
+        which will imply that the entire file will be on memory.
+        - if given, file data will be saved in a local file with the given name
+        This helps when downloading very big files to avoid memory issues. In addition, for
+        convenience the local file name will also be given in the response body.
+
+The status of the built html response will be, by default
+- 200 in case of success
+- 404 in case specified file was not found in the server (ftp code 550)
+
+or raise corresponding ftp exception otherwise
+
+The matching from server ftp command return codes to html response codes is defined in the
+CODE_MAPPING attribute of the handler class. The key 'default' is used for any code
+that is not explicitly present among the map keys. You may need to overwrite this
+mapping if want a different behaviour than default.
+
+In case of status 200 request, response.headers will come with two keys:
+    'Local Filename' - with the value of the local filename if given
+    'Size' - with size of the downloaded data
+"""
+
+import re
+from urlparse import urlparse
+from cStringIO import StringIO
+
+from twisted.internet import reactor
+from twisted.protocols.ftp import FTPClient, CommandFailed
+from twisted.internet.protocol import Protocol, ClientCreator
+
+from scrapy.http import Response
+from scrapy.responsetypes import responsetypes
+
+class ReceivedDataProtocol(Protocol):
+    def __init__(self, filename=None):
+        self.__filename = filename
+        self.body = open(filename, "w") if filename else StringIO()
+        self.size = 0
+
+    def dataReceived(self, data):
+        self.body.write(data)
+        self.size += len(data)
+
+    @property
+    def filename(self):
+        return self.__filename
+
+    def close(self):
+        self.body.close() if self.filename else self.body.reset()
+
+_CODE_RE = re.compile("\d+")
+class FTPDownloadHandler(object):
+
+    CODE_MAPPING = {
+        "550": 404,
+        "default": 503,
+    }
+
+    def __init__(self, setting):
+        pass
+
+    def download_request(self, request, spider):
+        parsed_url = urlparse(request.url)
+        creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],
+                                    request.meta["ftp_password"],
+                                    passive=request.meta.get("ftp_passive", 1))
+        return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
+                                request, parsed_url.path)
+
+    def gotClient(self, client, request, filepath):
+        self.client = client
+        protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
+        return client.retrieveFile(filepath, protocol)\
+                .addCallbacks(callback=self._build_response,
+                        callbackArgs=(request, protocol),
+                        errback=self._failed,
+                        errbackArgs=(request,))
+    
+    def _build_response(self, result, request, protocol):
+        self.result = result
+        respcls = responsetypes.from_args(url=request.url)
+        protocol.close()
+        body = protocol.filename or protocol.body.read()
+        headers = {"local filename": protocol.filename or '', "size": protocol.size}
+        return respcls(url=request.url, status=200, body=body, headers=headers)
+
+    def _failed(self, result, request):
+        message = result.getErrorMessage()
+        if result.type == CommandFailed:
+            m = _CODE_RE.search(message)
+            if m:
+                ftpcode = m.group()
+                httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
+                return Response(url=request.url, status=httpcode, body=message)
+        raise result.type(result.value)
+

scrapy/core/downloader/handlers/http11.py

@@ -7,24 +7,24 @@ from urlparse import urldefrag
 from zope.interface import implements
 from twisted.internet import defer, reactor, protocol
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.http import PotentialDataLoss
 from twisted.web.iweb import IBodyProducer
 from twisted.internet.error import TimeoutError
+from twisted.web.http import PotentialDataLoss
 from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
-        ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint
+    HTTPConnectionPool, TCP4ClientEndpoint
 
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
-from scrapy import log
-
 
 
 class HTTP11DownloadHandler(object):
 
     def __init__(self, settings):
         self._pool = HTTPConnectionPool(reactor, persistent=True)
+        self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
+        self._pool._factory.noisy = False
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         self._contextFactory = self._contextFactoryClass()
 
@@ -54,7 +54,7 @@ class ScrapyAgent(object):
         if proxy:
             scheme, _, host, port, _ = _parse(proxy)
             endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout,
-                bindAddress=bindaddress)
+                                          bindAddress=bindaddress)
             return self._ProxyAgent(endpoint)
 
         return self._Agent(reactor, contextFactory=self._contextFactory,
@@ -144,10 +144,11 @@ class _ResponseReader(protocol.Protocol):
     def connectionLost(self, reason):
         if self._finished.called:
             return
+
         body = self._bodybuf.getvalue()
         if reason.check(ResponseDone):
             self._finished.callback((self._txresponse, body, None))
-        elif reason.check(PotentialDataLoss, ResponseFailed):
+        elif reason.check(PotentialDataLoss):
             self._finished.callback((self._txresponse, body, ['partial']))
         else:
             self._finished.errback(reason)

scrapy/core/downloader/webclient.py

@@ -1,11 +1,9 @@
 from time import time
 from urlparse import urlparse, urlunparse, urldefrag
 
-from twisted.internet.ssl import ClientContextFactory
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
 from twisted.internet import defer
-from OpenSSL import SSL
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
@@ -60,12 +58,15 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.factory.gotHeaders(self.headers)
 
     def connectionLost(self, reason):
+        self._connection_lost_reason = reason
         HTTPClient.connectionLost(self, reason)
         self.factory.noPage(reason)
 
     def handleResponse(self, response):
         if self.factory.method.upper() == 'HEAD':
             self.factory.page('')
+        elif self.length is not None and self.length > 0:
+            self.factory.noPage(self._connection_lost_reason)
         else:
             self.factory.page(response)
         self.transport.loseConnection()

scrapy/http/common.py

@@ -1,10 +1,6 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-def deprecated_setter(setter, attrname):
+def obsolete_setter(setter, attrname):
     def newsetter(self, value):
         c = self.__class__.__name__
-        warnings.warn("Don't modify %s.%s attribute, use %s.replace() instead" % \
-            (c, attrname, c), ScrapyDeprecationWarning, stacklevel=2)
-        return setter(self, value)
+        msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c)
+        raise AttributeError(msg)
     return newsetter

scrapy/http/cookies.py

@@ -22,7 +22,10 @@ class CookieJar(object):
 
         # the cookiejar implementation iterates through all domains
         # instead we restrict to potential matches on the domain
-        req_host = urlparse_cached(request).netloc
+        req_host = urlparse_cached(request).hostname
+        if not req_host:
+            return
+
         if not IPV4_RE.search(req_host):
             hosts = potential_domain_matches(req_host)
             if req_host.find(".") == -1:

scrapy/http/request/__init__.py

@@ -13,7 +13,7 @@ from scrapy.http.headers import Headers
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.decorator import deprecated
 from scrapy.utils.url import escape_ajax
-from scrapy.http.common import deprecated_setter
+from scrapy.http.common import obsolete_setter
 
 class Request(object_ref):
 
@@ -60,7 +60,7 @@ class Request(object_ref):
         if ':' not in self._url:
             raise ValueError('Missing scheme in request url: %s' % self._url)
 
-    url = property(_get_url, deprecated_setter(_set_url, 'url'))
+    url = property(_get_url, obsolete_setter(_set_url, 'url'))
 
     def _get_body(self):
         return self._body
@@ -78,7 +78,7 @@ class Request(object_ref):
         else:
             raise TypeError("Request body must either str or unicode. Got: '%s'" % type(body).__name__)
 
-    body = property(_get_body, deprecated_setter(_set_body, 'body'))
+    body = property(_get_body, obsolete_setter(_set_body, 'body'))
 
     @property
     def encoding(self):

scrapy/http/response/__init__.py

@@ -9,7 +9,7 @@ import copy
 
 from scrapy.http.headers import Headers
 from scrapy.utils.trackref import object_ref
-from scrapy.http.common import deprecated_setter
+from scrapy.http.common import obsolete_setter
 
 class Response(object_ref):
 
@@ -39,7 +39,7 @@ class Response(object_ref):
             raise TypeError('%s url must be str, got %s:' % (type(self).__name__, \
                 type(url).__name__))
 
-    url = property(_get_url, deprecated_setter(_set_url, 'url'))
+    url = property(_get_url, obsolete_setter(_set_url, 'url'))
 
     def _get_body(self):
         return self._body
@@ -56,7 +56,7 @@ class Response(object_ref):
             raise TypeError("Response body must either str or unicode. Got: '%s'" \
                 % type(body).__name__)
 
-    body = property(_get_body, deprecated_setter(_set_body, 'body'))
+    body = property(_get_body, obsolete_setter(_set_body, 'body'))
 
     def __str__(self):
         return "<%d %s>" % (self.status, self.url)

scrapy/mail.py

@@ -11,7 +11,7 @@ from email.MIMEText import MIMEText
 from email.Utils import COMMASPACE, formatdate
 from email import Encoders
 
-from twisted.internet import defer, reactor
+from twisted.internet import defer, reactor, ssl
 from twisted.mail.smtp import ESMTPSenderFactory
 
 from scrapy import log
@@ -19,18 +19,21 @@ from scrapy import log
 class MailSender(object):
 
     def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
-            smtpuser=None, smtppass=None, smtpport=25, debug=False):
+            smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
         self.smtphost = smtphost
         self.smtpport = smtpport
         self.smtpuser = smtpuser
         self.smtppass = smtppass
+        self.smtptls = smtptls
+        self.smtpssl = smtpssl
         self.mailfrom = mailfrom
         self.debug = debug
 
     @classmethod
     def from_settings(cls, settings):
         return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'],
-            settings['MAIL_PASS'], settings.getint('MAIL_PORT'))
+            settings['MAIL_PASS'], settings.getint('MAIL_PORT'),
+            settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
 
     def send(self, to, subject, body, cc=None, attachs=(), _callback=None):
         if attachs:
@@ -91,7 +94,12 @@ class MailSender(object):
         d = defer.Deferred()
         factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
             to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \
-            requireTransportSecurity=False)
+            requireTransportSecurity=self.smtptls)
         factory.noisy = False
-        reactor.connectTCP(self.smtphost, self.smtpport, factory)
+
+        if self.smtpssl:
+            reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
+        else:
+            reactor.connectTCP(self.smtphost, self.smtpport, factory)
+
         return d

scrapy/resolver.py

@@ -1,5 +1,3 @@
-import socket
-
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
 

scrapy/settings/default_settings.py

@@ -26,6 +26,8 @@ CLOSESPIDER_ERRORCOUNT = 0
 
 COMMANDS_MODULE = ''
 
+COMPRESSION_ENABLED = True
+
 CONCURRENT_ITEMS = 100
 
 CONCURRENT_REQUESTS = 16
@@ -56,6 +58,7 @@ DOWNLOAD_HANDLERS_BASE = {
     'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
     's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
+    'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
 }
 
 DOWNLOAD_TIMEOUT = 180      # 3mins

scrapy/spider.py

@@ -6,7 +6,6 @@ See documentation in docs/topics/spiders.rst
 
 from scrapy import log
 from scrapy.http import Request
-from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 

scrapy/spidermanager.py

@@ -33,6 +33,7 @@ class SpiderManager(object):
     @classmethod
     def from_crawler(cls, crawler):
         sm = cls.from_settings(crawler.settings)
+        sm.crawler = crawler
         crawler.signals.connect(sm.close_spider, signals.spider_closed)
         return sm
 
@@ -41,7 +42,10 @@ class SpiderManager(object):
             spcls = self._spiders[spider_name]
         except KeyError:
             raise KeyError("Spider not found: %s" % spider_name)
-        return spcls(**spider_kwargs)
+        if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'):
+            return spcls.from_crawler(self.crawler, **spider_kwargs)
+        else:
+            return spcls(**spider_kwargs)
 
     def find_by_request(self, request):
         return [name for name, cls in self._spiders.iteritems()

scrapy/tests/mockserver.py

@@ -2,8 +2,29 @@ import sys, time, random, urllib
 from subprocess import Popen, PIPE
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
-from twisted.internet import reactor
-from twisted.internet.task import deferLater
+from twisted.internet import reactor, defer
+from scrapy import twisted_version
+
+
+if twisted_version < (11, 0, 0):
+    def deferLater(clock, delay, func, *args, **kw):
+        def _cancel_method():
+            _cancel_cb(None)
+            d.errback(Exception())
+
+        def _cancel_cb(result):
+            if cl.active():
+                cl.cancel()
+            return result
+
+        d = defer.Deferred()
+        d.cancel = _cancel_method
+        d.addCallback(lambda ignored: func(*args, **kw))
+        d.addBoth(_cancel_cb)
+        cl = clock.callLater(delay, d.callback, None)
+        return d
+else:
+    from twisted.internet.task import deferLater
 
 
 def getarg(request, name, default=None, type=str):
@@ -13,20 +34,39 @@ def getarg(request, name, default=None, type=str):
         return default
 
 
-class Follow(Resource):
+class LeafResource(Resource):
 
     isLeaf = True
 
+    def deferRequest(self, request, delay, f, *a, **kw):
+        def _cancelrequest(_):
+            # silence CancelledError
+            d.addErrback(lambda _: None)
+            d.cancel()
+
+        d = deferLater(reactor, delay, f, *a, **kw)
+        request.notifyFinish().addErrback(_cancelrequest)
+        return d
+
+
+class Follow(LeafResource):
+
     def render(self, request):
         total = getarg(request, "total", 100, type=int)
         show = getarg(request, "show", 1, type=int)
         order = getarg(request, "order", "desc")
+        maxlatency = getarg(request, "maxlatency", 0, type=float)
         n = getarg(request, "n", total, type=int)
         if order == "rand":
             nlist = [random.randint(1, total) for _ in range(show)]
         else:  # order == "desc"
             nlist = range(n, max(n - show, 0), -1)
 
+        lag = random.random() * maxlatency
+        self.deferRequest(request, lag, self.renderRequest, request, nlist)
+        return NOT_DONE_YET
+
+    def renderRequest(self, request, nlist):
         s = """<html> <head></head> <body>"""
         args = request.args.copy()
         for nl in nlist:
@@ -34,24 +74,11 @@ class Follow(Resource):
             argstr = urllib.urlencode(args, doseq=True)
             s += "<a href='/follow?%s'>follow %d</a><br>" % (argstr, nl)
         s += """</body>"""
-        return s
+        request.write(s)
+        request.finish()
 
 
-class DeferMixin(Resource):
-
-    def deferRequest(self, request, delay, f, *a, **kw):
-        def _cancelrequest(_):
-            # silence CancelledError
-            d.addErrback(lambda _: None)
-            d.cancel()
-        d = deferLater(reactor, delay, f, *a, **kw)
-        request.notifyFinish().addErrback(_cancelrequest)
-        return d
-
-
-class Delay(DeferMixin, Resource):
-
-    isLeaf = True
+class Delay(LeafResource):
 
     def render_GET(self, request):
         n = getarg(request, "n", 1, type=float)
@@ -67,9 +94,7 @@ class Delay(DeferMixin, Resource):
         request.finish()
 
 
-class Status(Resource):
-
-    isLeaf = True
+class Status(LeafResource):
 
     def render_GET(self, request):
         n = getarg(request, "n", 200, type=int)
@@ -77,9 +102,23 @@ class Status(Resource):
         return ""
 
 
-class Partial(DeferMixin, Resource):
+class Raw(LeafResource):
 
-    isLeaf = True
+    def render_GET(self, request):
+        request.startedWriting = 1
+        self.deferRequest(request, 0, self._delayedRender, request)
+        return NOT_DONE_YET
+    render_POST = render_GET
+
+    def _delayedRender(self, request):
+        raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n')
+        request.startedWriting = 1
+        request.write(raw)
+        request.channel.transport.loseConnection()
+        request.finish()
+
+
+class Partial(LeafResource):
 
     def render_GET(self, request):
         request.setHeader("Content-Length", "1024")
@@ -94,9 +133,16 @@ class Partial(DeferMixin, Resource):
 class Drop(Partial):
 
     def _delayedRender(self, request):
+        abort = getarg(request, "abort", 0, type=int)
         request.write("this connection will be dropped\n")
-        request.channel.transport.loseConnection()
-        request.finish()
+        tr = request.channel.transport
+        try:
+            if abort and hasattr(tr, 'abortConnection'):
+                tr.abortConnection()
+            else:
+                tr.loseConnection()
+        finally:
+            request.finish()
 
 
 class Root(Resource):
@@ -108,6 +154,7 @@ class Root(Resource):
         self.putChild("delay", Delay())
         self.putChild("partial", Partial())
         self.putChild("drop", Drop())
+        self.putChild("raw", Raw())
 
     def getChild(self, name, request):
         return self
@@ -134,6 +181,7 @@ if __name__ == "__main__":
     root = Root()
     factory = Site(root)
     port = reactor.listenTCP(8998, factory)
+
     def print_listening():
         h = port.getHost()
         print "Mock server running at http://%s:%d" % (h.host, h.port)

scrapy/tests/spiders.py

@@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking
 """
 
 import time
+from urllib import urlencode
 
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
@@ -27,11 +28,12 @@ class FollowAllSpider(MetaSpider):
     name = 'follow'
     link_extractor = SgmlLinkExtractor()
 
-    def __init__(self, total=10, show=20, order="rand", *args, **kwargs):
+    def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
         super(FollowAllSpider, self).__init__(*args, **kwargs)
         self.urls_visited = []
         self.times = []
-        url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order)
+        qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
+        url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
         self.start_urls = [url]
 
     def parse(self, response):

scrapy/tests/test_contrib_exporter.py

@@ -5,7 +5,7 @@ from scrapy.item import Item, Field
 from scrapy.utils.python import str_to_unicode
 from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \
     PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \
-    JsonItemExporter
+    JsonItemExporter, PythonItemExporter
 
 class TestItem(Item):
     name = Field()
@@ -69,7 +69,41 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3')
         self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
 
+class PythonItemExporterTest(BaseItemExporterTest):
+    def _get_exporter(self, **kwargs):
+        return PythonItemExporter(**kwargs)
 
+    def test_nested_item(self):
+        i1 = TestItem(name=u'Joseph', age='22')
+        i2 = TestItem(name=u'Maria', age=i1)
+        i3 = TestItem(name=u'Jesus', age=i2)
+        ie = self._get_exporter()
+        exported = ie.export_item(i3)
+        self.assertEqual(type(exported), dict)
+        self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'})
+        self.assertEqual(type(exported['age']), dict)
+        self.assertEqual(type(exported['age']['age']), dict)
+
+    def test_export_list(self):
+        i1 = TestItem(name=u'Joseph', age='22')
+        i2 = TestItem(name=u'Maria', age=[i1])
+        i3 = TestItem(name=u'Jesus', age=[i2])
+        ie = self._get_exporter()
+        exported = ie.export_item(i3)
+        self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
+        self.assertEqual(type(exported['age'][0]), dict)
+        self.assertEqual(type(exported['age'][0]['age'][0]), dict)
+
+    def test_export_item_dict_list(self):
+        i1 = TestItem(name=u'Joseph', age='22')
+        i2 = dict(name=u'Maria', age=[i1])
+        i3 = TestItem(name=u'Jesus', age=[i2])
+        ie = self._get_exporter()
+        exported = ie.export_item(i3)
+        self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'})
+        self.assertEqual(type(exported['age'][0]), dict)
+        self.assertEqual(type(exported['age'][0]['age'][0]), dict)
+  
 class PprintItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):
@@ -78,7 +112,6 @@ class PprintItemExporterTest(BaseItemExporterTest):
     def _check_output(self):
         self._assert_expected_item(eval(self.output.getvalue()))
 
-
 class PickleItemExporterTest(BaseItemExporterTest):
 
     def _get_exporter(self, **kwargs):

scrapy/tests/test_crawl.py

@@ -11,6 +11,7 @@ def docrawl(spider, settings=None):
     crawler.crawl(spider)
     return crawler.start()
 
+
 class CrawlTestCase(TestCase):
 
     def setUp(self):
@@ -24,16 +25,28 @@ class CrawlTestCase(TestCase):
     def test_follow_all(self):
         spider = FollowAllSpider()
         yield docrawl(spider)
-        self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url
+        self.assertEqual(len(spider.urls_visited), 11)  # 10 + start_url
 
     @defer.inlineCallbacks
     def test_delay(self):
-        spider = FollowAllSpider()
-        yield docrawl(spider, {"DOWNLOAD_DELAY": 1})
-        t = spider.times[0]
-        for t2 in spider.times[1:]:
-            self.assertTrue(t2-t > 0.45, "download delay too small: %s" % (t2-t))
-            t = t2
+        # short to long delays
+        yield self._test_delay(0.2, False)
+        yield self._test_delay(1, False)
+        # randoms
+        yield self._test_delay(0.2, True)
+        yield self._test_delay(1, True)
+
+    @defer.inlineCallbacks
+    def _test_delay(self, delay, randomize):
+        settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
+        spider = FollowAllSpider(maxlatency=delay * 2)
+        yield docrawl(spider, settings)
+        t = spider.times
+        totaltime = t[-1] - t[0]
+        avgd = totaltime / (len(t) - 1)
+        tolerance = 0.6 if randomize else 0.2
+        self.assertTrue(avgd > delay * (1 - tolerance),
+                        "download delay too small: %s" % avgd)
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
@@ -77,6 +90,47 @@ class CrawlTestCase(TestCase):
         yield docrawl(spider)
         self._assert_retried()
 
+    @defer.inlineCallbacks
+    def test_unbounded_response(self):
+        # Completeness of responses without Content-Length or Transfer-Encoding
+        # can not be determined, we treat them as valid but flagged as "partial"
+        from urllib import urlencode
+        query = urlencode({'raw': '''\
+HTTP/1.1 200 OK
+Server: Apache-Coyote/1.1
+X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0
+Set-Cookie: JSESSIONID=08515F572832D0E659FD2B0D8031D75F; Path=/
+Pragma: no-cache
+Expires: Thu, 01 Jan 1970 00:00:00 GMT
+Cache-Control: no-cache
+Cache-Control: no-store
+Content-Type: text/html;charset=UTF-8
+Content-Language: en
+Date: Tue, 27 Aug 2013 13:05:05 GMT
+Connection: close
+
+foo body
+with multiples lines
+'''})
+        spider = SimpleSpider("http://localhost:8998/raw?{0}".format(query))
+        yield docrawl(spider)
+        log = get_testlog()
+        self.assertEqual(log.count("Got response 200"), 1)
+
+    @defer.inlineCallbacks
+    def test_retry_conn_lost(self):
+        # connection lost after receiving data
+        spider = SimpleSpider("http://localhost:8998/drop?abort=0")
+        yield docrawl(spider)
+        self._assert_retried()
+
+    @defer.inlineCallbacks
+    def test_retry_conn_aborted(self):
+        # connection lost before receiving data
+        spider = SimpleSpider("http://localhost:8998/drop?abort=1")
+        yield docrawl(spider)
+        self._assert_retried()
+
     def _assert_retried(self):
         log = get_testlog()
         self.assertEqual(log.count("Retrying"), 2)

scrapy/tests/test_djangoitem/settings.py

@@ -4,3 +4,5 @@ DATABASES = {
         'NAME': ':memory:',
     }
 }
+
+SECRET_KEY = 'top-secret'

scrapy/tests/test_downloader_handlers.py

@@ -9,13 +9,19 @@ from twisted.web import server, static, util, resource
 from twisted.web.test.test_webclient import ForeverTakingResource, \
         NoLengthResource, HostHeaderResource, \
         PayloadResource, BrokenDownloadResource
+from twisted.protocols.ftp import FTPRealm, FTPFactory
+from twisted.cred import portal, checkers, credentials
+from twisted.protocols.ftp import FTPClient, ConnectionLost
 from w3lib.url import path_to_file_uri
 
+from scrapy import twisted_version
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
+from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
 from scrapy.settings import Settings
@@ -326,3 +332,86 @@ class S3TestCase(unittest.TestCase):
         httpreq = self.download_request(req, self.spider)
         self.assertEqual(httpreq.headers['Authorization'], \
                 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
+
+class FTPTestCase(unittest.TestCase):
+
+    username = "scrapy"
+    password = "passwd"
+
+    if twisted_version < (10, 2, 0):
+        skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"
+
+    def setUp(self):
+        # setup dirs and test file
+        self.directory = self.mktemp()
+        os.mkdir(self.directory)
+        userdir = os.path.join(self.directory, self.username)
+        os.mkdir(userdir)
+        FilePath(userdir).child('file.txt').setContent("I have the power!")
+
+        # setup server
+        realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
+        p = portal.Portal(realm)
+        users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
+        users_checker.addUser(self.username, self.password)
+        p.registerChecker(users_checker, credentials.IUsernamePassword)
+        self.factory = FTPFactory(portal=p)
+        self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
+        self.portNum = self.port.getHost().port
+        self.download_handler = FTPDownloadHandler(Settings())
+        self.addCleanup(self.port.stopListening)
+
+    def _add_test_callbacks(self, deferred, callback=None, errback=None):
+        def _clean(data):
+            self.download_handler.client.transport.loseConnection()
+            return data
+        deferred.addCallback(_clean)
+        if callback:
+            deferred.addCallback(callback)
+        if errback:
+            deferred.addErrback(errback)
+        return deferred
+
+    def test_ftp_download_success(self):
+        request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
+                meta={"ftp_user": self.username, "ftp_password": self.password})
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(r.status, 200)
+            self.assertEqual(r.body, 'I have the power!')
+            self.assertEqual(r.headers, {'Local Filename': [''], 'Size': [17]})
+        return self._add_test_callbacks(d, _test)
+
+    def test_ftp_download_notexist(self):
+        request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
+                meta={"ftp_user": self.username, "ftp_password": self.password})
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(r.status, 404)
+        return self._add_test_callbacks(d, _test)
+
+    def test_ftp_local_filename(self):
+        local_fname = "/tmp/file.txt"
+        request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
+                meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(r.body, local_fname)
+            self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': [17]})
+            self.assertTrue(os.path.exists(local_fname))
+            with open(local_fname) as f:
+                self.assertEqual(f.read(), "I have the power!")
+            os.remove(local_fname)
+        return self._add_test_callbacks(d, _test)
+
+    def test_invalid_credentials(self):
+        request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
+                meta={"ftp_user": self.username, "ftp_password": 'invalid'})
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(r.type, ConnectionLost)
+        return self._add_test_callbacks(d, errback=_test)

scrapy/tests/test_downloadermiddleware_cookies.py

@@ -113,3 +113,24 @@ class CookiesMiddlewareTest(TestCase):
         req4 = Request('http://scrapytest.org/', meta=res2.meta)
         assert self.mw.process_request(req4, self.spider) is None
         self.assertEquals(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce')
+
+        #cookies from hosts with port
+        req5_1 = Request('http://scrapytest.org:1104/')
+        assert self.mw.process_request(req5_1, self.spider) is None
+
+        headers = {'Set-Cookie': 'C1=value1; path=/'}
+        res5_1 = Response('http://scrapytest.org:1104/', headers=headers, request=req5_1)
+        assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1
+
+        req5_2 = Request('http://scrapytest.org:1104/some-redirected-path')
+        assert self.mw.process_request(req5_2, self.spider) is None
+        self.assertEquals(req5_2.headers.get('Cookie'), 'C1=value1')
+
+        req5_3 = Request('http://scrapytest.org/some-redirected-path')
+        assert self.mw.process_request(req5_3, self.spider) is None
+        self.assertEquals(req5_3.headers.get('Cookie'), 'C1=value1')
+
+        #skip cookie retrieval for not http request
+        req6 = Request('file:///scrapy/sometempfile')
+        assert self.mw.process_request(req6, self.spider) is None
+        self.assertEquals(req6.headers.get('Cookie'), None)

scrapy/tests/test_downloadermiddleware_retry.py

@@ -1,14 +1,16 @@
 import unittest
+from twisted.internet.error import TimeoutError as ServerTimeoutError, \
+    DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \
+    ConnectionLost
 
-from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
-                                   ConnectionRefusedError, ConnectionDone, ConnectError, \
-                                   ConnectionLost
-
+from scrapy import optional_features
 from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
+from scrapy.xlib.tx import ResponseFailed
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
 from scrapy.utils.test import get_crawler
 
+
 class RetryTest(unittest.TestCase):
     def setUp(self):
         crawler = get_crawler()
@@ -62,9 +64,15 @@ class RetryTest(unittest.TestCase):
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
     def test_twistederrors(self):
-        for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost):
+        exceptions = [ServerTimeoutError, DNSLookupError,
+                      ConnectionRefusedError, ConnectionDone, ConnectError,
+                      ConnectionLost]
+        if 'http11' in optional_features:
+            exceptions.append(ResponseFailed)
+
+        for exc in exceptions:
             req = Request('http://www.scrapytest.org/%s' % exc.__name__)
-            self._test_retry_exception(req, exc())
+            self._test_retry_exception(req, exc('foo'))
 
     def _test_retry_exception(self, req, exception):
         # first retry

scrapy/tests/test_http_request.py

@@ -1,7 +1,6 @@
 import cgi
 import unittest
 import xmlrpclib
-from cStringIO import StringIO
 from urlparse import urlparse
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse
@@ -175,6 +174,11 @@ class RequestTest(unittest.TestCase):
         r = self.request_class("http://www.example.com", method=u"POST")
         assert isinstance(r.method, str)
 
+    def test_immutable_attributes(self):
+        r = self.request_class("http://example.com")
+        self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
+        self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
+
 
 class FormRequestTest(RequestTest):
 

scrapy/tests/test_http_response.py

@@ -107,6 +107,11 @@ class BaseResponseTest(unittest.TestCase):
     def _assert_response_encoding(self, response, encoding):
         self.assertEqual(response.encoding, resolve_encoding(encoding))
 
+    def test_immutable_attributes(self):
+        r = self.response_class("http://example.com")
+        self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com')
+        self.assertRaises(AttributeError, setattr, r, 'body', 'xxx')
+
 class ResponseText(BaseResponseTest):
 
     def test_no_unicode_url(self):

scrapy/tests/test_pipeline_files.py

@@ -0,0 +1,108 @@
+import mock
+import os
+import time
+from tempfile import mkdtemp
+from shutil import rmtree
+
+from twisted.trial import unittest
+from twisted.internet import defer
+
+from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore
+from scrapy.item import Item, Field
+from scrapy.http import Request, Response
+
+
+def _mocked_download_func(request, info):
+    response = request.meta.get('response')
+    return response() if callable(response) else response
+
+
+class FilesPipelineTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = mkdtemp()
+        self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func)
+        self.pipeline.open_spider(None)
+
+    def tearDown(self):
+        rmtree(self.tempdir)
+
+    def test_file_path(self):
+        image_path = self.pipeline.file_key
+        self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"),
+                         'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
+        self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"),
+                         'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt')
+        self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"),
+                         'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc')
+        self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"),
+                         'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg')
+        self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"),
+                         'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2')
+        self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"),
+                         'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+
+    def test_fs_store(self):
+        assert isinstance(self.pipeline.store, FSFilesStore)
+        self.assertEqual(self.pipeline.store.basedir, self.tempdir)
+
+        key = 'some/image/key.jpg'
+        path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
+        self.assertEqual(self.pipeline.store._get_filesystem_path(key), path)
+
+    @defer.inlineCallbacks
+    def test_file_not_expired(self):
+        item_url = "http://example.com/file.pdf"
+        item = _create_item_with_files(item_url)
+        patchers = [
+            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True),
+            mock.patch.object(FSFilesStore, 'stat_file', return_value={
+                'checksum': 'abc', 'last_modified': time.time()}),
+            mock.patch.object(FilesPipeline, 'get_media_requests',
+                              return_value=[_prepare_request_object(item_url)])
+        ]
+        map(lambda p: p.start(), patchers)
+
+        result = yield self.pipeline.process_item(item, None)
+        self.assertEqual(result['files'][0]['checksum'], 'abc')
+
+        map(lambda p: p.stop(), patchers)
+
+    @defer.inlineCallbacks
+    def test_file_expired(self):
+        item_url = "http://example.com/file2.pdf"
+        item = _create_item_with_files(item_url)
+        patchers = [
+            mock.patch.object(FSFilesStore, 'stat_file', return_value={
+                'checksum': 'abc',
+                'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}),
+            mock.patch.object(FilesPipeline, 'get_media_requests',
+                              return_value=[_prepare_request_object(item_url)]),
+            mock.patch.object(FilesPipeline, 'inc_stats', return_value=True)
+        ]
+        map(lambda p: p.start(), patchers)
+
+        result = yield self.pipeline.process_item(item, None)
+        self.assertNotEqual(result['files'][0]['checksum'], 'abc')
+
+        map(lambda p: p.stop(), patchers)
+
+
+class ItemWithFiles(Item):
+    file_urls = Field()
+    files = Field()
+
+
+def _create_item_with_files(*files):
+    item = ItemWithFiles()
+    item['file_urls'] = files
+    return item
+
+
+def _prepare_request_object(item_url):
+    return Request(
+        item_url,
+        meta={'response': Response(item_url, status=200, body='data')})
+
+if __name__ == "__main__":
+    unittest.main()

scrapy/tests/test_pipeline_images.py

@@ -16,6 +16,7 @@ else:
     if not encoders.issubset(set(Image.core.__dict__)):
         skip = 'Missing JPEG encoders'
 
+
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
     return response() if callable(response) else response
@@ -34,7 +35,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
         rmtree(self.tempdir)
 
     def test_image_path(self):
-        image_path = self.pipeline.image_key
+        image_path = self.pipeline.file_key
         self.assertEqual(image_path("https://dev.mydeco.com/mydeco.gif"),
                          'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
         self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"),
@@ -60,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
         self.assertEqual(thumbnail_name("/tmp/some.name/foo", name),
                          'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg')
 
-    def test_fs_store(self):
-        from scrapy.contrib.pipeline.images import FSImagesStore
-        assert isinstance(self.pipeline.store, FSImagesStore)
-        self.assertEqual(self.pipeline.store.basedir, self.tempdir)
-
-        key = 'some/image/key.jpg'
-        path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg')
-        self.assertEqual(self.pipeline.store._get_filesystem_path(key), path)
-
     def test_convert_image(self):
         SIZE = (100, 100)
         # straigh forward case: RGB and JPEG
@@ -91,7 +83,6 @@ class ImagesPipelineTestCase(unittest.TestCase):
         self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))])
 
 
-
 def _create_image(format, *a, **kw):
     buf = StringIO()
     Image.new(*a, **kw).save(buf, format)

scrapy/tests/test_spidermanager/__init__.py

@@ -1,6 +1,5 @@
 import sys
 import os
-import weakref
 import shutil
 
 from zope.interface.verify import verifyObject
@@ -9,7 +8,6 @@ from twisted.trial import unittest
 
 # ugly hack to avoid cyclic imports of scrapy.spider when running this test
 # alone
-import scrapy.spider
 from scrapy.interfaces import ISpiderManager
 from scrapy.spidermanager import SpiderManager
 from scrapy.http import Request
@@ -36,7 +34,7 @@ class SpiderManagerTest(unittest.TestCase):
 
     def test_list(self):
         self.assertEqual(set(self.spiderman.list()),
-            set(['spider1', 'spider2', 'spider3']))
+            set(['spider1', 'spider2', 'spider3', 'spider4']))
 
     def test_create(self):
         spider1 = self.spiderman.create("spider1")
@@ -66,3 +64,7 @@ class SpiderManagerTest(unittest.TestCase):
     def test_load_base_spider(self):
         self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0'])
         assert len(self.spiderman._spiders) == 0
+
+    def test_load_from_crawler(self):
+        spider = self.spiderman.create('spider4', a='OK')
+        self.assertEqual(spider.a, 'OK')

scrapy/tests/test_spidermanager/test_spiders/spider4.py

@@ -0,0 +1,10 @@
+from scrapy.spider import BaseSpider
+
+class Spider4(BaseSpider):
+    name = "spider4"
+
+    @classmethod
+    def from_crawler(cls, crawler, **kwargs):
+        o = cls(**kwargs)
+        o.crawler = crawler
+        return o

scrapy/tests/test_utils_defer.py

@@ -14,7 +14,7 @@ class MustbeDeferredTest(unittest.TestCase):
             return steps
 
         dfd = mustbe_deferred(_append, 1)
-        dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred
+        dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
         steps.append(2) # add another value, that should be catched by assertEqual
         return dfd
 
@@ -27,7 +27,7 @@ class MustbeDeferredTest(unittest.TestCase):
             return dfd
 
         dfd = mustbe_deferred(_append, 1)
-        dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred
+        dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
         steps.append(2) # add another value, that should be catched by assertEqual
         return dfd
 

scrapy/tests/test_utils_misc/__init__.py

@@ -64,14 +64,14 @@ class UtilsMiscTestCase(unittest.TestCase):
         assert hasattr(arg_to_iter(None), '__iter__')
         assert hasattr(arg_to_iter(100), '__iter__')
         assert hasattr(arg_to_iter('lala'), '__iter__')
-        assert hasattr(arg_to_iter([1,2,3]), '__iter__')
+        assert hasattr(arg_to_iter([1, 2, 3]), '__iter__')
         assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
 
         self.assertEqual(list(arg_to_iter(None)), [])
         self.assertEqual(list(arg_to_iter('lala')), ['lala'])
         self.assertEqual(list(arg_to_iter(100)), [100])
         self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
-        self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3])
+        self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
         self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
         self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
 

scrapy/tests/test_utils_sitemap.py

@@ -120,12 +120,45 @@ Disallow: /s*/*tags
 Sitemap: http://example.com/sitemap.xml
 Sitemap: http://example.com/sitemap-product-index.xml
 
-# Forums 
+# Forums
 Disallow: /forum/search/
 Disallow: /forum/active/
 """
-        self.assertEqual(list(sitemap_urls_from_robots(robots)), 
+        self.assertEqual(list(sitemap_urls_from_robots(robots)),
              ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml'])
 
+    def test_sitemap_blanklines(self):
+        """Assert we can deal with starting blank lines before <xml> tag"""
+        s = Sitemap("""\
+
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+
+<!-- cache: cached = yes name = sitemap_jspCache key = sitemap -->
+<sitemap>
+<loc>http://www.example.com/sitemap1.xml</loc>
+<lastmod>2013-07-15</lastmod>
+</sitemap>
+
+<sitemap>
+<loc>http://www.example.com/sitemap2.xml</loc>
+<lastmod>2013-07-15</lastmod>
+</sitemap>
+
+<sitemap>
+<loc>http://www.example.com/sitemap3.xml</loc>
+<lastmod>2013-07-15</lastmod>
+</sitemap>
+
+<!-- end cache -->
+</sitemapindex>
+""")
+        self.assertEqual(list(s), [
+            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'},
+            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'},
+            {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'},
+        ])
+
+
 if __name__ == '__main__':
     unittest.main()

scrapy/utils/gz.py

@@ -14,7 +14,7 @@ def gunzip(data):
         try:
             chunk = f.read(8196)
             output += chunk
-        except (IOError, struct.error):
+        except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
             # some pages are quite small so output is '' and f.extrabuf

scrapy/utils/sitemap.py

@@ -4,18 +4,16 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+import lxml.etree
 
-from cStringIO import StringIO
-from xml.etree.cElementTree import ElementTree
 
 class Sitemap(object):
     """Class to parse Sitemap (type=urlset) and Sitemap Index
     (type=sitemapindex) files"""
 
     def __init__(self, xmltext):
-        tree = ElementTree()
-        tree.parse(StringIO(xmltext))
-        self._root = tree.getroot()
+        xmlp = lxml.etree.XMLParser(recover=True)
+        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
         rt = self._root.tag
         self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt
 
@@ -26,7 +24,9 @@ class Sitemap(object):
                 tag = el.tag
                 name = tag.split('}', 1)[1] if '}' in tag else tag
                 d[name] = el.text.strip() if el.text else ''
-            yield d
+            if 'loc' in d:
+                yield d
+
 
 def sitemap_urls_from_robots(robots_text):
     """Return an iterator over all sitemap urls contained in the given

scrapy/utils/url.py

@@ -79,9 +79,25 @@ def escape_ajax(url):
     Return the crawleable url according to:
     http://code.google.com/web/ajaxcrawling/docs/getting-started.html
 
-    TODO: add support for urls with query arguments
-
     >>> escape_ajax("www.example.com/ajax.html#!key=value")
     'www.example.com/ajax.html?_escaped_fragment_=key=value'
+    >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value")
+    'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key=value'
+    >>> escape_ajax("www.example.com/ajax.html?#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key=value'
+    >>> escape_ajax("www.example.com/ajax.html#!")
+    'www.example.com/ajax.html?_escaped_fragment_='
+
+    URLs that are not "AJAX crawlable" (according to Google) returned as-is:
+
+    >>> escape_ajax("www.example.com/ajax.html#key=value")
+    'www.example.com/ajax.html#key=value'
+    >>> escape_ajax("www.example.com/ajax.html#")
+    'www.example.com/ajax.html#'
+    >>> escape_ajax("www.example.com/ajax.html")
+    'www.example.com/ajax.html'
     """
-    return url.replace('#!', '?_escaped_fragment_=')
+    defrag, frag = urlparse.urldefrag(url)
+    if not frag.startswith('!'):
+        return url
+    return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])

scrapy/xlib/pydispatch/robustapply.py

@@ -6,44 +6,50 @@ and subset the given arguments to match only
 those which are acceptable.
 """
 
-def function( receiver ):
-	"""Get function-like callable object for given receiver
+import inspect
 
-	returns (function_or_method, codeObject, fromMethod)
+def function(receiver):
+    """Get function-like callable object for given receiver
 
-	If fromMethod is true, then the callable already
-	has its first argument bound
-	"""
-	if hasattr(receiver, '__call__'):
-		# receiver is a class instance; assume it is callable.
-		# Reassign receiver to the actual method that will be called.
-		if hasattr( receiver.__call__, 'im_func') or hasattr( receiver.__call__, 'im_code'):
-			receiver = receiver.__call__
-	if hasattr( receiver, 'im_func' ):
-		# an instance-method...
-		return receiver, receiver.im_func.func_code, 1
-	elif not hasattr( receiver, 'func_code'):
-		raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver)))
-	return receiver, receiver.func_code, 0
+    returns (function_or_method, codeObject, fromMethod)
+
+    If fromMethod is true, then the callable already
+    has its first argument bound
+    """
+    if inspect.isclass(receiver) and hasattr(receiver, '__call__'):
+        # receiver is a class instance; assume it is callable.
+        # Reassign receiver to the actual method that will be called.
+        if hasattr(receiver.__call__, 'im_func') or \
+           hasattr(receiver.__call__, 'im_code'):
+            receiver = receiver.__call__
+
+    if hasattr( receiver, 'im_func' ):
+        # an instance-method...
+        return receiver, receiver.im_func.func_code, 1
+    elif not hasattr(receiver, 'func_code'):
+        raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver)))
+
+    return receiver, receiver.func_code, 0
 
 def robustApply(receiver, *arguments, **named):
-	"""Call receiver with arguments and an appropriate subset of named
-	"""
-	receiver, codeObject, startIndex = function( receiver )
-	acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount]
-	for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]:
-		if named.has_key( name ):
-			raise TypeError(
-				"""Argument %r specified both positionally and as a keyword for calling %r"""% (
-					name, receiver,
-				)
-			)
-	if not (codeObject.co_flags & 8):
-		# fc does not have a **kwds type parameter, therefore 
-		# remove unacceptable arguments.
-		for arg in named.keys():
-			if arg not in acceptable:
-				del named[arg]
-	return receiver(*arguments, **named)
+    """Call receiver with arguments and an appropriate subset of named
+    """
+    receiver, codeObject, startIndex = function(receiver)
+    acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount]
+    for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]:
+        if named.has_key(name):
+            raise TypeError(
+                """Argument %r specified both positionally and as a keyword for calling %r"""% (
+                    name, receiver,
+                )
+            )
+
+    if not (codeObject.co_flags & 8):
+        # fc does not have a **kwds type parameter, therefore 
+        # remove unacceptable arguments.
+        for arg in named.keys():
+            if arg not in acceptable:
+                del named[arg]
+
+    return receiver(*arguments, **named)
 
-			

tox.ini

@@ -4,9 +4,25 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py26, py27
+envlist = py26, py27, lucid, precise
 
 [testenv]
+deps =
+    -r{toxinidir}/.travis/requirements-latest.txt
 commands =
-    pip install --use-mirrors -r .travis/requirements-latest.txt
-    trial scrapy
+    {toxinidir}/bin/runtests.sh []
+
+[testenv:lucid]
+basepython = python2.6
+deps =
+    -r{toxinidir}/.travis/requirements-lucid.txt
+
+[testenv:precise]
+basepython = python2.7
+deps =
+    -r{toxinidir}/.travis/requirements-precise.txt
+
+[testenv:windows]
+commands =
+    {toxinidir}/bin/runtests.bat []
+sitepackages = False