diff --git a/.travis.yml b/.travis.yml index 2012774a8..e00a1596b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,26 @@ language: python -python: - - 2.6 - - 2.7 env: - - BUILDENV=lucid - - BUILDENV=precise - - BUILDENV=latest + TRAVISBUG="#1027" +matrix: + exclude: + - env: TRAVISBUG="#1027" + include: + - python: "2.6" + env: BUILDENV=lucid + - python: "2.7" + env: BUILDENV=precise + - python: "2.7" + env: BUILDENV=latest + - python: "pypy" + env: BUILDENV=latest + allow_failures: + - python: "pypy" + env: BUILDENV=latest install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt + - pip install --use-mirrors . script: - trial scrapy -branches: - only: - - master - - /^[0-9].*$/ notifications: irc: channels: diff --git a/.travis/requirements-latest.txt b/.travis/requirements-latest.txt index 576f2494e..15dd8936f 100644 --- a/.travis/requirements-latest.txt +++ b/.travis/requirements-latest.txt @@ -3,4 +3,4 @@ lxml twisted boto Pillow -. +django diff --git a/.travis/requirements-lucid.txt b/.travis/requirements-lucid.txt index 20e601804..d55b57799 100644 --- a/.travis/requirements-lucid.txt +++ b/.travis/requirements-lucid.txt @@ -4,4 +4,3 @@ lxml==2.2.4 twisted==10.0.0 boto==1.9b Pillow<2.0 -. diff --git a/.travis/requirements-precise.txt b/.travis/requirements-precise.txt index 2bb99def0..625708aa2 100644 --- a/.travis/requirements-precise.txt +++ b/.travis/requirements-precise.txt @@ -4,4 +4,4 @@ lxml==2.3.2 twisted==11.1.0 boto==2.2.2 Pillow<2.0 -. +django==1.3.1 diff --git a/docs/faq.rst b/docs/faq.rst index 37ef4b8dc..88676a7ea 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -84,8 +84,8 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. -Does Scrapy crawl in breath-first or depth-first order? -------------------------------------------------------- +Does Scrapy crawl in breadth-first or depth-first order? +-------------------------------------------------------- By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which basically means that it crawls in `DFO order`_. This order is more convenient diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 1e9a62ee0..eed4bdac4 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -125,7 +125,7 @@ tag with ``id=specifications``:: .. highlight:: none -An XPath expression to select the description could be:: +An XPath expression to select the file size could be:: //div[@id='specifications']/p[2]/text()[2] diff --git a/docs/news.rst b/docs/news.rst index 3f6f67c9b..e3a7eda39 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,17 +3,122 @@ Release notes ============= -0.18 (unreleased) ------------------ +0.20 (not released yet) +----------------------- -- :ref:`benchmarking` -- moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on -- add scrapy commands using external libraries (:issue:`260`) -- added ``--pdb`` option to ``scrapy`` command line tool -- added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. -- several improvements to spider contracts +- Request/Response url/body attributes are now immutable (modifying them had + been deprecated for a long time) + +0.18.1 (released 2013-08-27) +---------------------------- + +- remove extra import added by cherry picked changes (:commit:`d20304e`) +- fix crawling tests under twisted pre 11.0.0 (:commit:`1994f38`) +- py26 can not format zero length fields {} (:commit:`abf756f`) +- test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`) +- Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`) +- do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`) +- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`) +- limit travis-ci build matrix (:commit:`3b01bb8`) +- Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`) +- Fixed so it refers to the correct folder (:commit:`3283809`) +- added quantal & raring to support ubuntu releases (:commit:`1411923`) +- fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`) +- fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`) +- minor updates to 0.18 release notes (:commit:`c45e5f1`) +- fix contributters list format (:commit:`0b60031`) + +0.18.0 (released 2013-08-09) +---------------------------- + +- Lot of improvements to testsuite run using Tox, including a way to test on pypi +- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) +- Use lxml recover option to parse sitemaps (:issue:`347`) +- Bugfix cookie merging by hostname and not by netloc (:issue:`352`) +- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`) +- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`) +- Support `dont_cache` request meta flag (:issue:`19`) +- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`) +- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`) +- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`) +- Support nested items in xml exporter (:issue:`66`) +- Improve cookies handling performance (:issue:`77`) +- Log dupe filtered requests once (:issue:`105`) +- Split redirection middleware into status and meta based middlewares (:issue:`78`) +- Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`) +- Support xpath form selection on `FormRequest.from_response` (:issue:`185`) +- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`) +- Bugfix signal dispatching on pypi interpreter (:issue:`205`) +- Improve request delay and concurrency handling (:issue:`206`) +- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`) +- Allow customization of messages logged by engine (:issue:`214`) +- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`) +- Extend Scrapy commands using setuptools entry points (:issue:`260`) +- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`) +- Support `settings.getdict` (:issue:`269`) +- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`) +- Added `Item.copy` (:issue:`290`) +- Collect idle downloader slots (:issue:`297`) +- Add `ftp://` scheme downloader handler (:issue:`329`) +- Added downloader benchmark webserver and spider tools :ref:`benchmarking` +- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on +- Add scrapy commands using external libraries (:issue:`260`) +- Added ``--pdb`` option to ``scrapy`` command line tool +- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. +- Several improvements to spider contracts - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- added from_crawler method to spiders +- added system tests with mock server +- more improvements to Mac OS compatibility (thanks Alex Cepoi) +- several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez) +- support custom download slots +- added --spider option to "shell" command. +- log overridden settings when scrapy starts + +Thanks to everyone who contribute to this release. Here is a list of +contributors sorted by number of commits:: + + 130 Pablo Hoffman + 97 Daniel Graña + 20 Nicolás Ramírez + 13 Mikhail Korobov + 12 Pedro Faustino + 11 Steven Almeroth + 5 Rolando Espinoza La fuente + 4 Michal Danilak + 4 Alex Cepoi + 4 Alexandr N Zamaraev (aka tonal) + 3 paul + 3 Martin Olveyra + 3 Jordi Llonch + 3 arijitchakraborty + 2 Shane Evans + 2 joehillen + 2 Hart + 2 Dan + 1 Zuhao Wan + 1 whodatninja + 1 vkrest + 1 tpeng + 1 Tom Mortimer-Jones + 1 Rocio Aramberri + 1 Pedro + 1 notsobad + 1 Natan L + 1 Mark Grey + 1 Luan + 1 Libor Nenadál + 1 Juan M Uys + 1 Jonas Brunsgaard + 1 Ilya Baryshev + 1 Hasnain Lakhani + 1 Emanuel Schorsch + 1 Chris Tilden + 1 Capi Etheriel + 1 cacovsky + 1 Berend Iwema + 0.16.5 (released 2013-05-30) ---------------------------- diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 64f0eb953..ac4945978 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -533,6 +533,19 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. +HttpCompressionMiddleware Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: COMPRESSION_ENABLED + +COMPRESSION_ENABLED +^^^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether the Compression middleware will be enabled. + + ChunkedTransferMiddleware ------------------------- diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 05628c479..a487299f1 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -39,11 +39,11 @@ MailSender class reference ========================== MailSender is the preferred class to use for sending emails from Scrapy, as it -uses `Twisted non-blocking IO`_, like the rest of the framework. +uses `Twisted non-blocking IO`_, like the rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) - :param smtphost: the SMTP host to use for sending the emails. If omitted, the + :param smtphost: the SMTP host to use for sending the emails. If omitted, the :setting:`MAIL_HOST` setting will be used. :type smtphost: str @@ -62,6 +62,12 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :param smtpport: the SMTP port to connect to :type smtpport: int + :param smtptls: enforce using SMTP STARTTLS + :type smtpport: boolean + + :param smtpssl: enforce using a secure SSL connection + :type smtpport: boolean + .. classmethod:: from_settings(settings) Instantiate using a Scrapy settings object, which will respect @@ -148,3 +154,21 @@ MAIL_PASS Default: ``None`` Password to use for SMTP authentication, along with :setting:`MAIL_USER`. + +.. setting:: MAIL_TLS + +MAIL_TLS +--------- + +Default: ``False`` + +Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS. + +.. setting:: MAIL_SSL + +MAIL_SSL +--------- + +Default: ``False`` + +Enforce connecting using an SSL encrypted connection diff --git a/docs/topics/images.rst b/docs/topics/images.rst index abbef96c4..9c1de4dd0 100644 --- a/docs/topics/images.rst +++ b/docs/topics/images.rst @@ -24,10 +24,13 @@ being scheduled for download, and connects those items that arrive containing the same image, to that queue. This avoids downloading the same image more than once when it's shared by several items. -The `Python Imaging Library`_ is used for thumbnailing and normalizing images -to JPEG/RGB format, so you need to install that library in order to use the -images pipeline. +`Pillow`_ is used for thumbnailing and normalizing images to JPEG/RGB format, +so you need to install this library in order to use the images pipeline. +`Python Imaging Library`_ (PIL) should also work in most cases, but it +is known to cause troubles in some setups, so we recommend to use `Pillow`_ +instead of `PIL `_. +.. _Pillow: https://github.com/python-imaging/Pillow .. _Python Imaging Library: http://www.pythonware.com/products/pil/ Using the Images Pipeline @@ -107,7 +110,7 @@ File system storage ------------------- The images are stored in files (one per image), using a `SHA1 hash`_ of their -URLs for the file names. +URLs for the file names. For example, the following image URL:: @@ -167,7 +170,7 @@ When you use this feature, the Images Pipeline will create thumbnails of the each specified size with this format:: /thumbs//.jpg - + Where: * ```` is the one specified in the :setting:`IMAGES_THUMBS` diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 091f4258d..675574e28 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -318,7 +318,7 @@ key-value fields, you can return a :class:`FormRequest` object (from your spider) like this:: return [FormRequest(url="http://www.example.com/post/action", - formdata={'name': 'John Doe', age: '27'}, + formdata={'name': 'John Doe', 'age': '27'}, callback=self.after_post)] .. _topics-request-response-ref-request-userlogin: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4e969498d..6586db668 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -61,7 +61,8 @@ Spiders receive arguments in their constructors:: class MySpider(BaseSpider): name = 'myspider' - def __init__(self, category=None): + def __init__(self, category=None, *args, **kwargs): + super(MySpider, self).__init__(*args, **kwargs) self.start_urls = ['http://www.example.com/categories/%s' % category] # ... diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 5b8476119..14498f7f4 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -23,7 +23,15 @@ with command:: lsb_release -cs Supported Ubuntu releases are: ``karmic``, ``lucid``, ``maverick``, ``natty``, -``oneiric``, ``precise``. +``oneiric``, ``precise``, ``quantal``, ``raring``. + +For Ubuntu Raring (13.04):: + + deb http://archive.scrapy.org/ubuntu raring main + +For Ubuntu Quantal (12.10):: + + deb http://archive.scrapy.org/ubuntu quantal main For Ubuntu Precise (12.04):: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..3605e94a7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +Twisted>=8.0 +w3lib>=1.2 +queuelib +lxml +pyOpenSSL diff --git a/scrapy/VERSION b/scrapy/VERSION index c5523bd09..1cf0537c3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -0.17.0 +0.19.0 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 17bf64202..e32618ebb 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,7 +7,7 @@ version_info = tuple(__version__.split('.')[:3]) import sys, os, warnings -if sys.version_info < (2,6): +if sys.version_info < (2, 6): print "Scrapy %s requires Python 2.6 or above" % __version__ sys.exit(1) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 9f6bc94e4..b781b1a64 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -34,7 +34,7 @@ class Command(ScrapyCommand): self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats)) + raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py index 25af3e051..60d87da7a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ b/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -3,12 +3,19 @@ import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.exceptions import NotConfigured class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - + + @classmethod + def from_crawler(cls, crawler): + if not crawler.settings.getbool('COMPRESSION_ENABLED'): + raise NotConfigured + return cls() + def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', 'x-gzip,gzip,deflate') diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index 9f41197c4..7eba3f78d 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -18,14 +18,16 @@ About HTTP errors to consider: indicate server overload, which would be something we want to retry """ -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError from twisted.internet.defer import TimeoutError as UserTimeoutError +from twisted.internet.error import TimeoutError as ServerTimeoutError, \ + DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost, TCPTimedOutError from scrapy import log from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message +from scrapy.xlib.tx import ResponseFailed + class RetryMiddleware(object): @@ -33,7 +35,7 @@ class RetryMiddleware(object): # decompress an empty response EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, - ConnectionLost, TCPTimedOutError, + ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) def __init__(self, settings): diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index ab0e9a94f..a20c92b54 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -3,13 +3,14 @@ Item Exporters are used to export/serialize items into different formats. """ import csv +import sys import pprint import marshal import json import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder - +from scrapy.item import BaseItem __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \ 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \ @@ -141,9 +142,23 @@ class XmlItemExporter(BaseItemExporter): for value in serialized_value: self._export_xml_field('value', value) else: - self.xg.characters(serialized_value) + self._xg_characters(serialized_value) self.xg.endElement(name) + # Workaround for http://bugs.python.org/issue17606 + # Before Python 2.7.4 xml.sax.saxutils required bytes; + # since 2.7.4 it requires unicode. The bug is likely to be + # fixed in 2.7.6, but 2.7.6 will still support unicode, + # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. + if sys.version_info[:3] >= (2, 7, 4): + def _xg_characters(self, serialized_value): + if not isinstance(serialized_value, unicode): + serialized_value = serialized_value.decode(self.encoding) + return self.xg.characters(serialized_value) + else: + def _xg_characters(self, serialized_value): + return self.xg.characters(serialized_value) + class CsvItemExporter(BaseItemExporter): @@ -183,7 +198,7 @@ class PickleItemExporter(BaseItemExporter): def __init__(self, file, protocol=2, **kwargs): self._configure(kwargs) - self.file =file + self.file = file self.protocol = protocol def export_item(self, item): @@ -200,7 +215,6 @@ class MarshalItemExporter(BaseItemExporter): def export_item(self, item): marshal.dump(dict(self._get_serialized_fields(item)), self.file) - class PprintItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): @@ -210,3 +224,30 @@ class PprintItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) self.file.write(pprint.pformat(itemdict) + '\n') + +class PythonItemExporter(BaseItemExporter): + """The idea behind this exporter is to have a mechanism to serialize items + to built-in python types so any serialization library (like + json, msgpack, binc, etc) can be used on top of it. Its main goal is to + seamless support what BaseItemExporter does plus nested items. + """ + + def serialize_field(self, field, name, value): + serializer = field.get('serializer', self._serialize_value) + return serializer(value) + + def _serialize_value(self, value): + if isinstance(value, BaseItem): + return self.export_item(value) + if isinstance(value, dict): + return dict(self._serialize_dict(value)) + if hasattr(value, '__iter__'): + return [self._serialize_value(v) for v in value] + return self._to_str_if_unicode(value) + + def _serialize_dict(self, value): + for key, val in value.iteritems(): + yield key, self._serialize_value(val) + + def export_item(self, item): + return dict(self._get_serialized_fields(item)) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py new file mode 100644 index 000000000..f03bf69df --- /dev/null +++ b/scrapy/contrib/pipeline/files.py @@ -0,0 +1,268 @@ +""" +Files Pipeline +""" + +import hashlib +import os +import os.path +import rfc822 +import time +import urlparse +from collections import defaultdict +from cStringIO import StringIO + +from twisted.internet import defer, threads + +from scrapy import log +from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.http import Request +from scrapy.utils.misc import md5sum + + +class FileException(Exception): + """General media error exception""" + + +class FSFilesStore(object): + + def __init__(self, basedir): + if '://' in basedir: + basedir = basedir.split('://', 1)[1] + self.basedir = basedir + self._mkdir(self.basedir) + self.created_directories = defaultdict(set) + + def persist_file(self, key, buf, info, meta=None, headers=None): + absolute_path = self._get_filesystem_path(key) + self._mkdir(os.path.dirname(absolute_path), info) + with open(absolute_path, 'wb') as f: + f.write(buf.getvalue()) + + def stat_file(self, key, info): + absolute_path = self._get_filesystem_path(key) + try: + last_modified = os.path.getmtime(absolute_path) + except: # FIXME: catching everything! + return {} + + with open(absolute_path, 'rb') as f: + checksum = md5sum(f) + + return {'last_modified': last_modified, 'checksum': checksum} + + def _get_filesystem_path(self, key): + path_comps = key.split('/') + return os.path.join(self.basedir, *path_comps) + + def _mkdir(self, dirname, domain=None): + seen = self.created_directories[domain] if domain else set() + if dirname not in seen: + if not os.path.exists(dirname): + os.makedirs(dirname) + seen.add(dirname) + + +class S3FilesStore(object): + + AWS_ACCESS_KEY_ID = None + AWS_SECRET_ACCESS_KEY = None + + POLICY = 'public-read' + HEADERS = { + 'Cache-Control': 'max-age=172800', + } + + def __init__(self, uri): + assert uri.startswith('s3://') + self.bucket, self.prefix = uri[5:].split('/', 1) + + def stat_file(self, key, info): + def _onsuccess(boto_key): + checksum = boto_key.etag.strip('"') + last_modified = boto_key.last_modified + modified_tuple = rfc822.parsedate_tz(last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + return {'checksum': checksum, 'last_modified': modified_stamp} + + return self._get_boto_key(key).addCallback(_onsuccess) + + def _get_boto_bucket(self): + from boto.s3.connection import S3Connection + # disable ssl (is_secure=False) because of this python bug: + # http://bugs.python.org/issue5103 + c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) + return c.get_bucket(self.bucket, validate=False) + + def _get_boto_key(self, key): + b = self._get_boto_bucket() + key_name = '%s%s' % (self.prefix, key) + return threads.deferToThread(b.get_key, key_name) + + def persist_file(self, key, buf, info, meta=None, headers=None): + """Upload file to S3 storage""" + b = self._get_boto_bucket() + key_name = '%s%s' % (self.prefix, key) + k = b.new_key(key_name) + if meta: + for metakey, metavalue in meta.iteritems(): + k.set_metadata(metakey, str(metavalue)) + h = self.HEADERS.copy() + if headers: + h.update(headers) + buf.seek(0) + return threads.deferToThread(k.set_contents_from_string, buf.getvalue(), + headers=h, policy=self.POLICY) + + +class FilesPipeline(MediaPipeline): + """Abstract pipeline that implement the file downloading + + This pipeline tries to minimize network transfers and file processing, + doing stat of the files and determining if file is new, uptodate or + expired. + + `new` files are those that pipeline never processed and needs to be + downloaded from supplier site the first time. + + `uptodate` files are the ones that the pipeline processed and are still + valid files. + + `expired` files are those that pipeline already processed but the last + modification was made long time ago, so a reprocessing is recommended to + refresh it in case of change. + + """ + + MEDIA_NAME = "file" + EXPIRES = 90 + STORE_SCHEMES = { + '': FSFilesStore, + 'file': FSFilesStore, + 's3': S3FilesStore, + } + + def __init__(self, store_uri, download_func=None): + if not store_uri: + raise NotConfigured + self.store = self._get_store(store_uri) + super(FilesPipeline, self).__init__(download_func=download_func) + + @classmethod + def from_settings(cls, settings): + s3store = cls.STORE_SCHEMES['s3'] + s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] + s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + + cls.EXPIRES = settings.getint('FILES_EXPIRES', 90) + store_uri = settings['FILES_STORE'] + return cls(store_uri) + + def _get_store(self, uri): + if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir + scheme = 'file' + else: + scheme = urlparse.urlparse(uri).scheme + store_cls = self.STORE_SCHEMES[scheme] + return store_cls(uri) + + def media_to_download(self, request, info): + def _onsuccess(result): + if not result: + return # returning None force download + + last_modified = result.get('last_modified', None) + if not last_modified: + return # returning None force download + + age_seconds = time.time() - last_modified + age_days = age_seconds / 60 / 60 / 24 + if age_days > self.EXPIRES: + return # returning None force download + + referer = request.headers.get('Referer') + log.msg(format='File (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', + level=log.DEBUG, spider=info.spider, + medianame=self.MEDIA_NAME, request=request, referer=referer) + self.inc_stats(info.spider, 'uptodate') + + checksum = result.get('checksum', None) + return {'url': request.url, 'path': key, 'checksum': checksum} + + key = self.file_key(request.url) + dfd = defer.maybeDeferred(self.store.stat_file, key, info) + dfd.addCallbacks(_onsuccess, lambda _: None) + dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_file') + return dfd + + def media_failed(self, failure, request, info): + if not isinstance(failure.value, IgnoreRequest): + referer = request.headers.get('Referer') + log.msg(format='File (unknown-error): Error downloading ' + '%(medianame)s from %(request)s referred in ' + '<%(referer)s>: %(exception)s', + level=log.WARNING, spider=info.spider, exception=failure.value, + medianame=self.MEDIA_NAME, request=request, referer=referer) + + raise FileException + + def media_downloaded(self, response, request, info): + referer = request.headers.get('Referer') + + if response.status != 200: + log.msg(format='File (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>', + level=log.WARNING, spider=info.spider, + status=response.status, request=request, referer=referer) + raise FileException('download-error') + + if not response.body: + log.msg(format='File (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content', + level=log.WARNING, spider=info.spider, + request=request, referer=referer) + raise FileException('empty-content') + + status = 'cached' if 'cached' in response.flags else 'downloaded' + log.msg(format='File (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>', + level=log.DEBUG, spider=info.spider, + status=status, request=request, referer=referer) + self.inc_stats(info.spider, status) + + try: + key = self.file_key(request.url) + checksum = self.file_downloaded(response, request, info) + except FileException as exc: + whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' + log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, + request=request, referer=referer, errormsg=str(exc)) + raise + except Exception as exc: + whyfmt = 'File (unknown-error): Error processing image from %(request)s referred in <%(referer)s>' + log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider) + raise FileException(str(exc)) + + return {'url': request.url, 'path': key, 'checksum': checksum} + + def inc_stats(self, spider, status): + spider.crawler.stats.inc_value('file_count', spider=spider) + spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + + ### Overridable Interface + def get_media_requests(self, item, info): + return [Request(x) for x in item.get('file_urls', [])] + + def file_key(self, url): + media_guid = hashlib.sha1(url).hexdigest() + media_ext = os.path.splitext(url)[1] + return 'full/%s%s' % (media_guid, media_ext) + + def file_downloaded(self, response, request, info): + key = self.file_key(request.url) + buf = StringIO(response.body) + self.store.persist_file(key, buf, info) + checksum = md5sum(buf) + return checksum + + def item_completed(self, results, item, info): + if 'files' in item.fields: + item['files'] = [x for ok, x in results if ok] + return item diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 4ccc8e282..aaad689a0 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -4,155 +4,35 @@ Images Pipeline See documentation in topics/images.rst """ -import os -import time import hashlib -import urlparse -import rfc822 from cStringIO import StringIO -from collections import defaultdict -from twisted.internet import defer, threads from PIL import Image -from scrapy import log from scrapy.utils.misc import md5sum from scrapy.http import Request -from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.exceptions import DropItem +#TODO: from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.contrib.pipeline.files import FileException, FilesPipeline class NoimagesDrop(DropItem): """Product with no images exception""" -class ImageException(Exception): +class ImageException(FileException): """General image error exception""" -class FSImagesStore(object): - - def __init__(self, basedir): - if '://' in basedir: - basedir = basedir.split('://', 1)[1] - self.basedir = basedir - self._mkdir(self.basedir) - self.created_directories = defaultdict(set) - - def persist_image(self, key, image, buf, info): - absolute_path = self._get_filesystem_path(key) - self._mkdir(os.path.dirname(absolute_path), info) - image.save(absolute_path) - - def stat_image(self, key, info): - absolute_path = self._get_filesystem_path(key) - try: - last_modified = os.path.getmtime(absolute_path) - except: # FIXME: catching everything! - return {} - - with open(absolute_path, 'rb') as imagefile: - checksum = md5sum(imagefile) - - return {'last_modified': last_modified, 'checksum': checksum} - - def _get_filesystem_path(self, key): - path_comps = key.split('/') - return os.path.join(self.basedir, *path_comps) - - def _mkdir(self, dirname, domain=None): - seen = self.created_directories[domain] if domain else set() - if dirname not in seen: - if not os.path.exists(dirname): - os.makedirs(dirname) - seen.add(dirname) - - -class S3ImagesStore(object): - - AWS_ACCESS_KEY_ID = None - AWS_SECRET_ACCESS_KEY = None - - POLICY = 'public-read' - HEADERS = { - 'Cache-Control': 'max-age=172800', - 'Content-Type': 'image/jpeg', - } - - def __init__(self, uri): - assert uri.startswith('s3://') - self.bucket, self.prefix = uri[5:].split('/', 1) - - def stat_image(self, key, info): - def _onsuccess(boto_key): - checksum = boto_key.etag.strip('"') - last_modified = boto_key.last_modified - modified_tuple = rfc822.parsedate_tz(last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - return {'checksum': checksum, 'last_modified': modified_stamp} - - return self._get_boto_key(key).addCallback(_onsuccess) - - def _get_boto_bucket(self): - from boto.s3.connection import S3Connection - # disable ssl (is_secure=False) because of this python bug: - # http://bugs.python.org/issue5103 - c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) - return c.get_bucket(self.bucket, validate=False) - - def _get_boto_key(self, key): - b = self._get_boto_bucket() - key_name = '%s%s' % (self.prefix, key) - return threads.deferToThread(b.get_key, key_name) - - def persist_image(self, key, image, buf, info): - """Upload image to S3 storage""" - width, height = image.size - b = self._get_boto_bucket() - key_name = '%s%s' % (self.prefix, key) - k = b.new_key(key_name) - k.set_metadata('width', str(width)) - k.set_metadata('height', str(height)) - buf.seek(0) - return threads.deferToThread(k.set_contents_from_file, buf, - headers=self.HEADERS, policy=self.POLICY) - - -class ImagesPipeline(MediaPipeline): - """Abstract pipeline that implement the image downloading and thumbnail generation logic - - This pipeline tries to minimize network transfers and image processing, - doing stat of the images and determining if image is new, uptodate or - expired. - - `new` images are those that pipeline never processed and needs to be - downloaded from supplier site the first time. - - `uptodate` images are the ones that the pipeline processed and are still - valid images. - - `expired` images are those that pipeline already processed but the last - modification was made long time ago, so a reprocessing is recommended to - refresh it in case of change. +class ImagesPipeline(FilesPipeline): + """Abstract pipeline that implement the image thumbnail generation logic """ MEDIA_NAME = 'image' MIN_WIDTH = 0 MIN_HEIGHT = 0 - EXPIRES = 90 THUMBS = {} - STORE_SCHEMES = { - '': FSImagesStore, - 'file': FSImagesStore, - 's3': S3ImagesStore, - } - - def __init__(self, store_uri, download_func=None): - if not store_uri: - raise NotConfigured - self.store = self._get_store(store_uri) - super(ImagesPipeline, self).__init__(download_func=download_func) @classmethod def from_settings(cls, settings): @@ -166,89 +46,11 @@ class ImagesPipeline(MediaPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri) - def _get_store(self, uri): - if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir - scheme = 'file' - else: - scheme = urlparse.urlparse(uri).scheme - store_cls = self.STORE_SCHEMES[scheme] - return store_cls(uri) + def file_key(self, url): + return self.image_key(url) - def media_downloaded(self, response, request, info): - referer = request.headers.get('Referer') - - if response.status != 200: - log.msg(format='Image (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>', - level=log.WARNING, spider=info.spider, - status=response.status, request=request, referer=referer) - raise ImageException('download-error') - - if not response.body: - log.msg(format='Image (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content', - level=log.WARNING, spider=info.spider, - request=request, referer=referer) - raise ImageException('empty-content') - - status = 'cached' if 'cached' in response.flags else 'downloaded' - log.msg(format='Image (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>', - level=log.DEBUG, spider=info.spider, - status=status, request=request, referer=referer) - self.inc_stats(info.spider, status) - - try: - key = self.image_key(request.url) - checksum = self.image_downloaded(response, request, info) - except ImageException as exc: - whyfmt = 'Image (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' - log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, - request=request, referer=referer, errormsg=str(exc)) - raise - except Exception as exc: - whyfmt = 'Image (unknown-error): Error processing image from %(request)s referred in <%(referer)s>' - log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider) - raise ImageException(str(exc)) - - return {'url': request.url, 'path': key, 'checksum': checksum} - - def media_failed(self, failure, request, info): - if not isinstance(failure.value, IgnoreRequest): - referer = request.headers.get('Referer') - log.msg(format='Image (unknown-error): Error downloading ' - '%(medianame)s from %(request)s referred in ' - '<%(referer)s>: %(exception)s', - level=log.WARNING, spider=info.spider, exception=failure.value, - medianame=self.MEDIA_NAME, request=request, referer=referer) - - raise ImageException - - def media_to_download(self, request, info): - def _onsuccess(result): - if not result: - return # returning None force download - - last_modified = result.get('last_modified', None) - if not last_modified: - return # returning None force download - - age_seconds = time.time() - last_modified - age_days = age_seconds / 60 / 60 / 24 - if age_days > self.EXPIRES: - return # returning None force download - - referer = request.headers.get('Referer') - log.msg(format='Image (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', - level=log.DEBUG, spider=info.spider, - medianame=self.MEDIA_NAME, request=request, referer=referer) - self.inc_stats(info.spider, 'uptodate') - - checksum = result.get('checksum', None) - return {'url': request.url, 'path': key, 'checksum': checksum} - - key = self.image_key(request.url) - dfd = defer.maybeDeferred(self.store.stat_image, key, info) - dfd.addCallbacks(_onsuccess, lambda _: None) - dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image') - return dfd + def file_downloaded(self, response, request, info): + return self.image_downloaded(response, request, info) def image_downloaded(self, response, request, info): checksum = None @@ -256,11 +58,15 @@ class ImagesPipeline(MediaPipeline): if checksum is None: buf.seek(0) checksum = md5sum(buf) - self.store.persist_image(key, image, buf, info) + width, height = image.size + self.store.persist_file( + key, buf, info, + meta={'width': width, 'height': height}, + headers={'Content-Type': 'image/jpeg'}) return checksum def get_images(self, response, request, info): - key = self.image_key(request.url) + key = self.file_key(request.url) orig_image = Image.open(StringIO(response.body)) width, height = orig_image.size @@ -276,10 +82,6 @@ class ImagesPipeline(MediaPipeline): thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_key, thumb_image, thumb_buf - def inc_stats(self, spider, status): - spider.crawler.stats.inc_value('image_count', spider=spider) - spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider) - def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) @@ -296,10 +98,6 @@ class ImagesPipeline(MediaPipeline): image.save(buf, 'JPEG') return image, buf - def image_key(self, url): - image_guid = hashlib.sha1(url).hexdigest() - return 'full/%s.jpg' % (image_guid) - def thumb_key(self, url, thumb_id): image_guid = hashlib.sha1(url).hexdigest() return 'thumbs/%s/%s.jpg' % (thumb_id, image_guid) @@ -307,6 +105,11 @@ class ImagesPipeline(MediaPipeline): def get_media_requests(self, item, info): return [Request(x) for x in item.get('image_urls', [])] + # backwards compatibility + def image_key(self, url): + media_guid = hashlib.sha1(url).hexdigest() + return 'full/%s.jpg' % (media_guid) + def item_completed(self, results, item, info): if 'images' in item.fields: item['images'] = [x for ok, x in results if ok] diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 9491836d2..21b09150a 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -7,6 +7,7 @@ from scrapy import log from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter + class MediaPipeline(object): LOG_FAILED_RESULTS = True @@ -65,7 +66,7 @@ class MediaPipeline(object): dfd.addCallback(self._check_media_to_download, request, info) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(log.err, spider=info.spider) - return dfd.addBoth(lambda _: wad) # it must return wad at last + return dfd.addBoth(lambda _: wad) # it must return wad at last def _check_media_to_download(self, result, request, info): if result is not None: @@ -91,11 +92,11 @@ class MediaPipeline(object): result.frames = [] result.stack = None info.downloading.remove(fp) - info.downloaded[fp] = result # cache result + info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): defer_result(result).chainDeferred(wad) - ### Overradiable Interface + ### Overridable Interface def media_to_download(self, request, info): """Check request before starting download""" pass diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index d18d51eb5..d67794d67 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -10,7 +10,6 @@ from cStringIO import StringIO from tempfile import mktemp from scrapy import log -from scrapy.http import Response from scrapy.responsetypes import responsetypes diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py new file mode 100644 index 000000000..70c526490 --- /dev/null +++ b/scrapy/core/downloader/handlers/ftp.py @@ -0,0 +1,104 @@ +""" +An asynchronous FTP file download handler for scrapy which somehow emulates an http response. + +FTP connection parameters are passed using the request meta field: +- ftp_user (required) +- ftp_password (required) +- ftp_passive (by default, enabled) sets FTP connection passive mode +- ftp_local_filename + - If not given, file data will come in the response.body, as a normal scrapy Response, + which will imply that the entire file will be on memory. + - if given, file data will be saved in a local file with the given name + This helps when downloading very big files to avoid memory issues. In addition, for + convenience the local file name will also be given in the response body. + +The status of the built html response will be, by default +- 200 in case of success +- 404 in case specified file was not found in the server (ftp code 550) + +or raise corresponding ftp exception otherwise + +The matching from server ftp command return codes to html response codes is defined in the +CODE_MAPPING attribute of the handler class. The key 'default' is used for any code +that is not explicitly present among the map keys. You may need to overwrite this +mapping if want a different behaviour than default. + +In case of status 200 request, response.headers will come with two keys: + 'Local Filename' - with the value of the local filename if given + 'Size' - with size of the downloaded data +""" + +import re +from urlparse import urlparse +from cStringIO import StringIO + +from twisted.internet import reactor +from twisted.protocols.ftp import FTPClient, CommandFailed +from twisted.internet.protocol import Protocol, ClientCreator + +from scrapy.http import Response +from scrapy.responsetypes import responsetypes + +class ReceivedDataProtocol(Protocol): + def __init__(self, filename=None): + self.__filename = filename + self.body = open(filename, "w") if filename else StringIO() + self.size = 0 + + def dataReceived(self, data): + self.body.write(data) + self.size += len(data) + + @property + def filename(self): + return self.__filename + + def close(self): + self.body.close() if self.filename else self.body.reset() + +_CODE_RE = re.compile("\d+") +class FTPDownloadHandler(object): + + CODE_MAPPING = { + "550": 404, + "default": 503, + } + + def __init__(self, setting): + pass + + def download_request(self, request, spider): + parsed_url = urlparse(request.url) + creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"], + request.meta["ftp_password"], + passive=request.meta.get("ftp_passive", 1)) + return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, + request, parsed_url.path) + + def gotClient(self, client, request, filepath): + self.client = client + protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) + return client.retrieveFile(filepath, protocol)\ + .addCallbacks(callback=self._build_response, + callbackArgs=(request, protocol), + errback=self._failed, + errbackArgs=(request,)) + + def _build_response(self, result, request, protocol): + self.result = result + respcls = responsetypes.from_args(url=request.url) + protocol.close() + body = protocol.filename or protocol.body.read() + headers = {"local filename": protocol.filename or '', "size": protocol.size} + return respcls(url=request.url, status=200, body=body, headers=headers) + + def _failed(self, result, request): + message = result.getErrorMessage() + if result.type == CommandFailed: + m = _CODE_RE.search(message) + if m: + ftpcode = m.group() + httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) + return Response(url=request.url, status=httpcode, body=message) + raise result.type(result.value) + diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 65ac8a5ae..04dd4952c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,24 +7,24 @@ from urlparse import urldefrag from zope.interface import implements from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer from twisted.internet.error import TimeoutError +from twisted.web.http import PotentialDataLoss from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint + HTTPConnectionPool, TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object -from scrapy import log - class HTTP11DownloadHandler(object): def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') + self._pool._factory.noisy = False self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) self._contextFactory = self._contextFactoryClass() @@ -54,7 +54,7 @@ class ScrapyAgent(object): if proxy: scheme, _, host, port, _ = _parse(proxy) endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout, - bindAddress=bindaddress) + bindAddress=bindaddress) return self._ProxyAgent(endpoint) return self._Agent(reactor, contextFactory=self._contextFactory, @@ -144,10 +144,11 @@ class _ResponseReader(protocol.Protocol): def connectionLost(self, reason): if self._finished.called: return + body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) - elif reason.check(PotentialDataLoss, ResponseFailed): + elif reason.check(PotentialDataLoss): self._finished.callback((self._txresponse, body, ['partial'])) else: self._finished.errback(reason) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ea69dc5ca..135d8f293 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,11 +1,9 @@ from time import time from urlparse import urlparse, urlunparse, urldefrag -from twisted.internet.ssl import ClientContextFactory from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient from twisted.internet import defer -from OpenSSL import SSL from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached @@ -60,12 +58,15 @@ class ScrapyHTTPPageGetter(HTTPClient): self.factory.gotHeaders(self.headers) def connectionLost(self, reason): + self._connection_lost_reason = reason HTTPClient.connectionLost(self, reason) self.factory.noPage(reason) def handleResponse(self, response): if self.factory.method.upper() == 'HEAD': self.factory.page('') + elif self.length is not None and self.length > 0: + self.factory.noPage(self._connection_lost_reason) else: self.factory.page(response) self.transport.loseConnection() diff --git a/scrapy/http/common.py b/scrapy/http/common.py index 34d5389bf..ba6ab277c 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,10 +1,6 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning - -def deprecated_setter(setter, attrname): +def obsolete_setter(setter, attrname): def newsetter(self, value): c = self.__class__.__name__ - warnings.warn("Don't modify %s.%s attribute, use %s.replace() instead" % \ - (c, attrname, c), ScrapyDeprecationWarning, stacklevel=2) - return setter(self, value) + msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) + raise AttributeError(msg) return newsetter diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8ad8de290..cc96cf8ac 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -22,7 +22,10 @@ class CookieJar(object): # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain - req_host = urlparse_cached(request).netloc + req_host = urlparse_cached(request).hostname + if not req_host: + return + if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) if req_host.find(".") == -1: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1b14e2068..609a0d433 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -13,7 +13,7 @@ from scrapy.http.headers import Headers from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated from scrapy.utils.url import escape_ajax -from scrapy.http.common import deprecated_setter +from scrapy.http.common import obsolete_setter class Request(object_ref): @@ -60,7 +60,7 @@ class Request(object_ref): if ':' not in self._url: raise ValueError('Missing scheme in request url: %s' % self._url) - url = property(_get_url, deprecated_setter(_set_url, 'url')) + url = property(_get_url, obsolete_setter(_set_url, 'url')) def _get_body(self): return self._body @@ -78,7 +78,7 @@ class Request(object_ref): else: raise TypeError("Request body must either str or unicode. Got: '%s'" % type(body).__name__) - body = property(_get_body, deprecated_setter(_set_body, 'body')) + body = property(_get_body, obsolete_setter(_set_body, 'body')) @property def encoding(self): diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 46be1e362..7ff683eb6 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -9,7 +9,7 @@ import copy from scrapy.http.headers import Headers from scrapy.utils.trackref import object_ref -from scrapy.http.common import deprecated_setter +from scrapy.http.common import obsolete_setter class Response(object_ref): @@ -39,7 +39,7 @@ class Response(object_ref): raise TypeError('%s url must be str, got %s:' % (type(self).__name__, \ type(url).__name__)) - url = property(_get_url, deprecated_setter(_set_url, 'url')) + url = property(_get_url, obsolete_setter(_set_url, 'url')) def _get_body(self): return self._body @@ -56,7 +56,7 @@ class Response(object_ref): raise TypeError("Response body must either str or unicode. Got: '%s'" \ % type(body).__name__) - body = property(_get_body, deprecated_setter(_set_body, 'body')) + body = property(_get_body, obsolete_setter(_set_body, 'body')) def __str__(self): return "<%d %s>" % (self.status, self.url) diff --git a/scrapy/mail.py b/scrapy/mail.py index 8af6efdc4..bedf06895 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -11,7 +11,7 @@ from email.MIMEText import MIMEText from email.Utils import COMMASPACE, formatdate from email import Encoders -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, ssl from twisted.mail.smtp import ESMTPSenderFactory from scrapy import log @@ -19,18 +19,21 @@ from scrapy import log class MailSender(object): def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', - smtpuser=None, smtppass=None, smtpport=25, debug=False): + smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport self.smtpuser = smtpuser self.smtppass = smtppass + self.smtptls = smtptls + self.smtpssl = smtpssl self.mailfrom = mailfrom self.debug = debug @classmethod def from_settings(cls, settings): return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'], - settings['MAIL_PASS'], settings.getint('MAIL_PORT')) + settings['MAIL_PASS'], settings.getint('MAIL_PORT'), + settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL')) def send(self, to, subject, body, cc=None, attachs=(), _callback=None): if attachs: @@ -91,7 +94,12 @@ class MailSender(object): d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ - requireTransportSecurity=False) + requireTransportSecurity=self.smtptls) factory.noisy = False - reactor.connectTCP(self.smtphost, self.smtpport, factory) + + if self.smtpssl: + reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()) + else: + reactor.connectTCP(self.smtphost, self.smtpport, factory) + return d diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6ba89ac94..7d9811727 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,5 +1,3 @@ -import socket - from twisted.internet import defer from twisted.internet.base import ThreadedResolver diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 94fac60d6..a16110b52 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -26,6 +26,8 @@ CLOSESPIDER_ERRORCOUNT = 0 COMMANDS_MODULE = '' +COMPRESSION_ENABLED = True + CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 @@ -56,6 +58,7 @@ DOWNLOAD_HANDLERS_BASE = { 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', + 'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler', } DOWNLOAD_TIMEOUT = 180 # 3mins diff --git a/scrapy/spider.py b/scrapy/spider.py index 36e91d636..25a59036d 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -6,7 +6,6 @@ See documentation in docs/topics/spiders.rst from scrapy import log from scrapy.http import Request -from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 6443c45fa..9ab570964 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -33,6 +33,7 @@ class SpiderManager(object): @classmethod def from_crawler(cls, crawler): sm = cls.from_settings(crawler.settings) + sm.crawler = crawler crawler.signals.connect(sm.close_spider, signals.spider_closed) return sm @@ -41,7 +42,10 @@ class SpiderManager(object): spcls = self._spiders[spider_name] except KeyError: raise KeyError("Spider not found: %s" % spider_name) - return spcls(**spider_kwargs) + if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'): + return spcls.from_crawler(self.crawler, **spider_kwargs) + else: + return spcls(**spider_kwargs) def find_by_request(self, request): return [name for name, cls in self._spiders.iteritems() diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 8b1cc09e8..d165a6bdc 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -2,8 +2,29 @@ import sys, time, random, urllib from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource -from twisted.internet import reactor -from twisted.internet.task import deferLater +from twisted.internet import reactor, defer +from scrapy import twisted_version + + +if twisted_version < (11, 0, 0): + def deferLater(clock, delay, func, *args, **kw): + def _cancel_method(): + _cancel_cb(None) + d.errback(Exception()) + + def _cancel_cb(result): + if cl.active(): + cl.cancel() + return result + + d = defer.Deferred() + d.cancel = _cancel_method + d.addCallback(lambda ignored: func(*args, **kw)) + d.addBoth(_cancel_cb) + cl = clock.callLater(delay, d.callback, None) + return d +else: + from twisted.internet.task import deferLater def getarg(request, name, default=None, type=str): @@ -13,20 +34,39 @@ def getarg(request, name, default=None, type=str): return default -class Follow(Resource): +class LeafResource(Resource): isLeaf = True + def deferRequest(self, request, delay, f, *a, **kw): + def _cancelrequest(_): + # silence CancelledError + d.addErrback(lambda _: None) + d.cancel() + + d = deferLater(reactor, delay, f, *a, **kw) + request.notifyFinish().addErrback(_cancelrequest) + return d + + +class Follow(LeafResource): + def render(self, request): total = getarg(request, "total", 100, type=int) show = getarg(request, "show", 1, type=int) order = getarg(request, "order", "desc") + maxlatency = getarg(request, "maxlatency", 0, type=float) n = getarg(request, "n", total, type=int) if order == "rand": nlist = [random.randint(1, total) for _ in range(show)] else: # order == "desc" nlist = range(n, max(n - show, 0), -1) + lag = random.random() * maxlatency + self.deferRequest(request, lag, self.renderRequest, request, nlist) + return NOT_DONE_YET + + def renderRequest(self, request, nlist): s = """ """ args = request.args.copy() for nl in nlist: @@ -34,24 +74,11 @@ class Follow(Resource): argstr = urllib.urlencode(args, doseq=True) s += "follow %d
" % (argstr, nl) s += """""" - return s + request.write(s) + request.finish() -class DeferMixin(Resource): - - def deferRequest(self, request, delay, f, *a, **kw): - def _cancelrequest(_): - # silence CancelledError - d.addErrback(lambda _: None) - d.cancel() - d = deferLater(reactor, delay, f, *a, **kw) - request.notifyFinish().addErrback(_cancelrequest) - return d - - -class Delay(DeferMixin, Resource): - - isLeaf = True +class Delay(LeafResource): def render_GET(self, request): n = getarg(request, "n", 1, type=float) @@ -67,9 +94,7 @@ class Delay(DeferMixin, Resource): request.finish() -class Status(Resource): - - isLeaf = True +class Status(LeafResource): def render_GET(self, request): n = getarg(request, "n", 200, type=int) @@ -77,9 +102,23 @@ class Status(Resource): return "" -class Partial(DeferMixin, Resource): +class Raw(LeafResource): - isLeaf = True + def render_GET(self, request): + request.startedWriting = 1 + self.deferRequest(request, 0, self._delayedRender, request) + return NOT_DONE_YET + render_POST = render_GET + + def _delayedRender(self, request): + raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n') + request.startedWriting = 1 + request.write(raw) + request.channel.transport.loseConnection() + request.finish() + + +class Partial(LeafResource): def render_GET(self, request): request.setHeader("Content-Length", "1024") @@ -94,9 +133,16 @@ class Partial(DeferMixin, Resource): class Drop(Partial): def _delayedRender(self, request): + abort = getarg(request, "abort", 0, type=int) request.write("this connection will be dropped\n") - request.channel.transport.loseConnection() - request.finish() + tr = request.channel.transport + try: + if abort and hasattr(tr, 'abortConnection'): + tr.abortConnection() + else: + tr.loseConnection() + finally: + request.finish() class Root(Resource): @@ -108,6 +154,7 @@ class Root(Resource): self.putChild("delay", Delay()) self.putChild("partial", Partial()) self.putChild("drop", Drop()) + self.putChild("raw", Raw()) def getChild(self, name, request): return self @@ -134,6 +181,7 @@ if __name__ == "__main__": root = Root() factory = Site(root) port = reactor.listenTCP(8998, factory) + def print_listening(): h = port.getHost() print "Mock server running at http://%s:%d" % (h.host, h.port) diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py index e1d06d85c..2d8dbd808 100644 --- a/scrapy/tests/spiders.py +++ b/scrapy/tests/spiders.py @@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking """ import time +from urllib import urlencode from scrapy.spider import BaseSpider from scrapy.http import Request @@ -27,11 +28,12 @@ class FollowAllSpider(MetaSpider): name = 'follow' link_extractor = SgmlLinkExtractor() - def __init__(self, total=10, show=20, order="rand", *args, **kwargs): + def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): super(FollowAllSpider, self).__init__(*args, **kwargs) self.urls_visited = [] self.times = [] - url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order) + qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} + url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) self.start_urls = [url] def parse(self, response): diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index d4865aafa..27f40551d 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -5,7 +5,7 @@ from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \ - JsonItemExporter + JsonItemExporter, PythonItemExporter class TestItem(Item): name = Field() @@ -69,7 +69,41 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') +class PythonItemExporterTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + return PythonItemExporter(**kwargs) + def test_nested_item(self): + i1 = TestItem(name=u'Joseph', age='22') + i2 = TestItem(name=u'Maria', age=i1) + i3 = TestItem(name=u'Jesus', age=i2) + ie = self._get_exporter() + exported = ie.export_item(i3) + self.assertEqual(type(exported), dict) + self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}) + self.assertEqual(type(exported['age']), dict) + self.assertEqual(type(exported['age']['age']), dict) + + def test_export_list(self): + i1 = TestItem(name=u'Joseph', age='22') + i2 = TestItem(name=u'Maria', age=[i1]) + i3 = TestItem(name=u'Jesus', age=[i2]) + ie = self._get_exporter() + exported = ie.export_item(i3) + self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual(type(exported['age'][0]), dict) + self.assertEqual(type(exported['age'][0]['age'][0]), dict) + + def test_export_item_dict_list(self): + i1 = TestItem(name=u'Joseph', age='22') + i2 = dict(name=u'Maria', age=[i1]) + i3 = TestItem(name=u'Jesus', age=[i2]) + ie = self._get_exporter() + exported = ie.export_item(i3) + self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual(type(exported['age'][0]), dict) + self.assertEqual(type(exported['age'][0]['age'][0]), dict) + class PprintItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -78,7 +112,6 @@ class PprintItemExporterTest(BaseItemExporterTest): def _check_output(self): self._assert_expected_item(eval(self.output.getvalue())) - class PickleItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 256f2d9b1..1878f7a1d 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -11,6 +11,7 @@ def docrawl(spider, settings=None): crawler.crawl(spider) return crawler.start() + class CrawlTestCase(TestCase): def setUp(self): @@ -24,16 +25,28 @@ class CrawlTestCase(TestCase): def test_follow_all(self): spider = FollowAllSpider() yield docrawl(spider) - self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url + self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks def test_delay(self): - spider = FollowAllSpider() - yield docrawl(spider, {"DOWNLOAD_DELAY": 1}) - t = spider.times[0] - for t2 in spider.times[1:]: - self.assertTrue(t2-t > 0.45, "download delay too small: %s" % (t2-t)) - t = t2 + # short to long delays + yield self._test_delay(0.2, False) + yield self._test_delay(1, False) + # randoms + yield self._test_delay(0.2, True) + yield self._test_delay(1, True) + + @defer.inlineCallbacks + def _test_delay(self, delay, randomize): + settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} + spider = FollowAllSpider(maxlatency=delay * 2) + yield docrawl(spider, settings) + t = spider.times + totaltime = t[-1] - t[0] + avgd = totaltime / (len(t) - 1) + tolerance = 0.6 if randomize else 0.2 + self.assertTrue(avgd > delay * (1 - tolerance), + "download delay too small: %s" % avgd) @defer.inlineCallbacks def test_timeout_success(self): @@ -77,6 +90,47 @@ class CrawlTestCase(TestCase): yield docrawl(spider) self._assert_retried() + @defer.inlineCallbacks + def test_unbounded_response(self): + # Completeness of responses without Content-Length or Transfer-Encoding + # can not be determined, we treat them as valid but flagged as "partial" + from urllib import urlencode + query = urlencode({'raw': '''\ +HTTP/1.1 200 OK +Server: Apache-Coyote/1.1 +X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0 +Set-Cookie: JSESSIONID=08515F572832D0E659FD2B0D8031D75F; Path=/ +Pragma: no-cache +Expires: Thu, 01 Jan 1970 00:00:00 GMT +Cache-Control: no-cache +Cache-Control: no-store +Content-Type: text/html;charset=UTF-8 +Content-Language: en +Date: Tue, 27 Aug 2013 13:05:05 GMT +Connection: close + +foo body +with multiples lines +'''}) + spider = SimpleSpider("http://localhost:8998/raw?{0}".format(query)) + yield docrawl(spider) + log = get_testlog() + self.assertEqual(log.count("Got response 200"), 1) + + @defer.inlineCallbacks + def test_retry_conn_lost(self): + # connection lost after receiving data + spider = SimpleSpider("http://localhost:8998/drop?abort=0") + yield docrawl(spider) + self._assert_retried() + + @defer.inlineCallbacks + def test_retry_conn_aborted(self): + # connection lost before receiving data + spider = SimpleSpider("http://localhost:8998/drop?abort=1") + yield docrawl(spider) + self._assert_retried() + def _assert_retried(self): log = get_testlog() self.assertEqual(log.count("Retrying"), 2) diff --git a/scrapy/tests/test_djangoitem/settings.py b/scrapy/tests/test_djangoitem/settings.py index e3300db9d..1bee92477 100644 --- a/scrapy/tests/test_djangoitem/settings.py +++ b/scrapy/tests/test_djangoitem/settings.py @@ -4,3 +4,5 @@ DATABASES = { 'NAME': ':memory:', } } + +SECRET_KEY = 'top-secret' diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 18be4ed35..c71bfbcb8 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -9,13 +9,19 @@ from twisted.web import server, static, util, resource from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource +from twisted.protocols.ftp import FTPRealm, FTPFactory +from twisted.cred import portal, checkers, credentials +from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri +from scrapy import twisted_version from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler +from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler + from scrapy.spider import BaseSpider from scrapy.http import Request from scrapy.settings import Settings @@ -326,3 +332,86 @@ class S3TestCase(unittest.TestCase): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') + +class FTPTestCase(unittest.TestCase): + + username = "scrapy" + password = "passwd" + + if twisted_version < (10, 2, 0): + skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" + + def setUp(self): + # setup dirs and test file + self.directory = self.mktemp() + os.mkdir(self.directory) + userdir = os.path.join(self.directory, self.username) + os.mkdir(userdir) + FilePath(userdir).child('file.txt').setContent("I have the power!") + + # setup server + realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) + p = portal.Portal(realm) + users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() + users_checker.addUser(self.username, self.password) + p.registerChecker(users_checker, credentials.IUsernamePassword) + self.factory = FTPFactory(portal=p) + self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") + self.portNum = self.port.getHost().port + self.download_handler = FTPDownloadHandler(Settings()) + self.addCleanup(self.port.stopListening) + + def _add_test_callbacks(self, deferred, callback=None, errback=None): + def _clean(data): + self.download_handler.client.transport.loseConnection() + return data + deferred.addCallback(_clean) + if callback: + deferred.addCallback(callback) + if errback: + deferred.addErrback(errback) + return deferred + + def test_ftp_download_success(self): + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": self.password}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.status, 200) + self.assertEqual(r.body, 'I have the power!') + self.assertEqual(r.headers, {'Local Filename': [''], 'Size': [17]}) + return self._add_test_callbacks(d, _test) + + def test_ftp_download_notexist(self): + request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": self.password}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.status, 404) + return self._add_test_callbacks(d, _test) + + def test_ftp_local_filename(self): + local_fname = "/tmp/file.txt" + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.body, local_fname) + self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': [17]}) + self.assertTrue(os.path.exists(local_fname)) + with open(local_fname) as f: + self.assertEqual(f.read(), "I have the power!") + os.remove(local_fname) + return self._add_test_callbacks(d, _test) + + def test_invalid_credentials(self): + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": 'invalid'}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.type, ConnectionLost) + return self._add_test_callbacks(d, errback=_test) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d9ed476f1..5f5e7a3d5 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -113,3 +113,24 @@ class CookiesMiddlewareTest(TestCase): req4 = Request('http://scrapytest.org/', meta=res2.meta) assert self.mw.process_request(req4, self.spider) is None self.assertEquals(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce') + + #cookies from hosts with port + req5_1 = Request('http://scrapytest.org:1104/') + assert self.mw.process_request(req5_1, self.spider) is None + + headers = {'Set-Cookie': 'C1=value1; path=/'} + res5_1 = Response('http://scrapytest.org:1104/', headers=headers, request=req5_1) + assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1 + + req5_2 = Request('http://scrapytest.org:1104/some-redirected-path') + assert self.mw.process_request(req5_2, self.spider) is None + self.assertEquals(req5_2.headers.get('Cookie'), 'C1=value1') + + req5_3 = Request('http://scrapytest.org/some-redirected-path') + assert self.mw.process_request(req5_3, self.spider) is None + self.assertEquals(req5_3.headers.get('Cookie'), 'C1=value1') + + #skip cookie retrieval for not http request + req6 = Request('file:///scrapy/sometempfile') + assert self.mw.process_request(req6, self.spider) is None + self.assertEquals(req6.headers.get('Cookie'), None) diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index d9bf1a543..c5732dc4d 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -1,14 +1,16 @@ import unittest +from twisted.internet.error import TimeoutError as ServerTimeoutError, \ + DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost - +from scrapy import optional_features from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware +from scrapy.xlib.tx import ResponseFailed from scrapy.spider import BaseSpider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler + class RetryTest(unittest.TestCase): def setUp(self): crawler = get_crawler() @@ -62,9 +64,15 @@ class RetryTest(unittest.TestCase): assert self.mw.process_response(req, rsp, self.spider) is rsp def test_twistederrors(self): - for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost): + exceptions = [ServerTimeoutError, DNSLookupError, + ConnectionRefusedError, ConnectionDone, ConnectError, + ConnectionLost] + if 'http11' in optional_features: + exceptions.append(ResponseFailed) + + for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) - self._test_retry_exception(req, exc()) + self._test_retry_exception(req, exc('foo')) def _test_retry_exception(self, req, exception): # first retry diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 9d2f235cc..29358399e 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,7 +1,6 @@ import cgi import unittest import xmlrpclib -from cStringIO import StringIO from urlparse import urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse @@ -175,6 +174,11 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com", method=u"POST") assert isinstance(r.method, str) + def test_immutable_attributes(self): + r = self.request_class("http://example.com") + self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') + self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + class FormRequestTest(RequestTest): diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 26c66453a..0809340a1 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -107,6 +107,11 @@ class BaseResponseTest(unittest.TestCase): def _assert_response_encoding(self, response, encoding): self.assertEqual(response.encoding, resolve_encoding(encoding)) + def test_immutable_attributes(self): + r = self.response_class("http://example.com") + self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') + self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): diff --git a/scrapy/tests/test_pipeline_files.py b/scrapy/tests/test_pipeline_files.py new file mode 100644 index 000000000..f0aadfbfa --- /dev/null +++ b/scrapy/tests/test_pipeline_files.py @@ -0,0 +1,108 @@ +import mock +import os +import time +from tempfile import mkdtemp +from shutil import rmtree + +from twisted.trial import unittest +from twisted.internet import defer + +from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore +from scrapy.item import Item, Field +from scrapy.http import Request, Response + + +def _mocked_download_func(request, info): + response = request.meta.get('response') + return response() if callable(response) else response + + +class FilesPipelineTestCase(unittest.TestCase): + + def setUp(self): + self.tempdir = mkdtemp() + self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline.open_spider(None) + + def tearDown(self): + rmtree(self.tempdir) + + def test_file_path(self): + image_path = self.pipeline.file_key + self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + + def test_fs_store(self): + assert isinstance(self.pipeline.store, FSFilesStore) + self.assertEqual(self.pipeline.store.basedir, self.tempdir) + + key = 'some/image/key.jpg' + path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') + self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) + + @defer.inlineCallbacks + def test_file_not_expired(self): + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), + mock.patch.object(FSFilesStore, 'stat_file', return_value={ + 'checksum': 'abc', 'last_modified': time.time()}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + @defer.inlineCallbacks + def test_file_expired(self): + item_url = "http://example.com/file2.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FSFilesStore, 'stat_file', return_value={ + 'checksum': 'abc', + 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]), + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertNotEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + +class ItemWithFiles(Item): + file_urls = Field() + files = Field() + + +def _create_item_with_files(*files): + item = ItemWithFiles() + item['file_urls'] = files + return item + + +def _prepare_request_object(item_url): + return Request( + item_url, + meta={'response': Response(item_url, status=200, body='data')}) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 03871383e..692e91afa 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -16,6 +16,7 @@ else: if not encoders.issubset(set(Image.core.__dict__)): skip = 'Missing JPEG encoders' + def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -34,7 +35,7 @@ class ImagesPipelineTestCase(unittest.TestCase): rmtree(self.tempdir) def test_image_path(self): - image_path = self.pipeline.image_key + image_path = self.pipeline.file_key self.assertEqual(image_path("https://dev.mydeco.com/mydeco.gif"), 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"), @@ -60,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(thumbnail_name("/tmp/some.name/foo", name), 'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg') - def test_fs_store(self): - from scrapy.contrib.pipeline.images import FSImagesStore - assert isinstance(self.pipeline.store, FSImagesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) - - key = 'some/image/key.jpg' - path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') - self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) - def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG @@ -91,7 +83,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) - def _create_image(format, *a, **kw): buf = StringIO() Image.new(*a, **kw).save(buf, format) diff --git a/scrapy/tests/test_spidermanager/__init__.py b/scrapy/tests/test_spidermanager/__init__.py index 4b46022e6..c8697d3b5 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/scrapy/tests/test_spidermanager/__init__.py @@ -1,6 +1,5 @@ import sys import os -import weakref import shutil from zope.interface.verify import verifyObject @@ -9,7 +8,6 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spider when running this test # alone -import scrapy.spider from scrapy.interfaces import ISpiderManager from scrapy.spidermanager import SpiderManager from scrapy.http import Request @@ -36,7 +34,7 @@ class SpiderManagerTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spiderman.list()), - set(['spider1', 'spider2', 'spider3'])) + set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_create(self): spider1 = self.spiderman.create("spider1") @@ -66,3 +64,7 @@ class SpiderManagerTest(unittest.TestCase): def test_load_base_spider(self): self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0']) assert len(self.spiderman._spiders) == 0 + + def test_load_from_crawler(self): + spider = self.spiderman.create('spider4', a='OK') + self.assertEqual(spider.a, 'OK') diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider4.py b/scrapy/tests/test_spidermanager/test_spiders/spider4.py new file mode 100644 index 000000000..6f66ad37e --- /dev/null +++ b/scrapy/tests/test_spidermanager/test_spiders/spider4.py @@ -0,0 +1,10 @@ +from scrapy.spider import BaseSpider + +class Spider4(BaseSpider): + name = "spider4" + + @classmethod + def from_crawler(cls, crawler, **kwargs): + o = cls(**kwargs) + o.crawler = crawler + return o diff --git a/scrapy/tests/test_utils_defer.py b/scrapy/tests/test_utils_defer.py index 4473002dc..670116b34 100644 --- a/scrapy/tests/test_utils_defer.py +++ b/scrapy/tests/test_utils_defer.py @@ -14,7 +14,7 @@ class MustbeDeferredTest(unittest.TestCase): return steps dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred steps.append(2) # add another value, that should be catched by assertEqual return dfd @@ -27,7 +27,7 @@ class MustbeDeferredTest(unittest.TestCase): return dfd dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred steps.append(2) # add another value, that should be catched by assertEqual return dfd diff --git a/scrapy/tests/test_utils_misc/__init__.py b/scrapy/tests/test_utils_misc/__init__.py index 916b5e25b..143c9b644 100644 --- a/scrapy/tests/test_utils_misc/__init__.py +++ b/scrapy/tests/test_utils_misc/__init__.py @@ -64,14 +64,14 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(None), '__iter__') assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') - assert hasattr(arg_to_iter([1,2,3]), '__iter__') + assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) - self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) + self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py index 99b66feec..8be94a309 100644 --- a/scrapy/tests/test_utils_sitemap.py +++ b/scrapy/tests/test_utils_sitemap.py @@ -120,12 +120,45 @@ Disallow: /s*/*tags Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml -# Forums +# Forums Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots)), + self.assertEqual(list(sitemap_urls_from_robots(robots)), ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + def test_sitemap_blanklines(self): + """Assert we can deal with starting blank lines before tag""" + s = Sitemap("""\ + + + + + + +http://www.example.com/sitemap1.xml +2013-07-15 + + + +http://www.example.com/sitemap2.xml +2013-07-15 + + + +http://www.example.com/sitemap3.xml +2013-07-15 + + + + +""") + self.assertEqual(list(s), [ + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'}, + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'}, + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'}, + ]) + + if __name__ == '__main__': unittest.main() diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fe82ed39..aa8ffc6fc 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -14,7 +14,7 @@ def gunzip(data): try: chunk = f.read(8196) output += chunk - except (IOError, struct.error): + except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 71d8122ab..38e38d6a9 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,18 +4,16 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +import lxml.etree -from cStringIO import StringIO -from xml.etree.cElementTree import ElementTree class Sitemap(object): """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" def __init__(self, xmltext): - tree = ElementTree() - tree.parse(StringIO(xmltext)) - self._root = tree.getroot() + xmlp = lxml.etree.XMLParser(recover=True) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) rt = self._root.tag self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt @@ -26,7 +24,9 @@ class Sitemap(object): tag = el.tag name = tag.split('}', 1)[1] if '}' in tag else tag d[name] = el.text.strip() if el.text else '' - yield d + if 'loc' in d: + yield d + def sitemap_urls_from_robots(robots_text): """Return an iterator over all sitemap urls contained in the given diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dd7ca85ab..4ef014fd9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -79,9 +79,25 @@ def escape_ajax(url): Return the crawleable url according to: http://code.google.com/web/ajaxcrawling/docs/getting-started.html - TODO: add support for urls with query arguments - >>> escape_ajax("www.example.com/ajax.html#!key=value") 'www.example.com/ajax.html?_escaped_fragment_=key=value' + >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value") + 'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key=value' + >>> escape_ajax("www.example.com/ajax.html?#!key=value") + 'www.example.com/ajax.html?_escaped_fragment_=key=value' + >>> escape_ajax("www.example.com/ajax.html#!") + 'www.example.com/ajax.html?_escaped_fragment_=' + + URLs that are not "AJAX crawlable" (according to Google) returned as-is: + + >>> escape_ajax("www.example.com/ajax.html#key=value") + 'www.example.com/ajax.html#key=value' + >>> escape_ajax("www.example.com/ajax.html#") + 'www.example.com/ajax.html#' + >>> escape_ajax("www.example.com/ajax.html") + 'www.example.com/ajax.html' """ - return url.replace('#!', '?_escaped_fragment_=') + defrag, frag = urlparse.urldefrag(url) + if not frag.startswith('!'): + return url + return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:]) diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py index 0350e60cf..11d25d377 100644 --- a/scrapy/xlib/pydispatch/robustapply.py +++ b/scrapy/xlib/pydispatch/robustapply.py @@ -6,44 +6,50 @@ and subset the given arguments to match only those which are acceptable. """ -def function( receiver ): - """Get function-like callable object for given receiver +import inspect - returns (function_or_method, codeObject, fromMethod) +def function(receiver): + """Get function-like callable object for given receiver - If fromMethod is true, then the callable already - has its first argument bound - """ - if hasattr(receiver, '__call__'): - # receiver is a class instance; assume it is callable. - # Reassign receiver to the actual method that will be called. - if hasattr( receiver.__call__, 'im_func') or hasattr( receiver.__call__, 'im_code'): - receiver = receiver.__call__ - if hasattr( receiver, 'im_func' ): - # an instance-method... - return receiver, receiver.im_func.func_code, 1 - elif not hasattr( receiver, 'func_code'): - raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver))) - return receiver, receiver.func_code, 0 + returns (function_or_method, codeObject, fromMethod) + + If fromMethod is true, then the callable already + has its first argument bound + """ + if inspect.isclass(receiver) and hasattr(receiver, '__call__'): + # receiver is a class instance; assume it is callable. + # Reassign receiver to the actual method that will be called. + if hasattr(receiver.__call__, 'im_func') or \ + hasattr(receiver.__call__, 'im_code'): + receiver = receiver.__call__ + + if hasattr( receiver, 'im_func' ): + # an instance-method... + return receiver, receiver.im_func.func_code, 1 + elif not hasattr(receiver, 'func_code'): + raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver))) + + return receiver, receiver.func_code, 0 def robustApply(receiver, *arguments, **named): - """Call receiver with arguments and an appropriate subset of named - """ - receiver, codeObject, startIndex = function( receiver ) - acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount] - for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]: - if named.has_key( name ): - raise TypeError( - """Argument %r specified both positionally and as a keyword for calling %r"""% ( - name, receiver, - ) - ) - if not (codeObject.co_flags & 8): - # fc does not have a **kwds type parameter, therefore - # remove unacceptable arguments. - for arg in named.keys(): - if arg not in acceptable: - del named[arg] - return receiver(*arguments, **named) + """Call receiver with arguments and an appropriate subset of named + """ + receiver, codeObject, startIndex = function(receiver) + acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount] + for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]: + if named.has_key(name): + raise TypeError( + """Argument %r specified both positionally and as a keyword for calling %r"""% ( + name, receiver, + ) + ) + + if not (codeObject.co_flags & 8): + # fc does not have a **kwds type parameter, therefore + # remove unacceptable arguments. + for arg in named.keys(): + if arg not in acceptable: + del named[arg] + + return receiver(*arguments, **named) - \ No newline at end of file diff --git a/tox.ini b/tox.ini index 157c0c217..9ee3923f2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,25 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27 +envlist = py26, py27, lucid, precise [testenv] +deps = + -r{toxinidir}/.travis/requirements-latest.txt commands = - pip install --use-mirrors -r .travis/requirements-latest.txt - trial scrapy + {toxinidir}/bin/runtests.sh [] + +[testenv:lucid] +basepython = python2.6 +deps = + -r{toxinidir}/.travis/requirements-lucid.txt + +[testenv:precise] +basepython = python2.7 +deps = + -r{toxinidir}/.travis/requirements-precise.txt + +[testenv:windows] +commands = + {toxinidir}/bin/runtests.bat [] +sitepackages = False