From 8778af5c5be50a5d746751352f8d710d1f24681c Mon Sep 17 00:00:00 2001 From: joehillen Date: Wed, 12 Dec 2012 08:42:18 -0800 Subject: [PATCH 01/62] Explicitly check if an object is a class is pydispatch. * This is to fix a disagreement in between CPython and PyPy. * https://gist.github.com/4220533 --- scrapy/xlib/pydispatch/robustapply.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py index 0350e60cf..f7a83da09 100644 --- a/scrapy/xlib/pydispatch/robustapply.py +++ b/scrapy/xlib/pydispatch/robustapply.py @@ -6,6 +6,8 @@ and subset the given arguments to match only those which are acceptable. """ +import inspect + def function( receiver ): """Get function-like callable object for given receiver @@ -14,7 +16,7 @@ def function( receiver ): If fromMethod is true, then the callable already has its first argument bound """ - if hasattr(receiver, '__call__'): + if inspect.isclass(receiver) and hasattr(receiver, '__call__'): # receiver is a class instance; assume it is callable. # Reassign receiver to the actual method that will be called. if hasattr( receiver.__call__, 'im_func') or hasattr( receiver.__call__, 'im_code'): @@ -46,4 +48,4 @@ def robustApply(receiver, *arguments, **named): del named[arg] return receiver(*arguments, **named) - \ No newline at end of file + From c505b3367283e8e6f77b3cea00f924ca5e5e2758 Mon Sep 17 00:00:00 2001 From: joehillen Date: Wed, 12 Dec 2012 08:53:32 -0800 Subject: [PATCH 02/62] PEP8 on pydispatch/robustapply.py --- scrapy/xlib/pydispatch/robustapply.py | 76 ++++++++++++++------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/scrapy/xlib/pydispatch/robustapply.py b/scrapy/xlib/pydispatch/robustapply.py index f7a83da09..11d25d377 100644 --- a/scrapy/xlib/pydispatch/robustapply.py +++ b/scrapy/xlib/pydispatch/robustapply.py @@ -8,44 +8,48 @@ those which are acceptable. import inspect -def function( receiver ): - """Get function-like callable object for given receiver +def function(receiver): + """Get function-like callable object for given receiver - returns (function_or_method, codeObject, fromMethod) + returns (function_or_method, codeObject, fromMethod) - If fromMethod is true, then the callable already - has its first argument bound - """ - if inspect.isclass(receiver) and hasattr(receiver, '__call__'): - # receiver is a class instance; assume it is callable. - # Reassign receiver to the actual method that will be called. - if hasattr( receiver.__call__, 'im_func') or hasattr( receiver.__call__, 'im_code'): - receiver = receiver.__call__ - if hasattr( receiver, 'im_func' ): - # an instance-method... - return receiver, receiver.im_func.func_code, 1 - elif not hasattr( receiver, 'func_code'): - raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver))) - return receiver, receiver.func_code, 0 + If fromMethod is true, then the callable already + has its first argument bound + """ + if inspect.isclass(receiver) and hasattr(receiver, '__call__'): + # receiver is a class instance; assume it is callable. + # Reassign receiver to the actual method that will be called. + if hasattr(receiver.__call__, 'im_func') or \ + hasattr(receiver.__call__, 'im_code'): + receiver = receiver.__call__ + + if hasattr( receiver, 'im_func' ): + # an instance-method... + return receiver, receiver.im_func.func_code, 1 + elif not hasattr(receiver, 'func_code'): + raise ValueError('unknown reciever type %s %s'%(receiver, type(receiver))) + + return receiver, receiver.func_code, 0 def robustApply(receiver, *arguments, **named): - """Call receiver with arguments and an appropriate subset of named - """ - receiver, codeObject, startIndex = function( receiver ) - acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount] - for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]: - if named.has_key( name ): - raise TypeError( - """Argument %r specified both positionally and as a keyword for calling %r"""% ( - name, receiver, - ) - ) - if not (codeObject.co_flags & 8): - # fc does not have a **kwds type parameter, therefore - # remove unacceptable arguments. - for arg in named.keys(): - if arg not in acceptable: - del named[arg] - return receiver(*arguments, **named) + """Call receiver with arguments and an appropriate subset of named + """ + receiver, codeObject, startIndex = function(receiver) + acceptable = codeObject.co_varnames[startIndex+len(arguments):codeObject.co_argcount] + for name in codeObject.co_varnames[startIndex:startIndex+len(arguments)]: + if named.has_key(name): + raise TypeError( + """Argument %r specified both positionally and as a keyword for calling %r"""% ( + name, receiver, + ) + ) + + if not (codeObject.co_flags & 8): + # fc does not have a **kwds type parameter, therefore + # remove unacceptable arguments. + for arg in named.keys(): + if arg not in acceptable: + del named[arg] + + return receiver(*arguments, **named) - From c70f050bdaeec6d2c32039c005511a32d3b21cc5 Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Thu, 27 Jun 2013 11:45:29 -0300 Subject: [PATCH 03/62] from_crawler added to spiders --- scrapy/spidermanager.py | 6 +++++- scrapy/tests/test_spidermanager/__init__.py | 8 +++++--- .../tests/test_spidermanager/test_spiders/spider4.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 scrapy/tests/test_spidermanager/test_spiders/spider4.py diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 6443c45fa..9ab570964 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -33,6 +33,7 @@ class SpiderManager(object): @classmethod def from_crawler(cls, crawler): sm = cls.from_settings(crawler.settings) + sm.crawler = crawler crawler.signals.connect(sm.close_spider, signals.spider_closed) return sm @@ -41,7 +42,10 @@ class SpiderManager(object): spcls = self._spiders[spider_name] except KeyError: raise KeyError("Spider not found: %s" % spider_name) - return spcls(**spider_kwargs) + if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'): + return spcls.from_crawler(self.crawler, **spider_kwargs) + else: + return spcls(**spider_kwargs) def find_by_request(self, request): return [name for name, cls in self._spiders.iteritems() diff --git a/scrapy/tests/test_spidermanager/__init__.py b/scrapy/tests/test_spidermanager/__init__.py index 4b46022e6..c8697d3b5 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/scrapy/tests/test_spidermanager/__init__.py @@ -1,6 +1,5 @@ import sys import os -import weakref import shutil from zope.interface.verify import verifyObject @@ -9,7 +8,6 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spider when running this test # alone -import scrapy.spider from scrapy.interfaces import ISpiderManager from scrapy.spidermanager import SpiderManager from scrapy.http import Request @@ -36,7 +34,7 @@ class SpiderManagerTest(unittest.TestCase): def test_list(self): self.assertEqual(set(self.spiderman.list()), - set(['spider1', 'spider2', 'spider3'])) + set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_create(self): spider1 = self.spiderman.create("spider1") @@ -66,3 +64,7 @@ class SpiderManagerTest(unittest.TestCase): def test_load_base_spider(self): self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0']) assert len(self.spiderman._spiders) == 0 + + def test_load_from_crawler(self): + spider = self.spiderman.create('spider4', a='OK') + self.assertEqual(spider.a, 'OK') diff --git a/scrapy/tests/test_spidermanager/test_spiders/spider4.py b/scrapy/tests/test_spidermanager/test_spiders/spider4.py new file mode 100644 index 000000000..6f66ad37e --- /dev/null +++ b/scrapy/tests/test_spidermanager/test_spiders/spider4.py @@ -0,0 +1,10 @@ +from scrapy.spider import BaseSpider + +class Spider4(BaseSpider): + name = "spider4" + + @classmethod + def from_crawler(cls, crawler, **kwargs): + o = cls(**kwargs) + o.crawler = crawler + return o From 1af47165b4ffaabd1dc9eb41603c98a705c0bafa Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Thu, 27 Jun 2013 16:52:59 -0200 Subject: [PATCH 04/62] added ftp handler --- scrapy/core/downloader/handlers/ftp.py | 104 +++++++++++++++++++++++ scrapy/settings/default_settings.py | 1 + scrapy/tests/test_downloader_handlers.py | 85 ++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 scrapy/core/downloader/handlers/ftp.py diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py new file mode 100644 index 000000000..70c526490 --- /dev/null +++ b/scrapy/core/downloader/handlers/ftp.py @@ -0,0 +1,104 @@ +""" +An asynchronous FTP file download handler for scrapy which somehow emulates an http response. + +FTP connection parameters are passed using the request meta field: +- ftp_user (required) +- ftp_password (required) +- ftp_passive (by default, enabled) sets FTP connection passive mode +- ftp_local_filename + - If not given, file data will come in the response.body, as a normal scrapy Response, + which will imply that the entire file will be on memory. + - if given, file data will be saved in a local file with the given name + This helps when downloading very big files to avoid memory issues. In addition, for + convenience the local file name will also be given in the response body. + +The status of the built html response will be, by default +- 200 in case of success +- 404 in case specified file was not found in the server (ftp code 550) + +or raise corresponding ftp exception otherwise + +The matching from server ftp command return codes to html response codes is defined in the +CODE_MAPPING attribute of the handler class. The key 'default' is used for any code +that is not explicitly present among the map keys. You may need to overwrite this +mapping if want a different behaviour than default. + +In case of status 200 request, response.headers will come with two keys: + 'Local Filename' - with the value of the local filename if given + 'Size' - with size of the downloaded data +""" + +import re +from urlparse import urlparse +from cStringIO import StringIO + +from twisted.internet import reactor +from twisted.protocols.ftp import FTPClient, CommandFailed +from twisted.internet.protocol import Protocol, ClientCreator + +from scrapy.http import Response +from scrapy.responsetypes import responsetypes + +class ReceivedDataProtocol(Protocol): + def __init__(self, filename=None): + self.__filename = filename + self.body = open(filename, "w") if filename else StringIO() + self.size = 0 + + def dataReceived(self, data): + self.body.write(data) + self.size += len(data) + + @property + def filename(self): + return self.__filename + + def close(self): + self.body.close() if self.filename else self.body.reset() + +_CODE_RE = re.compile("\d+") +class FTPDownloadHandler(object): + + CODE_MAPPING = { + "550": 404, + "default": 503, + } + + def __init__(self, setting): + pass + + def download_request(self, request, spider): + parsed_url = urlparse(request.url) + creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"], + request.meta["ftp_password"], + passive=request.meta.get("ftp_passive", 1)) + return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, + request, parsed_url.path) + + def gotClient(self, client, request, filepath): + self.client = client + protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename")) + return client.retrieveFile(filepath, protocol)\ + .addCallbacks(callback=self._build_response, + callbackArgs=(request, protocol), + errback=self._failed, + errbackArgs=(request,)) + + def _build_response(self, result, request, protocol): + self.result = result + respcls = responsetypes.from_args(url=request.url) + protocol.close() + body = protocol.filename or protocol.body.read() + headers = {"local filename": protocol.filename or '', "size": protocol.size} + return respcls(url=request.url, status=200, body=body, headers=headers) + + def _failed(self, result, request): + message = result.getErrorMessage() + if result.type == CommandFailed: + m = _CODE_RE.search(message) + if m: + ftpcode = m.group() + httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) + return Response(url=request.url, status=httpcode, body=message) + raise result.type(result.value) + diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 94fac60d6..9abbba616 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -56,6 +56,7 @@ DOWNLOAD_HANDLERS_BASE = { 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler', + 'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler', } DOWNLOAD_TIMEOUT = 180 # 3mins diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 18be4ed35..857cd3c9b 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -9,6 +9,9 @@ from twisted.web import server, static, util, resource from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource +from twisted.protocols.ftp import FTPRealm, FTPFactory +from twisted.cred import portal, checkers, credentials +from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers.file import FileDownloadHandler @@ -16,6 +19,8 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownlo from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler +from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler + from scrapy.spider import BaseSpider from scrapy.http import Request from scrapy.settings import Settings @@ -326,3 +331,83 @@ class S3TestCase(unittest.TestCase): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') + +class FTPTestCase(unittest.TestCase): + + username = "scrapy" + password = "passwd" + + def setUp(self): + # setup dirs and test file + self.directory = self.mktemp() + os.mkdir(self.directory) + userdir = os.path.join(self.directory, self.username) + os.mkdir(userdir) + FilePath(userdir).child('file.txt').setContent("I have the power!") + + # setup server + realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) + p = portal.Portal(realm) + users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() + users_checker.addUser(self.username, self.password) + p.registerChecker(users_checker, credentials.IUsernamePassword) + self.factory = FTPFactory(portal=p) + self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") + self.portNum = self.port.getHost().port + self.download_handler = FTPDownloadHandler(Settings()) + self.addCleanup(self.port.stopListening) + + def _add_test_callbacks(self, deferred, callback=None, errback=None): + def _clean(data): + self.download_handler.client.transport.loseConnection() + return data + deferred.addCallback(_clean) + if callback: + deferred.addCallback(callback) + if errback: + deferred.addErrback(errback) + return deferred + + def test_ftp_download_success(self): + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": self.password}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.status, 200) + self.assertEqual(r.body, 'I have the power!') + self.assertEqual(r.headers, {'Local Filename': [''], 'Size': [17]}) + return self._add_test_callbacks(d, _test) + + def test_ftp_download_notexist(self): + request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": self.password}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.status, 404) + return self._add_test_callbacks(d, _test) + + def test_ftp_local_filename(self): + local_fname = "/tmp/file.txt" + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.body, local_fname) + self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': [17]}) + self.assertTrue(os.path.exists(local_fname)) + with open(local_fname) as f: + self.assertEqual(f.read(), "I have the power!") + os.remove(local_fname) + return self._add_test_callbacks(d, _test) + + def test_invalid_credentials(self): + request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, + meta={"ftp_user": self.username, "ftp_password": 'invalid'}) + d = self.download_handler.download_request(request, None) + + def _test(r): + self.assertEqual(r.type, ConnectionLost) + return self._add_test_callbacks(d, errback=_test) From 35fbec2e63d12ce6651ad3000e304cc5e4e8a43b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 8 Jul 2013 22:51:03 +0600 Subject: [PATCH 05/62] allow passing arguments to tox e.g. "tox -- scrapy.tests.test_contrib_loader" runs test_contrib_loader for Python 2.6 and 2.7 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 157c0c217..b97460c73 100644 --- a/tox.ini +++ b/tox.ini @@ -9,4 +9,4 @@ envlist = py26, py27 [testenv] commands = pip install --use-mirrors -r .travis/requirements-latest.txt - trial scrapy + trial {posargs:scrapy} From b98e80cbb835b26d74de4514e2fff103b76af360 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 00:26:59 +0600 Subject: [PATCH 06/62] use tox deps instead of "manual" pip install; add lucid and precise environments to tox.ini --- .travis.yml | 1 + .travis/requirements-latest.txt | 1 - .travis/requirements-lucid.txt | 1 - .travis/requirements-precise.txt | 1 - tox.ini | 15 +++++++++++++-- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2012774a8..c14882249 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: - BUILDENV=latest install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt + - pip install . script: - trial scrapy branches: diff --git a/.travis/requirements-latest.txt b/.travis/requirements-latest.txt index 576f2494e..585968196 100644 --- a/.travis/requirements-latest.txt +++ b/.travis/requirements-latest.txt @@ -3,4 +3,3 @@ lxml twisted boto Pillow -. diff --git a/.travis/requirements-lucid.txt b/.travis/requirements-lucid.txt index 20e601804..d55b57799 100644 --- a/.travis/requirements-lucid.txt +++ b/.travis/requirements-lucid.txt @@ -4,4 +4,3 @@ lxml==2.2.4 twisted==10.0.0 boto==1.9b Pillow<2.0 -. diff --git a/.travis/requirements-precise.txt b/.travis/requirements-precise.txt index 2bb99def0..276f7523b 100644 --- a/.travis/requirements-precise.txt +++ b/.travis/requirements-precise.txt @@ -4,4 +4,3 @@ lxml==2.3.2 twisted==11.1.0 boto==2.2.2 Pillow<2.0 -. diff --git a/tox.ini b/tox.ini index b97460c73..036c3148c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,20 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27 +envlist = py26, py27, lucid, precise [testenv] +deps = + -r{toxinidir}/.travis/requirements-latest.txt commands = - pip install --use-mirrors -r .travis/requirements-latest.txt trial {posargs:scrapy} + +[testenv:lucid] +basepython = python2.6 +deps = + -r{toxinidir}/.travis/requirements-lucid.txt + +[testenv:precise] +basepython = python2.7 +deps = + -r{toxinidir}/.travis/requirements-precise.txt From 2105ec58eb63d4ac30317c59b5cd21afdad0341d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 00:28:39 +0600 Subject: [PATCH 07/62] add django to test requirements and fix test_djangoitem for modern django versions --- .travis/requirements-latest.txt | 1 + .travis/requirements-precise.txt | 1 + scrapy/tests/test_djangoitem/settings.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/.travis/requirements-latest.txt b/.travis/requirements-latest.txt index 585968196..15dd8936f 100644 --- a/.travis/requirements-latest.txt +++ b/.travis/requirements-latest.txt @@ -3,3 +3,4 @@ lxml twisted boto Pillow +django diff --git a/.travis/requirements-precise.txt b/.travis/requirements-precise.txt index 276f7523b..625708aa2 100644 --- a/.travis/requirements-precise.txt +++ b/.travis/requirements-precise.txt @@ -4,3 +4,4 @@ lxml==2.3.2 twisted==11.1.0 boto==2.2.2 Pillow<2.0 +django==1.3.1 diff --git a/scrapy/tests/test_djangoitem/settings.py b/scrapy/tests/test_djangoitem/settings.py index e3300db9d..1bee92477 100644 --- a/scrapy/tests/test_djangoitem/settings.py +++ b/scrapy/tests/test_djangoitem/settings.py @@ -4,3 +4,5 @@ DATABASES = { 'NAME': ':memory:', } } + +SECRET_KEY = 'top-secret' From 3a6dff3090e9fa9b46cd64b7702d6dd9f8e95acc Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 00:30:39 +0600 Subject: [PATCH 08/62] make tox test running more consistent with bin/runtests.sh script --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 036c3148c..f06c7b840 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = py26, py27, lucid, precise deps = -r{toxinidir}/.travis/requirements-latest.txt commands = - trial {posargs:scrapy} + trial {posargs:--reporter=text scrapy} [testenv:lucid] basepython = python2.6 From f29e5d0ca121e183174d17d59aac6d65984a92e7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 00:38:00 +0600 Subject: [PATCH 09/62] reuse runtests.sh and runtests.bat scripts in tox.ini --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f06c7b840..ddfc69371 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = py26, py27, lucid, precise deps = -r{toxinidir}/.travis/requirements-latest.txt commands = - trial {posargs:--reporter=text scrapy} + {toxinidir}/bin/runtests.sh [testenv:lucid] basepython = python2.6 @@ -21,3 +21,7 @@ deps = basepython = python2.7 deps = -r{toxinidir}/.travis/requirements-precise.txt + +[testenv:windows] +commands = + {toxinidir}/bin/runtests.bat From 0d9f7843ba49ef0ab25208e83f5846a9a3ca936f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 00:41:13 +0600 Subject: [PATCH 10/62] allow passing arguments to runtests script via tox --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index ddfc69371..9f0e77753 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = py26, py27, lucid, precise deps = -r{toxinidir}/.travis/requirements-latest.txt commands = - {toxinidir}/bin/runtests.sh + {toxinidir}/bin/runtests.sh [] [testenv:lucid] basepython = python2.6 @@ -24,4 +24,4 @@ deps = [testenv:windows] commands = - {toxinidir}/bin/runtests.bat + {toxinidir}/bin/runtests.bat [] From 8c555fd46782d398017168f7a63236692d6e37e7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 01:50:52 +0600 Subject: [PATCH 11/62] Enable Travis CI for pull requests and feature branches. Fix GH-340. --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2012774a8..9b34bc3aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,6 @@ install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt script: - trial scrapy -branches: - only: - - master - - /^[0-9].*$/ notifications: irc: channels: From 3fe2a32683bce14e1fa1576cdac3a13b9f579b0a Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 9 Jul 2013 01:58:50 +0600 Subject: [PATCH 12/62] handle GET parameters for AJAX crawlable URLs: --- scrapy/utils/url.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dd7ca85ab..4ef014fd9 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -79,9 +79,25 @@ def escape_ajax(url): Return the crawleable url according to: http://code.google.com/web/ajaxcrawling/docs/getting-started.html - TODO: add support for urls with query arguments - >>> escape_ajax("www.example.com/ajax.html#!key=value") 'www.example.com/ajax.html?_escaped_fragment_=key=value' + >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value") + 'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key=value' + >>> escape_ajax("www.example.com/ajax.html?#!key=value") + 'www.example.com/ajax.html?_escaped_fragment_=key=value' + >>> escape_ajax("www.example.com/ajax.html#!") + 'www.example.com/ajax.html?_escaped_fragment_=' + + URLs that are not "AJAX crawlable" (according to Google) returned as-is: + + >>> escape_ajax("www.example.com/ajax.html#key=value") + 'www.example.com/ajax.html#key=value' + >>> escape_ajax("www.example.com/ajax.html#") + 'www.example.com/ajax.html#' + >>> escape_ajax("www.example.com/ajax.html") + 'www.example.com/ajax.html' """ - return url.replace('#!', '?_escaped_fragment_=') + defrag, frag = urlparse.urldefrag(url) + if not frag.startswith('!'): + return url + return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:]) From aa7b529975748ef39b9f73fa087af064c80563c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 10 Jul 2013 12:32:43 -0300 Subject: [PATCH 13/62] do not log Starting/Stopping factory messages --- scrapy/core/downloader/handlers/http11.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 65ac8a5ae..1f72035cd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -25,6 +25,7 @@ class HTTP11DownloadHandler(object): def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool._factory.noisy = False self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) self._contextFactory = self._contextFactoryClass() From 1a1c93fafe9bd3f522bce87f64d2347f4736d14f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 15 Jul 2013 15:47:34 +0600 Subject: [PATCH 14/62] tiny FormRequest doc fix --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 091f4258d..675574e28 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -318,7 +318,7 @@ key-value fields, you can return a :class:`FormRequest` object (from your spider) like this:: return [FormRequest(url="http://www.example.com/post/action", - formdata={'name': 'John Doe', age: '27'}, + formdata={'name': 'John Doe', 'age': '27'}, callback=self.after_post)] .. _topics-request-response-ref-request-userlogin: From 9ff82d6695860d57a00b6ecf9c63928788d303fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 15 Jul 2013 12:26:16 -0300 Subject: [PATCH 15/62] use lxml parser with recover option to parse invalid sitemaps. closes #347 thanks chrissboo for reporting the issue and proposing a fix. --- scrapy/tests/test_utils_sitemap.py | 37 ++++++++++++++++++++++++++++-- scrapy/utils/sitemap.py | 12 +++++----- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/scrapy/tests/test_utils_sitemap.py b/scrapy/tests/test_utils_sitemap.py index 99b66feec..8be94a309 100644 --- a/scrapy/tests/test_utils_sitemap.py +++ b/scrapy/tests/test_utils_sitemap.py @@ -120,12 +120,45 @@ Disallow: /s*/*tags Sitemap: http://example.com/sitemap.xml Sitemap: http://example.com/sitemap-product-index.xml -# Forums +# Forums Disallow: /forum/search/ Disallow: /forum/active/ """ - self.assertEqual(list(sitemap_urls_from_robots(robots)), + self.assertEqual(list(sitemap_urls_from_robots(robots)), ['http://example.com/sitemap.xml', 'http://example.com/sitemap-product-index.xml']) + def test_sitemap_blanklines(self): + """Assert we can deal with starting blank lines before tag""" + s = Sitemap("""\ + + + + + + +http://www.example.com/sitemap1.xml +2013-07-15 + + + +http://www.example.com/sitemap2.xml +2013-07-15 + + + +http://www.example.com/sitemap3.xml +2013-07-15 + + + + +""") + self.assertEqual(list(s), [ + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap1.xml'}, + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap2.xml'}, + {'lastmod': '2013-07-15', 'loc': 'http://www.example.com/sitemap3.xml'}, + ]) + + if __name__ == '__main__': unittest.main() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 71d8122ab..38e38d6a9 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -4,18 +4,16 @@ Module for processing Sitemaps. Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +import lxml.etree -from cStringIO import StringIO -from xml.etree.cElementTree import ElementTree class Sitemap(object): """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" def __init__(self, xmltext): - tree = ElementTree() - tree.parse(StringIO(xmltext)) - self._root = tree.getroot() + xmlp = lxml.etree.XMLParser(recover=True) + self._root = lxml.etree.fromstring(xmltext, parser=xmlp) rt = self._root.tag self.type = self._root.tag.split('}', 1)[1] if '}' in rt else rt @@ -26,7 +24,9 @@ class Sitemap(object): tag = el.tag name = tag.split('}', 1)[1] if '}' in tag else tag d[name] = el.text.strip() if el.text else '' - yield d + if 'loc' in d: + yield d + def sitemap_urls_from_robots(robots_text): """Return an iterator over all sitemap urls contained in the given From 66aa1331fced51221349381230c9eec86fe9ee71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 16 Jul 2013 00:49:35 -0300 Subject: [PATCH 16/62] Twisted pre 10.2 (lucid) can not setup the required ftp server for tests --- scrapy/tests/test_downloader_handlers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 857cd3c9b..c71bfbcb8 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -11,9 +11,10 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ PayloadResource, BrokenDownloadResource from twisted.protocols.ftp import FTPRealm, FTPFactory from twisted.cred import portal, checkers, credentials -from twisted.protocols.ftp import FTPClient, ConnectionLost +from twisted.protocols.ftp import FTPClient, ConnectionLost from w3lib.url import path_to_file_uri +from scrapy import twisted_version from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -333,10 +334,13 @@ class S3TestCase(unittest.TestCase): 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') class FTPTestCase(unittest.TestCase): - + username = "scrapy" password = "passwd" + if twisted_version < (10, 2, 0): + skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" + def setUp(self): # setup dirs and test file self.directory = self.mktemp() From e12b689c4f399e44e7b5ab9df989dfc8ac79023e Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 16 Jul 2013 14:26:53 -0400 Subject: [PATCH 17/62] Updated documentation of spider arguments to include required super call. --- docs/topics/spiders.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4e969498d..3174efd52 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -61,9 +61,10 @@ Spiders receive arguments in their constructors:: class MySpider(BaseSpider): name = 'myspider' - def __init__(self, category=None): + def __init__(self, category=None, *args, **kwargs): self.start_urls = ['http://www.example.com/categories/%s' % category] # ... + super(MySpider, self).__init__(*args, **kwargs) Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. From 1ca31244b087d1d15b25456e644d40e62d247847 Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 16 Jul 2013 14:50:10 -0400 Subject: [PATCH 18/62] Fixed ordering of super argument call. --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 3174efd52..6586db668 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -62,9 +62,9 @@ Spiders receive arguments in their constructors:: name = 'myspider' def __init__(self, category=None, *args, **kwargs): + super(MySpider, self).__init__(*args, **kwargs) self.start_urls = ['http://www.example.com/categories/%s' % category] # ... - super(MySpider, self).__init__(*args, **kwargs) Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. From d74b60051d6b4901c7c3058332a2d14c979b01d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 16 Jul 2013 16:22:03 -0300 Subject: [PATCH 19/62] take in count response latencies when testing download delays --- scrapy/tests/mockserver.py | 57 +++++++++++++++++++++----------------- scrapy/tests/spiders.py | 6 ++-- scrapy/tests/test_crawl.py | 29 ++++++++++++++----- 3 files changed, 58 insertions(+), 34 deletions(-) diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 8b1cc09e8..4dde6c2a9 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -13,31 +13,7 @@ def getarg(request, name, default=None, type=str): return default -class Follow(Resource): - - isLeaf = True - - def render(self, request): - total = getarg(request, "total", 100, type=int) - show = getarg(request, "show", 1, type=int) - order = getarg(request, "order", "desc") - n = getarg(request, "n", total, type=int) - if order == "rand": - nlist = [random.randint(1, total) for _ in range(show)] - else: # order == "desc" - nlist = range(n, max(n - show, 0), -1) - - s = """ """ - args = request.args.copy() - for nl in nlist: - args["n"] = [str(nl)] - argstr = urllib.urlencode(args, doseq=True) - s += "follow %d
" % (argstr, nl) - s += """""" - return s - - -class DeferMixin(Resource): +class DeferMixin(object): def deferRequest(self, request, delay, f, *a, **kw): def _cancelrequest(_): @@ -49,6 +25,37 @@ class DeferMixin(Resource): return d +class Follow(DeferMixin, Resource): + + isLeaf = True + + def render(self, request): + total = getarg(request, "total", 100, type=int) + show = getarg(request, "show", 1, type=int) + order = getarg(request, "order", "desc") + maxlatency = getarg(request, "maxlatency", 0, type=float) + n = getarg(request, "n", total, type=int) + if order == "rand": + nlist = [random.randint(1, total) for _ in range(show)] + else: # order == "desc" + nlist = range(n, max(n - show, 0), -1) + + lag = random.random() * maxlatency + self.deferRequest(request, lag, self.renderRequest, request, nlist) + return NOT_DONE_YET + + def renderRequest(self, request, nlist): + s = """ """ + args = request.args.copy() + for nl in nlist: + args["n"] = [str(nl)] + argstr = urllib.urlencode(args, doseq=True) + s += "follow %d
" % (argstr, nl) + s += """""" + request.write(s) + request.finish() + + class Delay(DeferMixin, Resource): isLeaf = True diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py index e1d06d85c..2d8dbd808 100644 --- a/scrapy/tests/spiders.py +++ b/scrapy/tests/spiders.py @@ -3,6 +3,7 @@ Some spiders used for testing and benchmarking """ import time +from urllib import urlencode from scrapy.spider import BaseSpider from scrapy.http import Request @@ -27,11 +28,12 @@ class FollowAllSpider(MetaSpider): name = 'follow' link_extractor = SgmlLinkExtractor() - def __init__(self, total=10, show=20, order="rand", *args, **kwargs): + def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): super(FollowAllSpider, self).__init__(*args, **kwargs) self.urls_visited = [] self.times = [] - url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order) + qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} + url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) self.start_urls = [url] def parse(self, response): diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 256f2d9b1..2a5fbdb62 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -11,6 +11,7 @@ def docrawl(spider, settings=None): crawler.crawl(spider) return crawler.start() + class CrawlTestCase(TestCase): def setUp(self): @@ -24,16 +25,30 @@ class CrawlTestCase(TestCase): def test_follow_all(self): spider = FollowAllSpider() yield docrawl(spider) - self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url + self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks def test_delay(self): - spider = FollowAllSpider() - yield docrawl(spider, {"DOWNLOAD_DELAY": 1}) - t = spider.times[0] - for t2 in spider.times[1:]: - self.assertTrue(t2-t > 0.45, "download delay too small: %s" % (t2-t)) - t = t2 + # short to long delays + yield self._test_delay(0.2, False) + yield self._test_delay(1, False) + # randoms + yield self._test_delay(0.2, True) + yield self._test_delay(1, True) + + @defer.inlineCallbacks + def _test_delay(self, delay, randomize): + settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} + spider = FollowAllSpider(maxlatency=delay * 2) + yield docrawl(spider, settings) + t = spider.times + totaltime = t[-1] - t[0] + avgd = totaltime / (len(t) - 1) + tolerance = 0.6 if randomize else 0.2 + self.assertTrue(avgd > delay * (1 - tolerance), + "download delay too small: %s" % avgd) + self.assertTrue(avgd < delay * (1 + tolerance), + "download delay too big: %s" % avgd) @defer.inlineCallbacks def test_timeout_success(self): From d7e11082f290766bf469ba68628f3903ec5f738a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 18 Jul 2013 11:07:42 -0300 Subject: [PATCH 20/62] it is not possible to enforce an upper limit when latency is out of control --- scrapy/tests/test_crawl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 2a5fbdb62..dcc186c2d 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -47,8 +47,6 @@ class CrawlTestCase(TestCase): tolerance = 0.6 if randomize else 0.2 self.assertTrue(avgd > delay * (1 - tolerance), "download delay too small: %s" % avgd) - self.assertTrue(avgd < delay * (1 + tolerance), - "download delay too big: %s" % avgd) @defer.inlineCallbacks def test_timeout_success(self): From ebc136ddb068173f7bc44e8e60e415cef4537f8a Mon Sep 17 00:00:00 2001 From: arijitchakraborty Date: Thu, 18 Jul 2013 19:42:58 +0530 Subject: [PATCH 21/62] Fixes for bug - Cookie retrieval for hosts with port + unittests for the fix --- scrapy/http/cookies.py | 5 +++++ .../tests/test_downloadermiddleware_cookies.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 8ad8de290..6902d29e8 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -23,6 +23,11 @@ class CookieJar(object): # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain req_host = urlparse_cached(request).netloc + + # Strip port numbers from netloc, if present + if ':' in req_host: + req_host = req_host.split(':')[0] + if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) if req_host.find(".") == -1: diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d9ed476f1..19c720a75 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -113,3 +113,19 @@ class CookiesMiddlewareTest(TestCase): req4 = Request('http://scrapytest.org/', meta=res2.meta) assert self.mw.process_request(req4, self.spider) is None self.assertEquals(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce') + + #cookies from hosts with port + req5_1 = Request('http://scrapytest.org:1104/') + assert self.mw.process_request(req5_1, self.spider) is None + + headers = {'Set-Cookie': 'C1=value1; path=/'} + res5_1 = Response('http://scrapytest.org:1104/', headers=headers, request=req5_1) + assert self.mw.process_response(req5_1, res5_1, self.spider) is res5_1 + + req5_2 = Request('http://scrapytest.org:1104/some-redirected-path') + assert self.mw.process_request(req5_2, self.spider) is None + self.assertEquals(req5_2.headers.get('Cookie'), 'C1=value1') + + req5_3 = Request('http://scrapytest.org/some-redirected-path') + assert self.mw.process_request(req5_3, self.spider) is None + self.assertEquals(req5_3.headers.get('Cookie'), 'C1=value1') From 66ff34cf05f5ed1624e8efb39d2b311ffe3ba5f4 Mon Sep 17 00:00:00 2001 From: arijitchakraborty Date: Thu, 18 Jul 2013 20:51:56 +0530 Subject: [PATCH 22/62] improving hostname extraction --- scrapy/http/cookies.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 6902d29e8..c137d5ad1 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -22,11 +22,7 @@ class CookieJar(object): # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain - req_host = urlparse_cached(request).netloc - - # Strip port numbers from netloc, if present - if ':' in req_host: - req_host = req_host.split(':')[0] + req_host = urlparse_cached(request).hostname if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) From fb770852e87d97196e31f27c33ee8eee89aecc27 Mon Sep 17 00:00:00 2001 From: arijitchakraborty Date: Mon, 22 Jul 2013 19:41:01 +0530 Subject: [PATCH 23/62] Skipping cookie retrieval for non http requests --- scrapy/http/cookies.py | 2 ++ scrapy/tests/test_downloadermiddleware_cookies.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index c137d5ad1..cc96cf8ac 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -23,6 +23,8 @@ class CookieJar(object): # the cookiejar implementation iterates through all domains # instead we restrict to potential matches on the domain req_host = urlparse_cached(request).hostname + if not req_host: + return if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index 19c720a75..5f5e7a3d5 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -129,3 +129,8 @@ class CookiesMiddlewareTest(TestCase): req5_3 = Request('http://scrapytest.org/some-redirected-path') assert self.mw.process_request(req5_3, self.spider) is None self.assertEquals(req5_3.headers.get('Cookie'), 'C1=value1') + + #skip cookie retrieval for not http request + req6 = Request('file:///scrapy/sometempfile') + assert self.mw.process_request(req6, self.spider) is None + self.assertEquals(req6.headers.get('Cookie'), None) From d227d530f6b898d6b6d35b539e8804c81d1a3ea3 Mon Sep 17 00:00:00 2001 From: Rocio Aramberri Date: Wed, 31 Jul 2013 18:37:46 -0300 Subject: [PATCH 24/62] Added COMPRESSION_ENABLED setting to enable or disable the HttpCompressionMiddleware Added COMPRESSION_ENABLE setting to docs Added COMPRESSION_ENABLED setting to default settings --- docs/topics/downloader-middleware.rst | 13 +++++++++++++ .../contrib/downloadermiddleware/httpcompression.py | 9 ++++++++- scrapy/settings/default_settings.py | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 64f0eb953..ac4945978 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -533,6 +533,19 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. +HttpCompressionMiddleware Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: COMPRESSION_ENABLED + +COMPRESSION_ENABLED +^^^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether the Compression middleware will be enabled. + + ChunkedTransferMiddleware ------------------------- diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py index 25af3e051..60d87da7a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ b/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -3,12 +3,19 @@ import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.exceptions import NotConfigured class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - + + @classmethod + def from_crawler(cls, crawler): + if not crawler.settings.getbool('COMPRESSION_ENABLED'): + raise NotConfigured + return cls() + def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', 'x-gzip,gzip,deflate') diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9abbba616..a16110b52 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -26,6 +26,8 @@ CLOSESPIDER_ERRORCOUNT = 0 COMMANDS_MODULE = '' +COMPRESSION_ENABLED = True + CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 From 0ad01c34d44c926d074d12d64e7bb2aae2a329a3 Mon Sep 17 00:00:00 2001 From: Hart Date: Sat, 3 Aug 2013 17:06:10 -0700 Subject: [PATCH 25/62] fixed typo to parallel fix on 0.16 branch --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 37ef4b8dc..88676a7ea 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -84,8 +84,8 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. -Does Scrapy crawl in breath-first or depth-first order? -------------------------------------------------------- +Does Scrapy crawl in breadth-first or depth-first order? +-------------------------------------------------------- By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which basically means that it crawls in `DFO order`_. This order is more convenient From c00c4d714886449e82bddf02a1557a9f99108d74 Mon Sep 17 00:00:00 2001 From: Hart Date: Sat, 3 Aug 2013 17:08:58 -0700 Subject: [PATCH 26/62] correction to description of example XPath retrieval in overview doc --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 1e9a62ee0..eed4bdac4 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -125,7 +125,7 @@ tag with ``id=specifications``:: .. highlight:: none -An XPath expression to select the description could be:: +An XPath expression to select the file size could be:: //div[@id='specifications']/p[2]/text()[2] From 4dc76e7cca5a4e80fc373c4726c1564ed3c0c2a7 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 9 Aug 2013 18:20:04 -0300 Subject: [PATCH 27/62] fixed scrapy.utils.gz.gunzip() broken after changes from Python 2.7.3 to 2.7.4 --- requirements.txt | 5 +++++ scrapy/utils/gz.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..3605e94a7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +Twisted>=8.0 +w3lib>=1.2 +queuelib +lxml +pyOpenSSL diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 7fe82ed39..aa8ffc6fc 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -14,7 +14,7 @@ def gunzip(data): try: chunk = f.read(8196) output += chunk - except (IOError, struct.error): + except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output is '' and f.extrabuf From a6693c9a5c89e9acc28f8886dd50778ff6ecace5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 9 Aug 2013 15:38:21 -0300 Subject: [PATCH 28/62] updated release notes and bumped version to 0.18.0 --- docs/news.rst | 89 +++++++++++++++++++++++++++++++++++++++++++++----- scrapy/VERSION | 2 +- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 3f6f67c9b..5e64ecaa3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,17 +3,88 @@ Release notes ============= -0.18 (unreleased) ------------------ +0.18.0 (released 2013-08-09) +---------------------------- -- :ref:`benchmarking` -- moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on -- add scrapy commands using external libraries (:issue:`260`) -- added ``--pdb`` option to ``scrapy`` command line tool -- added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. -- several improvements to spider contracts +- Lot of improvements to testsuite run using Tox, including a way to test on pypi +- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`) +- Use lxml recover option to parse sitemaps (:issue:`347`) +- Bugfix cookie merging by hostname and not by netloc (:issue:`352`) +- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`) +- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`) +- Support `dont_cache` request meta flag (:issue:`19`) +- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`) +- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`) +- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`) +- Support nested items in xml exporter (:issue:`66`) +- Improve cookies handling performance (:issue:`77`) +- Log dupe filtered requests once (:issue:`105`) +- Split redirection middleware into status and meta based middlewares (:issue:`78`) +- Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`) +- Support xpath form selection on `FormRequest.from_response` (:issue:`185`) +- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`) +- Bugfix signal dispatching on pypi interpreter (:issue:`205`) +- Improve request delay and concurrency handling (:issue:`206`) +- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`) +- Allow customization of messages logged by engine (:issue:`214`) +- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`) +- Extend Scrapy commands using setuptools entry points (:issue:`260`) +- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`) +- Support `settings.getdict` (:issue:`269`) +- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`) +- Added `Item.copy` (:issue:`290`) +- Collect idle downloader slots (:issue:`297`) +- Add `ftp://` scheme downloader handler (:issue:`329`) +- Added downloader benchmark webserver and spider tools :ref:`benchmarking` +- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on +- Add scrapy commands using external libraries (:issue:`260`) +- Added ``--pdb`` option to ``scrapy`` command line tool +- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. +- Several improvements to spider contracts - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 + +Contributters by number of commits:: + 130 Pablo Hoffman + 97 Daniel Graña + 20 Nicolás Ramírez + 13 Mikhail Korobov + 12 Pedro Faustino + 11 Steven Almeroth + 5 Rolando Espinoza La fuente + 4 Michal Danilak + 4 Alex Cepoi + 4 Alexandr N Zamaraev (aka tonal) + 3 paul + 3 Martin Olveyra + 3 Jordi Llonch + 3 arijitchakraborty + 2 Shane Evans + 2 joehillen + 2 Hart + 2 Dan + 1 Zuhao Wan + 1 whodatninja + 1 vkrest + 1 tpeng + 1 Tom Mortimer-Jones + 1 Rocio Aramberri + 1 Pedro + 1 notsobad + 1 Natan L + 1 Mark Grey + 1 Luan + 1 Libor Nenadál + 1 Juan M Uys + 1 Jonas Brunsgaard + 1 Ilya Baryshev + 1 Hasnain Lakhani + 1 Emanuel Schorsch + 1 Chris Tilden + 1 Capi Etheriel + 1 cacovsky + 1 Berend Iwema + 0.16.5 (released 2013-05-30) ---------------------------- diff --git a/scrapy/VERSION b/scrapy/VERSION index c5523bd09..66333910a 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -0.17.0 +0.18.0 From 80e25c5980dcc1078744fd923e96d00415a60ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 9 Aug 2013 19:09:47 -0300 Subject: [PATCH 29/62] bumped version to 0.19.0 --- scrapy/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/VERSION b/scrapy/VERSION index 66333910a..1cf0537c3 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -0.18.0 +0.19.0 From ed5b9068d22575acaa725c33bb18b08a9a680f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Sun, 11 Aug 2013 21:49:56 -0300 Subject: [PATCH 30/62] fix contributters list format --- docs/news.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/news.rst b/docs/news.rst index 5e64ecaa3..f48b11c9e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -45,6 +45,7 @@ Release notes - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 Contributters by number of commits:: + 130 Pablo Hoffman 97 Daniel Graña 20 Nicolás Ramírez From 892386ee01329a6d60681721ff455140b5f0a273 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 12 Aug 2013 18:59:13 -0300 Subject: [PATCH 31/62] tox.ini: disable sitepackages on windows, as a compiler is often not available --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9f0e77753..9ee3923f2 100644 --- a/tox.ini +++ b/tox.ini @@ -25,3 +25,4 @@ deps = [testenv:windows] commands = {toxinidir}/bin/runtests.bat [] +sitepackages = False From b43b5f575e6cfc47dd5568df3dab99ff49485692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 12 Aug 2013 22:42:45 -0300 Subject: [PATCH 32/62] adjust http11 pool size to per-domain concurrency --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1f72035cd..6ec10f723 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -20,11 +20,11 @@ from scrapy.utils.misc import load_object from scrapy import log - class HTTP11DownloadHandler(object): def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) + self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) self._contextFactory = self._contextFactoryClass() From 456b6f2ef5875d89b7ede5ab25d3676c1ef93921 Mon Sep 17 00:00:00 2001 From: olveyra Date: Tue, 13 Aug 2013 13:48:28 +0000 Subject: [PATCH 33/62] added a python native classes item exporter --- scrapy/contrib/exporter/__init__.py | 25 ++++++++++++++++-- scrapy/tests/test_contrib_exporter.py | 37 +++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index ab0e9a94f..89649915c 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -9,7 +9,7 @@ import json import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder - +from scrapy.item import BaseItem __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', \ 'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter', \ @@ -200,7 +200,6 @@ class MarshalItemExporter(BaseItemExporter): def export_item(self, item): marshal.dump(dict(self._get_serialized_fields(item)), self.file) - class PprintItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): @@ -210,3 +209,25 @@ class PprintItemExporter(BaseItemExporter): def export_item(self, item): itemdict = dict(self._get_serialized_fields(item)) self.file.write(pprint.pformat(itemdict) + '\n') + +class PythonItemExporter(BaseItemExporter): + + def serialize_field(self, field, name, value): + serializer = field.get('serializer', self._serialize_value) + return serializer(value) + + def _serialize_value(self, value): + if isinstance(value, BaseItem): + return self.export_item(value) + if isinstance(value, dict): + return dict(self._serialize_dict(value)) + if hasattr(value, '__iter__'): + return [self._serialize_value(v) for v in value] + return self._to_str_if_unicode(value) + + def _serialize_dict(self, value): + for key, val in value.iteritems(): + yield key, self._serialize_value(val) + + def export_item(self, item): + return dict(self._get_serialized_fields(item)) diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index d4865aafa..27f40551d 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -5,7 +5,7 @@ from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, \ - JsonItemExporter + JsonItemExporter, PythonItemExporter class TestItem(Item): name = Field() @@ -69,7 +69,41 @@ class BaseItemExporterTest(unittest.TestCase): self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), 'John\xc2\xa3') self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24') +class PythonItemExporterTest(BaseItemExporterTest): + def _get_exporter(self, **kwargs): + return PythonItemExporter(**kwargs) + def test_nested_item(self): + i1 = TestItem(name=u'Joseph', age='22') + i2 = TestItem(name=u'Maria', age=i1) + i3 = TestItem(name=u'Jesus', age=i2) + ie = self._get_exporter() + exported = ie.export_item(i3) + self.assertEqual(type(exported), dict) + self.assertEqual(exported, {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}) + self.assertEqual(type(exported['age']), dict) + self.assertEqual(type(exported['age']['age']), dict) + + def test_export_list(self): + i1 = TestItem(name=u'Joseph', age='22') + i2 = TestItem(name=u'Maria', age=[i1]) + i3 = TestItem(name=u'Jesus', age=[i2]) + ie = self._get_exporter() + exported = ie.export_item(i3) + self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual(type(exported['age'][0]), dict) + self.assertEqual(type(exported['age'][0]['age'][0]), dict) + + def test_export_item_dict_list(self): + i1 = TestItem(name=u'Joseph', age='22') + i2 = dict(name=u'Maria', age=[i1]) + i3 = TestItem(name=u'Jesus', age=[i2]) + ie = self._get_exporter() + exported = ie.export_item(i3) + self.assertEqual(exported, {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}) + self.assertEqual(type(exported['age'][0]), dict) + self.assertEqual(type(exported['age'][0]['age'][0]), dict) + class PprintItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): @@ -78,7 +112,6 @@ class PprintItemExporterTest(BaseItemExporterTest): def _check_output(self): self._assert_expected_item(eval(self.output.getvalue())) - class PickleItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): From c0b26e3d49a5d397de8e69ff95090332da5bceb4 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 14 Aug 2013 01:39:44 -0300 Subject: [PATCH 34/62] minor updates to 0.18 release notes --- docs/news.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index f48b11c9e..04cfa6e8b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -43,8 +43,16 @@ Release notes - Several improvements to spider contracts - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 +- added from_crawler method to spiders +- added system tests with mock server +- more improvements to Mac OS compatibility (thanks Alex Cepoi) +- several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez) +- support custom download slots +- added --spider option to "shell" command. +- log overridden settings when scrapy starts -Contributters by number of commits:: +Thanks to everyone who contribute to this release. Here is a list of +contributors sorted by number of commits:: 130 Pablo Hoffman 97 Daniel Graña From 49952a45ae9a77bcbb5d1c340212bb28490ea002 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 14 Aug 2013 02:48:10 -0300 Subject: [PATCH 35/62] add docstring to PythonItemExporter --- scrapy/contrib/exporter/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 89649915c..a841eda2f 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -211,6 +211,11 @@ class PprintItemExporter(BaseItemExporter): self.file.write(pprint.pformat(itemdict) + '\n') class PythonItemExporter(BaseItemExporter): + """The idea behind this exporter is to have a mechanism to serialize items + to built-in python types so any serialization library (like + json, msgpack, binc, etc) can be used on top of it. Its main goal is to + seamless support what BaseItemExporter does plus nested items. + """ def serialize_field(self, field, name, value): serializer = field.get('serializer', self._serialize_value) From 32b6364bcd1a17bfb6199af06ae613adf69e5058 Mon Sep 17 00:00:00 2001 From: Berend Iwema Date: Wed, 14 Aug 2013 12:59:01 +0200 Subject: [PATCH 36/62] #327 - Support STARTTLS / SSL option in email sender --- docs/topics/email.rst | 28 ++++++++++++++++++++++++++-- scrapy/mail.py | 18 +++++++++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 05628c479..a487299f1 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -39,11 +39,11 @@ MailSender class reference ========================== MailSender is the preferred class to use for sending emails from Scrapy, as it -uses `Twisted non-blocking IO`_, like the rest of the framework. +uses `Twisted non-blocking IO`_, like the rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) - :param smtphost: the SMTP host to use for sending the emails. If omitted, the + :param smtphost: the SMTP host to use for sending the emails. If omitted, the :setting:`MAIL_HOST` setting will be used. :type smtphost: str @@ -62,6 +62,12 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :param smtpport: the SMTP port to connect to :type smtpport: int + :param smtptls: enforce using SMTP STARTTLS + :type smtpport: boolean + + :param smtpssl: enforce using a secure SSL connection + :type smtpport: boolean + .. classmethod:: from_settings(settings) Instantiate using a Scrapy settings object, which will respect @@ -148,3 +154,21 @@ MAIL_PASS Default: ``None`` Password to use for SMTP authentication, along with :setting:`MAIL_USER`. + +.. setting:: MAIL_TLS + +MAIL_TLS +--------- + +Default: ``False`` + +Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS. + +.. setting:: MAIL_SSL + +MAIL_SSL +--------- + +Default: ``False`` + +Enforce connecting using an SSL encrypted connection diff --git a/scrapy/mail.py b/scrapy/mail.py index 8af6efdc4..bedf06895 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -11,7 +11,7 @@ from email.MIMEText import MIMEText from email.Utils import COMMASPACE, formatdate from email import Encoders -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, ssl from twisted.mail.smtp import ESMTPSenderFactory from scrapy import log @@ -19,18 +19,21 @@ from scrapy import log class MailSender(object): def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', - smtpuser=None, smtppass=None, smtpport=25, debug=False): + smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport self.smtpuser = smtpuser self.smtppass = smtppass + self.smtptls = smtptls + self.smtpssl = smtpssl self.mailfrom = mailfrom self.debug = debug @classmethod def from_settings(cls, settings): return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'], - settings['MAIL_PASS'], settings.getint('MAIL_PORT')) + settings['MAIL_PASS'], settings.getint('MAIL_PORT'), + settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL')) def send(self, to, subject, body, cc=None, attachs=(), _callback=None): if attachs: @@ -91,7 +94,12 @@ class MailSender(object): d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ - requireTransportSecurity=False) + requireTransportSecurity=self.smtptls) factory.noisy = False - reactor.connectTCP(self.smtphost, self.smtpport, factory) + + if self.smtpssl: + reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()) + else: + reactor.connectTCP(self.smtphost, self.smtpport, factory) + return d From 76ce8c5238c3056d31b141d3a717740964b96969 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Fri, 16 Aug 2013 17:02:31 +0200 Subject: [PATCH 37/62] FilesPipeline which enalbes to download any files. It has been extracted from ImagesPipelines. ImagesPipeline is built on top of FilesPipeline and consist only with convert image and thumbnail generation logic. --- scrapy/contrib/pipeline/files.py | 268 +++++++++++++++++++++++++++ scrapy/contrib/pipeline/images.py | 238 ++---------------------- scrapy/tests/test_pipeline_images.py | 8 +- 3 files changed, 289 insertions(+), 225 deletions(-) create mode 100644 scrapy/contrib/pipeline/files.py diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py new file mode 100644 index 000000000..5079040b6 --- /dev/null +++ b/scrapy/contrib/pipeline/files.py @@ -0,0 +1,268 @@ +""" +Files Pipeline +""" + +import hashlib +import os +import os.path +import rfc822 +import time +import urlparse +from collections import defaultdict +from cStringIO import StringIO + +from twisted.internet import defer, threads + +from scrapy import log +from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.http import Request +from scrapy.utils.misc import md5sum + + +class FileException(Exception): + """General media error exception""" + + +class FSFilesStore(object): + + def __init__(self, basedir): + if '://' in basedir: + basedir = basedir.split('://', 1)[1] + self.basedir = basedir + self._mkdir(self.basedir) + self.created_directories = defaultdict(set) + + def persist_file(self, key, buf, info, meta=None, headers=None): + absolute_path = self._get_filesystem_path(key) + self._mkdir(os.path.dirname(absolute_path), info) + with open(absolute_path, 'w') as f: + f.write(buf.getvalue()) + + def stat_image(self, key, info): + absolute_path = self._get_filesystem_path(key) + try: + last_modified = os.path.getmtime(absolute_path) + except: # FIXME: catching everything! + return {} + + with open(absolute_path, 'rb') as imagefile: + checksum = md5sum(imagefile) + + return {'last_modified': last_modified, 'checksum': checksum} + + def _get_filesystem_path(self, key): + path_comps = key.split('/') + return os.path.join(self.basedir, *path_comps) + + def _mkdir(self, dirname, domain=None): + seen = self.created_directories[domain] if domain else set() + if dirname not in seen: + if not os.path.exists(dirname): + os.makedirs(dirname) + seen.add(dirname) + + +class S3FilesStore(object): + + AWS_ACCESS_KEY_ID = None + AWS_SECRET_ACCESS_KEY = None + + POLICY = 'public-read' + HEADERS = { + 'Cache-Control': 'max-age=172800', + } + + def __init__(self, uri): + assert uri.startswith('s3://') + self.bucket, self.prefix = uri[5:].split('/', 1) + + def stat_image(self, key, info): + def _onsuccess(boto_key): + checksum = boto_key.etag.strip('"') + last_modified = boto_key.last_modified + modified_tuple = rfc822.parsedate_tz(last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + return {'checksum': checksum, 'last_modified': modified_stamp} + + return self._get_boto_key(key).addCallback(_onsuccess) + + def _get_boto_bucket(self): + from boto.s3.connection import S3Connection + # disable ssl (is_secure=False) because of this python bug: + # http://bugs.python.org/issue5103 + c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) + return c.get_bucket(self.bucket, validate=False) + + def _get_boto_key(self, key): + b = self._get_boto_bucket() + key_name = '%s%s' % (self.prefix, key) + return threads.deferToThread(b.get_key, key_name) + + def persist_file(self, key, buf, info, meta=None, headers=None): + """Upload file to S3 storage""" + b = self._get_boto_bucket() + key_name = '%s%s' % (self.prefix, key) + k = b.new_key(key_name) + if meta: + for metakey, metavalue in meta.iteritems(): + k.set_metadata(metakey, str(metavalue)) + h = self.HEADERS.copy() + if headers: + h.update(headers) + buf.seek(0) + return threads.deferToThread(k.set_contents_from_file, buf, + headers=h, policy=self.POLICY) + + +class FilesPipeline(MediaPipeline): + """Abstract pipeline that implement the file downloading + + This pipeline tries to minimize network transfers and file processing, + doing stat of the files and determining if file is new, uptodate or + expired. + + `new` files are those that pipeline never processed and needs to be + downloaded from supplier site the first time. + + `uptodate` files are the ones that the pipeline processed and are still + valid files. + + `expired` files are those that pipeline already processed but the last + modification was made long time ago, so a reprocessing is recommended to + refresh it in case of change. + + """ + + MEDIA_NAME = "file" + EXPIRES = 90 + STORE_SCHEMES = { + '': FSFilesStore, + 'file': FSFilesStore, + 's3': S3FilesStore, + } + + def __init__(self, store_uri, download_func=None): + if not store_uri: + raise NotConfigured + self.store = self._get_store(store_uri) + super(FilesPipeline, self).__init__(download_func=download_func) + + @classmethod + def from_settings(cls, settings): + s3store = cls.STORE_SCHEMES['s3'] + s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] + s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + + cls.EXPIRES = settings.getint('FILES_EXPIRES', 90) + store_uri = settings['FILES_STORE'] + return cls(store_uri) + + def _get_store(self, uri): + if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir + scheme = 'file' + else: + scheme = urlparse.urlparse(uri).scheme + store_cls = self.STORE_SCHEMES[scheme] + return store_cls(uri) + + def media_to_download(self, request, info): + def _onsuccess(result): + if not result: + return # returning None force download + + last_modified = result.get('last_modified', None) + if not last_modified: + return # returning None force download + + age_seconds = time.time() - last_modified + age_days = age_seconds / 60 / 60 / 24 + if age_days > self.EXPIRES: + return # returning None force download + + referer = request.headers.get('Referer') + log.msg(format='File (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', + level=log.DEBUG, spider=info.spider, + medianame=self.MEDIA_NAME, request=request, referer=referer) + self.inc_stats(info.spider, 'uptodate') + + checksum = result.get('checksum', None) + return {'url': request.url, 'path': key, 'checksum': checksum} + + key = self.file_key(request.url) + dfd = defer.maybeDeferred(self.store.stat_image, key, info) + dfd.addCallbacks(_onsuccess, lambda _: None) + dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image') + return dfd + + def media_failed(self, failure, request, info): + if not isinstance(failure.value, IgnoreRequest): + referer = request.headers.get('Referer') + log.msg(format='File (unknown-error): Error downloading ' + '%(medianame)s from %(request)s referred in ' + '<%(referer)s>: %(exception)s', + level=log.WARNING, spider=info.spider, exception=failure.value, + medianame=self.MEDIA_NAME, request=request, referer=referer) + + raise FileException + + def media_downloaded(self, response, request, info): + referer = request.headers.get('Referer') + + if response.status != 200: + log.msg(format='File (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>', + level=log.WARNING, spider=info.spider, + status=response.status, request=request, referer=referer) + raise FileException('download-error') + + if not response.body: + log.msg(format='File (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content', + level=log.WARNING, spider=info.spider, + request=request, referer=referer) + raise FileException('empty-content') + + status = 'cached' if 'cached' in response.flags else 'downloaded' + log.msg(format='File (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>', + level=log.DEBUG, spider=info.spider, + status=status, request=request, referer=referer) + self.inc_stats(info.spider, status) + + try: + key = self.file_key(request.url) + checksum = self.process_downloaded_media(response, request, info) + except FileException as exc: + whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' + log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, + request=request, referer=referer, errormsg=str(exc)) + raise + except Exception as exc: + whyfmt = 'File (unknown-error): Error processing image from %(request)s referred in <%(referer)s>' + log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider) + raise FileException(str(exc)) + + return {'url': request.url, 'path': key, 'checksum': checksum} + + def inc_stats(self, spider, status): + spider.crawler.stats.inc_value('file_count', spider=spider) + spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) + + ### Overradiable Interface + def get_media_requests(self, item, info): + return [Request(x) for x in item.get('file_urls', [])] + + def file_key(self, url): + media_guid = hashlib.sha1(url).hexdigest() + media_ext = os.path.splitext(url)[1] + return 'full/%s%s' % (media_guid, media_ext) + + def process_downloaded_media(self, response, request, info): + key = self.file_key(request.url) + buf = StringIO(response.body) + self.store.persist_file(key, buf, info) + checksum = md5sum(buf) + return checksum + + def item_completed(self, results, item, info): + if 'files' in item.fields: + item['files'] = [x for ok, x in results if ok] + return item diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 4ccc8e282..f0d647e5e 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -4,155 +4,35 @@ Images Pipeline See documentation in topics/images.rst """ -import os -import time import hashlib -import urlparse -import rfc822 from cStringIO import StringIO -from collections import defaultdict -from twisted.internet import defer, threads from PIL import Image -from scrapy import log from scrapy.utils.misc import md5sum from scrapy.http import Request -from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest -from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.exceptions import DropItem +#TODO: from scrapy.contrib.pipeline.media import MediaPipeline +from scrapy.contrib.pipeline.files import FileException, FilesPipeline class NoimagesDrop(DropItem): """Product with no images exception""" -class ImageException(Exception): +class ImageException(FileException): """General image error exception""" -class FSImagesStore(object): - - def __init__(self, basedir): - if '://' in basedir: - basedir = basedir.split('://', 1)[1] - self.basedir = basedir - self._mkdir(self.basedir) - self.created_directories = defaultdict(set) - - def persist_image(self, key, image, buf, info): - absolute_path = self._get_filesystem_path(key) - self._mkdir(os.path.dirname(absolute_path), info) - image.save(absolute_path) - - def stat_image(self, key, info): - absolute_path = self._get_filesystem_path(key) - try: - last_modified = os.path.getmtime(absolute_path) - except: # FIXME: catching everything! - return {} - - with open(absolute_path, 'rb') as imagefile: - checksum = md5sum(imagefile) - - return {'last_modified': last_modified, 'checksum': checksum} - - def _get_filesystem_path(self, key): - path_comps = key.split('/') - return os.path.join(self.basedir, *path_comps) - - def _mkdir(self, dirname, domain=None): - seen = self.created_directories[domain] if domain else set() - if dirname not in seen: - if not os.path.exists(dirname): - os.makedirs(dirname) - seen.add(dirname) - - -class S3ImagesStore(object): - - AWS_ACCESS_KEY_ID = None - AWS_SECRET_ACCESS_KEY = None - - POLICY = 'public-read' - HEADERS = { - 'Cache-Control': 'max-age=172800', - 'Content-Type': 'image/jpeg', - } - - def __init__(self, uri): - assert uri.startswith('s3://') - self.bucket, self.prefix = uri[5:].split('/', 1) - - def stat_image(self, key, info): - def _onsuccess(boto_key): - checksum = boto_key.etag.strip('"') - last_modified = boto_key.last_modified - modified_tuple = rfc822.parsedate_tz(last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - return {'checksum': checksum, 'last_modified': modified_stamp} - - return self._get_boto_key(key).addCallback(_onsuccess) - - def _get_boto_bucket(self): - from boto.s3.connection import S3Connection - # disable ssl (is_secure=False) because of this python bug: - # http://bugs.python.org/issue5103 - c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) - return c.get_bucket(self.bucket, validate=False) - - def _get_boto_key(self, key): - b = self._get_boto_bucket() - key_name = '%s%s' % (self.prefix, key) - return threads.deferToThread(b.get_key, key_name) - - def persist_image(self, key, image, buf, info): - """Upload image to S3 storage""" - width, height = image.size - b = self._get_boto_bucket() - key_name = '%s%s' % (self.prefix, key) - k = b.new_key(key_name) - k.set_metadata('width', str(width)) - k.set_metadata('height', str(height)) - buf.seek(0) - return threads.deferToThread(k.set_contents_from_file, buf, - headers=self.HEADERS, policy=self.POLICY) - - -class ImagesPipeline(MediaPipeline): - """Abstract pipeline that implement the image downloading and thumbnail generation logic - - This pipeline tries to minimize network transfers and image processing, - doing stat of the images and determining if image is new, uptodate or - expired. - - `new` images are those that pipeline never processed and needs to be - downloaded from supplier site the first time. - - `uptodate` images are the ones that the pipeline processed and are still - valid images. - - `expired` images are those that pipeline already processed but the last - modification was made long time ago, so a reprocessing is recommended to - refresh it in case of change. +class ImagesPipeline(FilesPipeline): + """Abstract pipeline that implement the image thumbnail generation logic """ MEDIA_NAME = 'image' MIN_WIDTH = 0 MIN_HEIGHT = 0 - EXPIRES = 90 THUMBS = {} - STORE_SCHEMES = { - '': FSImagesStore, - 'file': FSImagesStore, - 's3': S3ImagesStore, - } - - def __init__(self, store_uri, download_func=None): - if not store_uri: - raise NotConfigured - self.store = self._get_store(store_uri) - super(ImagesPipeline, self).__init__(download_func=download_func) @classmethod def from_settings(cls, settings): @@ -166,101 +46,21 @@ class ImagesPipeline(MediaPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri) - def _get_store(self, uri): - if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir - scheme = 'file' - else: - scheme = urlparse.urlparse(uri).scheme - store_cls = self.STORE_SCHEMES[scheme] - return store_cls(uri) - - def media_downloaded(self, response, request, info): - referer = request.headers.get('Referer') - - if response.status != 200: - log.msg(format='Image (code: %(status)s): Error downloading image from %(request)s referred in <%(referer)s>', - level=log.WARNING, spider=info.spider, - status=response.status, request=request, referer=referer) - raise ImageException('download-error') - - if not response.body: - log.msg(format='Image (empty-content): Empty image from %(request)s referred in <%(referer)s>: no-content', - level=log.WARNING, spider=info.spider, - request=request, referer=referer) - raise ImageException('empty-content') - - status = 'cached' if 'cached' in response.flags else 'downloaded' - log.msg(format='Image (%(status)s): Downloaded image from %(request)s referred in <%(referer)s>', - level=log.DEBUG, spider=info.spider, - status=status, request=request, referer=referer) - self.inc_stats(info.spider, status) - - try: - key = self.image_key(request.url) - checksum = self.image_downloaded(response, request, info) - except ImageException as exc: - whyfmt = 'Image (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' - log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, - request=request, referer=referer, errormsg=str(exc)) - raise - except Exception as exc: - whyfmt = 'Image (unknown-error): Error processing image from %(request)s referred in <%(referer)s>' - log.err(None, whyfmt % {'request': request, 'referer': referer}, spider=info.spider) - raise ImageException(str(exc)) - - return {'url': request.url, 'path': key, 'checksum': checksum} - - def media_failed(self, failure, request, info): - if not isinstance(failure.value, IgnoreRequest): - referer = request.headers.get('Referer') - log.msg(format='Image (unknown-error): Error downloading ' - '%(medianame)s from %(request)s referred in ' - '<%(referer)s>: %(exception)s', - level=log.WARNING, spider=info.spider, exception=failure.value, - medianame=self.MEDIA_NAME, request=request, referer=referer) - - raise ImageException - - def media_to_download(self, request, info): - def _onsuccess(result): - if not result: - return # returning None force download - - last_modified = result.get('last_modified', None) - if not last_modified: - return # returning None force download - - age_seconds = time.time() - last_modified - age_days = age_seconds / 60 / 60 / 24 - if age_days > self.EXPIRES: - return # returning None force download - - referer = request.headers.get('Referer') - log.msg(format='Image (uptodate): Downloaded %(medianame)s from %(request)s referred in <%(referer)s>', - level=log.DEBUG, spider=info.spider, - medianame=self.MEDIA_NAME, request=request, referer=referer) - self.inc_stats(info.spider, 'uptodate') - - checksum = result.get('checksum', None) - return {'url': request.url, 'path': key, 'checksum': checksum} - - key = self.image_key(request.url) - dfd = defer.maybeDeferred(self.store.stat_image, key, info) - dfd.addCallbacks(_onsuccess, lambda _: None) - dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image') - return dfd - - def image_downloaded(self, response, request, info): + def process_downloaded_media(self, response, request, info): checksum = None for key, image, buf in self.get_images(response, request, info): if checksum is None: buf.seek(0) checksum = md5sum(buf) - self.store.persist_image(key, image, buf, info) + width, height = image.size + self.store.persist_image( + key, buf, info, + meta={'width': width, 'height': height}, + headers={'Content-Type': 'image/jpeg'}) return checksum def get_images(self, response, request, info): - key = self.image_key(request.url) + key = self.file_key(request.url) orig_image = Image.open(StringIO(response.body)) width, height = orig_image.size @@ -276,10 +76,6 @@ class ImagesPipeline(MediaPipeline): thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_key, thumb_image, thumb_buf - def inc_stats(self, spider, status): - spider.crawler.stats.inc_value('image_count', spider=spider) - spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider) - def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': background = Image.new('RGBA', image.size, (255, 255, 255)) @@ -296,10 +92,6 @@ class ImagesPipeline(MediaPipeline): image.save(buf, 'JPEG') return image, buf - def image_key(self, url): - image_guid = hashlib.sha1(url).hexdigest() - return 'full/%s.jpg' % (image_guid) - def thumb_key(self, url, thumb_id): image_guid = hashlib.sha1(url).hexdigest() return 'thumbs/%s/%s.jpg' % (thumb_id, image_guid) @@ -307,6 +99,10 @@ class ImagesPipeline(MediaPipeline): def get_media_requests(self, item, info): return [Request(x) for x in item.get('image_urls', [])] + def file_key(self, url): + media_guid = hashlib.sha1(url).hexdigest() + return 'full/%s.jpg' % (media_guid) + def item_completed(self, results, item, info): if 'images' in item.fields: item['images'] = [x for ok, x in results if ok] diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 03871383e..74acac9b3 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -16,6 +16,7 @@ else: if not encoders.issubset(set(Image.core.__dict__)): skip = 'Missing JPEG encoders' + def _mocked_download_func(request, info): response = request.meta.get('response') return response() if callable(response) else response @@ -34,7 +35,7 @@ class ImagesPipelineTestCase(unittest.TestCase): rmtree(self.tempdir) def test_image_path(self): - image_path = self.pipeline.image_key + image_path = self.pipeline.file_key self.assertEqual(image_path("https://dev.mydeco.com/mydeco.gif"), 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.jpg"), @@ -61,8 +62,8 @@ class ImagesPipelineTestCase(unittest.TestCase): 'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg') def test_fs_store(self): - from scrapy.contrib.pipeline.images import FSImagesStore - assert isinstance(self.pipeline.store, FSImagesStore) + from scrapy.contrib.pipeline.files import FSFilesStore + assert isinstance(self.pipeline.store, FSFilesStore) self.assertEqual(self.pipeline.store.basedir, self.tempdir) key = 'some/image/key.jpg' @@ -91,7 +92,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) - def _create_image(format, *a, **kw): buf = StringIO() Image.new(*a, **kw).save(buf, format) From 45ff6ec28ac44114b1c8b98e151ff3c21cd9b994 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Fri, 16 Aug 2013 18:47:56 +0200 Subject: [PATCH 38/62] Test reorganization and new tests for Files and Images Pipelines, PEP8 changes in MediaPipeline --- scrapy/contrib/pipeline/media.py | 5 +- scrapy/tests/test_pipeline_files.py | 108 +++++++++++++++++++++++++++ scrapy/tests/test_pipeline_images.py | 9 --- 3 files changed, 111 insertions(+), 11 deletions(-) create mode 100644 scrapy/tests/test_pipeline_files.py diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 9491836d2..42217fd96 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -7,6 +7,7 @@ from scrapy import log from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter + class MediaPipeline(object): LOG_FAILED_RESULTS = True @@ -65,7 +66,7 @@ class MediaPipeline(object): dfd.addCallback(self._check_media_to_download, request, info) dfd.addBoth(self._cache_result_and_execute_waiters, fp, info) dfd.addErrback(log.err, spider=info.spider) - return dfd.addBoth(lambda _: wad) # it must return wad at last + return dfd.addBoth(lambda _: wad) # it must return wad at last def _check_media_to_download(self, result, request, info): if result is not None: @@ -91,7 +92,7 @@ class MediaPipeline(object): result.frames = [] result.stack = None info.downloading.remove(fp) - info.downloaded[fp] = result # cache result + info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): defer_result(result).chainDeferred(wad) diff --git a/scrapy/tests/test_pipeline_files.py b/scrapy/tests/test_pipeline_files.py new file mode 100644 index 000000000..c1b59c0f0 --- /dev/null +++ b/scrapy/tests/test_pipeline_files.py @@ -0,0 +1,108 @@ +import mock +import os +import time +from tempfile import mkdtemp +from shutil import rmtree + +from twisted.trial import unittest +from twisted.internet import defer + +from scrapy.contrib.pipeline.files import FilesPipeline, FSFilesStore +from scrapy.item import Item, Field +from scrapy.http import Request, Response + + +def _mocked_download_func(request, info): + response = request.meta.get('response') + return response() if callable(response) else response + + +class FilesPipelineTestCase(unittest.TestCase): + + def setUp(self): + self.tempdir = mkdtemp() + self.pipeline = FilesPipeline(self.tempdir, download_func=_mocked_download_func) + self.pipeline.open_spider(None) + + def tearDown(self): + rmtree(self.tempdir) + + def test_file_path(self): + image_path = self.pipeline.file_key + self.assertEqual(image_path("https://dev.mydeco.com/mydeco.pdf"), + 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') + self.assertEqual(image_path("http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"), + 'full/4ce274dd83db0368bafd7e406f382ae088e39219.txt') + self.assertEqual(image_path("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc"), + 'full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc') + self.assertEqual(image_path("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg"), + 'full/4507be485f38b0da8a0be9eb2e1dfab8a19223f2.jpg') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532/"), + 'full/97ee6f8a46cbbb418ea91502fd24176865cf39b2') + self.assertEqual(image_path("http://www.dorma.co.uk/images/product_details/2532"), + 'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1') + + def test_fs_store(self): + assert isinstance(self.pipeline.store, FSFilesStore) + self.assertEqual(self.pipeline.store.basedir, self.tempdir) + + key = 'some/image/key.jpg' + path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') + self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) + + @defer.inlineCallbacks + def test_file_not_expired(self): + item_url = "http://example.com/file.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), + mock.patch.object(FSFilesStore, 'stat_image', return_value={ + 'checksum': 'abc', 'last_modified': time.time()}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + @defer.inlineCallbacks + def test_file_expired(self): + item_url = "http://example.com/file2.pdf" + item = _create_item_with_files(item_url) + patchers = [ + mock.patch.object(FSFilesStore, 'stat_image', return_value={ + 'checksum': 'abc', + 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}), + mock.patch.object(FilesPipeline, 'get_media_requests', + return_value=[_prepare_request_object(item_url)]), + mock.patch.object(FilesPipeline, 'inc_stats', return_value=True) + ] + map(lambda p: p.start(), patchers) + + result = yield self.pipeline.process_item(item, None) + self.assertNotEqual(result['files'][0]['checksum'], 'abc') + + map(lambda p: p.stop(), patchers) + + +class ItemWithFiles(Item): + file_urls = Field() + files = Field() + + +def _create_item_with_files(*files): + item = ItemWithFiles() + item['file_urls'] = files + return item + + +def _prepare_request_object(item_url): + return Request( + item_url, + meta={'response': Response(item_url, status=200, body='data')}) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 74acac9b3..692e91afa 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -61,15 +61,6 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEqual(thumbnail_name("/tmp/some.name/foo", name), 'thumbs/50/92dac2a6a2072c5695a5dff1f865b3cb70c657bb.jpg') - def test_fs_store(self): - from scrapy.contrib.pipeline.files import FSFilesStore - assert isinstance(self.pipeline.store, FSFilesStore) - self.assertEqual(self.pipeline.store.basedir, self.tempdir) - - key = 'some/image/key.jpg' - path = os.path.join(self.tempdir, 'some', 'image', 'key.jpg') - self.assertEqual(self.pipeline.store._get_filesystem_path(key), path) - def test_convert_image(self): SIZE = (100, 100) # straigh forward case: RGB and JPEG From 034ffae60f97921677e7529db40b589d8cb2d542 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sun, 18 Aug 2013 00:44:01 +0600 Subject: [PATCH 39/62] Recommend Pillow instead of PIL. Closes GH-317. --- docs/topics/images.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/topics/images.rst b/docs/topics/images.rst index abbef96c4..9c1de4dd0 100644 --- a/docs/topics/images.rst +++ b/docs/topics/images.rst @@ -24,10 +24,13 @@ being scheduled for download, and connects those items that arrive containing the same image, to that queue. This avoids downloading the same image more than once when it's shared by several items. -The `Python Imaging Library`_ is used for thumbnailing and normalizing images -to JPEG/RGB format, so you need to install that library in order to use the -images pipeline. +`Pillow`_ is used for thumbnailing and normalizing images to JPEG/RGB format, +so you need to install this library in order to use the images pipeline. +`Python Imaging Library`_ (PIL) should also work in most cases, but it +is known to cause troubles in some setups, so we recommend to use `Pillow`_ +instead of `PIL `_. +.. _Pillow: https://github.com/python-imaging/Pillow .. _Python Imaging Library: http://www.pythonware.com/products/pil/ Using the Images Pipeline @@ -107,7 +110,7 @@ File system storage ------------------- The images are stored in files (one per image), using a `SHA1 hash`_ of their -URLs for the file names. +URLs for the file names. For example, the following image URL:: @@ -167,7 +170,7 @@ When you use this feature, the Images Pipeline will create thumbnails of the each specified size with this format:: /thumbs//.jpg - + Where: * ```` is the one specified in the :setting:`IMAGES_THUMBS` From 96077ccf48a2a434f245db870eb4cf0b74ed278f Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Tue, 20 Aug 2013 11:29:55 +0200 Subject: [PATCH 40/62] typo --- scrapy/contrib/pipeline/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index f0d647e5e..a899fe53e 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -53,7 +53,7 @@ class ImagesPipeline(FilesPipeline): buf.seek(0) checksum = md5sum(buf) width, height = image.size - self.store.persist_image( + self.store.persist_file( key, buf, info, meta={'width': width, 'height': height}, headers={'Content-Type': 'image/jpeg'}) From a6e6ca06b92d5f8d1b6ba7640218a770444e5863 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 21 Aug 2013 06:05:40 +0600 Subject: [PATCH 41/62] fix XmlItemExporter in Python 2.7.4 and 2.7.5 --- scrapy/contrib/exporter/__init__.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index a841eda2f..584786a87 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -3,6 +3,7 @@ Item Exporters are used to export/serialize items into different formats. """ import csv +import sys import pprint import marshal import json @@ -141,9 +142,23 @@ class XmlItemExporter(BaseItemExporter): for value in serialized_value: self._export_xml_field('value', value) else: - self.xg.characters(serialized_value) + self._xg_characters(serialized_value) self.xg.endElement(name) + # Workaround for http://bugs.python.org/issue17606 + # Before Python 2.7.4 xml.sax.saxutils required bytes; + # since 2.7.4 it requires unicode. The bug is likely to be + # fixed in 2.7.6, but 2.7.6 will still support unicode, + # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. + if sys.version_info[:3] >= (2, 7, 4): + def _xg_characters(self, serialized_value): + if not isinstance(serialized_value, unicode): + serialized_value = serialized_value.decode(self.encoding) + return self.xg.characters(serialized_value) + else: + def _xg_characters(self, serialized_value): + return self.xg.characters(serialized_value) + class CsvItemExporter(BaseItemExporter): From 071172cbd615c7a48733ab6423cbca92b5ff8d08 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 21 Aug 2013 14:32:22 -0300 Subject: [PATCH 42/62] fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 --- scrapy/contrib/downloadermiddleware/retry.py | 3 ++- scrapy/tests/mockserver.py | 2 +- scrapy/tests/test_crawl.py | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index 9f41197c4..fdb6fb462 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -22,6 +22,7 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost, TCPTimedOutError from twisted.internet.defer import TimeoutError as UserTimeoutError +from scrapy.xlib.tx._newclient import ResponseFailed from scrapy import log from scrapy.exceptions import NotConfigured @@ -33,7 +34,7 @@ class RetryMiddleware(object): # decompress an empty response EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, - ConnectionLost, TCPTimedOutError, + ConnectionLost, TCPTimedOutError, ResponseFailed, IOError) def __init__(self, settings): diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 4dde6c2a9..1347a7eae 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -102,7 +102,7 @@ class Drop(Partial): def _delayedRender(self, request): request.write("this connection will be dropped\n") - request.channel.transport.loseConnection() + request.channel.transport.abortConnection() request.finish() diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index dcc186c2d..818226ce3 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -90,6 +90,12 @@ class CrawlTestCase(TestCase): yield docrawl(spider) self._assert_retried() + @defer.inlineCallbacks + def test_retry_dropped_connection(self): + spider = SimpleSpider("http://localhost:8998/drop") + yield docrawl(spider) + self._assert_retried() + def _assert_retried(self): log = get_testlog() self.assertEqual(log.count("Retrying"), 2) From 3c64a989ca1da3a666155966f2a499fcc5d27397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 21 Aug 2013 15:17:40 -0300 Subject: [PATCH 43/62] New HTTP client wraps connection losts in ResponseFailed exception. fix #373 --- scrapy/contrib/downloadermiddleware/retry.py | 9 +++++---- scrapy/core/downloader/handlers/http11.py | 10 ++++------ scrapy/tests/mockserver.py | 8 ++++++-- scrapy/tests/test_crawl.py | 12 ++++++++++-- scrapy/tests/test_downloadermiddleware_retry.py | 14 +++++++++----- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index fdb6fb462..7eba3f78d 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -18,15 +18,16 @@ About HTTP errors to consider: indicate server overload, which would be something we want to retry """ -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError from twisted.internet.defer import TimeoutError as UserTimeoutError -from scrapy.xlib.tx._newclient import ResponseFailed +from twisted.internet.error import TimeoutError as ServerTimeoutError, \ + DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost, TCPTimedOutError from scrapy import log from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message +from scrapy.xlib.tx import ResponseFailed + class RetryMiddleware(object): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6ec10f723..c74581991 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,17 +7,16 @@ from urlparse import urldefrag from zope.interface import implements from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders -from twisted.web.http import PotentialDataLoss from twisted.web.iweb import IBodyProducer +from twisted.web.http import PotentialDataLoss from twisted.internet.error import TimeoutError from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - ResponseFailed, HTTPConnectionPool, TCP4ClientEndpoint + HTTPConnectionPool, TCP4ClientEndpoint, ResponseFailed from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.utils.misc import load_object -from scrapy import log class HTTP11DownloadHandler(object): @@ -55,7 +54,7 @@ class ScrapyAgent(object): if proxy: scheme, _, host, port, _ = _parse(proxy) endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout, - bindAddress=bindaddress) + bindAddress=bindaddress) return self._ProxyAgent(endpoint) return self._Agent(reactor, contextFactory=self._contextFactory, @@ -145,10 +144,9 @@ class _ResponseReader(protocol.Protocol): def connectionLost(self, reason): if self._finished.called: return + body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) - elif reason.check(PotentialDataLoss, ResponseFailed): - self._finished.callback((self._txresponse, body, ['partial'])) else: self._finished.errback(reason) diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 1347a7eae..e91fd1194 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -101,9 +101,13 @@ class Partial(DeferMixin, Resource): class Drop(Partial): def _delayedRender(self, request): + abort = getarg(request, "abort", 0, type=int) request.write("this connection will be dropped\n") - request.channel.transport.abortConnection() - request.finish() + if abort: + request.channel.transport.abortConnection() + else: + request.channel.transport.loseConnection() + request.finish() class Root(Resource): diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 818226ce3..7e5a80512 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -91,8 +91,16 @@ class CrawlTestCase(TestCase): self._assert_retried() @defer.inlineCallbacks - def test_retry_dropped_connection(self): - spider = SimpleSpider("http://localhost:8998/drop") + def test_retry_conn_lost(self): + # connection lost after receiving data + spider = SimpleSpider("http://localhost:8998/drop?abort=0") + yield docrawl(spider) + self._assert_retried() + + @defer.inlineCallbacks + def test_retry_conn_aborted(self): + # connection lost before receiving data + spider = SimpleSpider("http://localhost:8998/drop?abort=1") yield docrawl(spider) self._assert_retried() diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index d9bf1a543..c398afdba 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -1,14 +1,16 @@ import unittest -from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost +from twisted.internet.error import TimeoutError as ServerTimeoutError, \ + DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware +from scrapy.xlib.tx import ResponseFailed from scrapy.spider import BaseSpider from scrapy.http import Request, Response from scrapy.utils.test import get_crawler + class RetryTest(unittest.TestCase): def setUp(self): crawler = get_crawler() @@ -62,9 +64,11 @@ class RetryTest(unittest.TestCase): assert self.mw.process_response(req, rsp, self.spider) is rsp def test_twistederrors(self): - for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost): + for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, + ConnectionDone, ConnectError, ConnectionLost, + ResponseFailed): req = Request('http://www.scrapytest.org/%s' % exc.__name__) - self._test_retry_exception(req, exc()) + self._test_retry_exception(req, exc('foo')) def _test_retry_exception(self, req, exception): # first retry From efb4f32d802589bd8540452da197704b0dead8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 21 Aug 2013 19:04:12 -0300 Subject: [PATCH 44/62] do no include ResponseFailed if http11 handler is not enabled --- scrapy/tests/test_downloadermiddleware_retry.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index c398afdba..c5732dc4d 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -1,9 +1,9 @@ import unittest - from twisted.internet.error import TimeoutError as ServerTimeoutError, \ DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, \ ConnectionLost +from scrapy import optional_features from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware from scrapy.xlib.tx import ResponseFailed from scrapy.spider import BaseSpider @@ -64,9 +64,13 @@ class RetryTest(unittest.TestCase): assert self.mw.process_response(req, rsp, self.spider) is rsp def test_twistederrors(self): - for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, - ConnectionDone, ConnectError, ConnectionLost, - ResponseFailed): + exceptions = [ServerTimeoutError, DNSLookupError, + ConnectionRefusedError, ConnectionDone, ConnectError, + ConnectionLost] + if 'http11' in optional_features: + exceptions.append(ResponseFailed) + + for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) self._test_retry_exception(req, exc('foo')) From 4c89d8b2984667574dd42b5aa5b74625c06f9f17 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Thu, 22 Aug 2013 12:26:56 +0200 Subject: [PATCH 45/62] remarks --- scrapy/contrib/pipeline/files.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 5079040b6..49d266032 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -36,18 +36,18 @@ class FSFilesStore(object): def persist_file(self, key, buf, info, meta=None, headers=None): absolute_path = self._get_filesystem_path(key) self._mkdir(os.path.dirname(absolute_path), info) - with open(absolute_path, 'w') as f: + with open(absolute_path, 'wb') as f: f.write(buf.getvalue()) - def stat_image(self, key, info): + def stat_file(self, key, info): absolute_path = self._get_filesystem_path(key) try: last_modified = os.path.getmtime(absolute_path) except: # FIXME: catching everything! return {} - with open(absolute_path, 'rb') as imagefile: - checksum = md5sum(imagefile) + with open(absolute_path, 'rb') as f: + checksum = md5sum(f) return {'last_modified': last_modified, 'checksum': checksum} @@ -77,7 +77,7 @@ class S3FilesStore(object): assert uri.startswith('s3://') self.bucket, self.prefix = uri[5:].split('/', 1) - def stat_image(self, key, info): + def stat_file(self, key, info): def _onsuccess(boto_key): checksum = boto_key.etag.strip('"') last_modified = boto_key.last_modified @@ -190,9 +190,9 @@ class FilesPipeline(MediaPipeline): return {'url': request.url, 'path': key, 'checksum': checksum} key = self.file_key(request.url) - dfd = defer.maybeDeferred(self.store.stat_image, key, info) + dfd = defer.maybeDeferred(self.store.stat_file, key, info) dfd.addCallbacks(_onsuccess, lambda _: None) - dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_image') + dfd.addErrback(log.err, self.__class__.__name__ + '.store.stat_file') return dfd def media_failed(self, failure, request, info): From 02d08722e75e06c2b91246dfa9ac22c8e9b01601 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Thu, 22 Aug 2013 14:33:28 +0200 Subject: [PATCH 46/62] typo --- scrapy/contrib/pipeline/files.py | 2 +- scrapy/contrib/pipeline/media.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 49d266032..101d45fea 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -246,7 +246,7 @@ class FilesPipeline(MediaPipeline): spider.crawler.stats.inc_value('file_count', spider=spider) spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider) - ### Overradiable Interface + ### Overridable Interface def get_media_requests(self, item, info): return [Request(x) for x in item.get('file_urls', [])] diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 42217fd96..21b09150a 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -96,7 +96,7 @@ class MediaPipeline(object): for wad in info.waiting.pop(fp): defer_result(result).chainDeferred(wad) - ### Overradiable Interface + ### Overridable Interface def media_to_download(self, request, info): """Check request before starting download""" pass From 2b11e7da939993668c798f1118e2fe010d783587 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Thu, 22 Aug 2013 14:44:04 +0200 Subject: [PATCH 47/62] backwards compatibility: image_key --- scrapy/contrib/pipeline/images.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index a899fe53e..e15ce1e7c 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -103,6 +103,9 @@ class ImagesPipeline(FilesPipeline): media_guid = hashlib.sha1(url).hexdigest() return 'full/%s.jpg' % (media_guid) + # backwards compatibility + image_key = file_key + def item_completed(self, results, item, info): if 'images' in item.fields: item['images'] = [x for ok, x in results if ok] From 83bd151c579c5b02abc9816cd10794223e916303 Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Thu, 22 Aug 2013 15:13:59 +0200 Subject: [PATCH 48/62] backwards compatibility: image_key & image_downloaded --- scrapy/contrib/pipeline/files.py | 4 ++-- scrapy/contrib/pipeline/images.py | 14 +++++++++----- scrapy/tests/test_pipeline_files.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 101d45fea..b38a4c21b 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -229,7 +229,7 @@ class FilesPipeline(MediaPipeline): try: key = self.file_key(request.url) - checksum = self.process_downloaded_media(response, request, info) + checksum = self.file_downloaded(response, request, info) except FileException as exc: whyfmt = 'File (error): Error processing image from %(request)s referred in <%(referer)s>: %(errormsg)s' log.msg(format=whyfmt, level=log.WARNING, spider=info.spider, @@ -255,7 +255,7 @@ class FilesPipeline(MediaPipeline): media_ext = os.path.splitext(url)[1] return 'full/%s%s' % (media_guid, media_ext) - def process_downloaded_media(self, response, request, info): + def file_downloaded(self, response, request, info): key = self.file_key(request.url) buf = StringIO(response.body) self.store.persist_file(key, buf, info) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index e15ce1e7c..aaad689a0 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -46,7 +46,13 @@ class ImagesPipeline(FilesPipeline): store_uri = settings['IMAGES_STORE'] return cls(store_uri) - def process_downloaded_media(self, response, request, info): + def file_key(self, url): + return self.image_key(url) + + def file_downloaded(self, response, request, info): + return self.image_downloaded(response, request, info) + + def image_downloaded(self, response, request, info): checksum = None for key, image, buf in self.get_images(response, request, info): if checksum is None: @@ -99,13 +105,11 @@ class ImagesPipeline(FilesPipeline): def get_media_requests(self, item, info): return [Request(x) for x in item.get('image_urls', [])] - def file_key(self, url): + # backwards compatibility + def image_key(self, url): media_guid = hashlib.sha1(url).hexdigest() return 'full/%s.jpg' % (media_guid) - # backwards compatibility - image_key = file_key - def item_completed(self, results, item, info): if 'images' in item.fields: item['images'] = [x for ok, x in results if ok] diff --git a/scrapy/tests/test_pipeline_files.py b/scrapy/tests/test_pipeline_files.py index c1b59c0f0..f0aadfbfa 100644 --- a/scrapy/tests/test_pipeline_files.py +++ b/scrapy/tests/test_pipeline_files.py @@ -56,7 +56,7 @@ class FilesPipelineTestCase(unittest.TestCase): item = _create_item_with_files(item_url) patchers = [ mock.patch.object(FilesPipeline, 'inc_stats', return_value=True), - mock.patch.object(FSFilesStore, 'stat_image', return_value={ + mock.patch.object(FSFilesStore, 'stat_file', return_value={ 'checksum': 'abc', 'last_modified': time.time()}), mock.patch.object(FilesPipeline, 'get_media_requests', return_value=[_prepare_request_object(item_url)]) @@ -73,7 +73,7 @@ class FilesPipelineTestCase(unittest.TestCase): item_url = "http://example.com/file2.pdf" item = _create_item_with_files(item_url) patchers = [ - mock.patch.object(FSFilesStore, 'stat_image', return_value={ + mock.patch.object(FSFilesStore, 'stat_file', return_value={ 'checksum': 'abc', 'last_modified': time.time() - (FilesPipeline.EXPIRES * 60 * 60 * 24 * 2)}), mock.patch.object(FilesPipeline, 'get_media_requests', From 86230c0ab807419159521de74b4d33d9a90c3bba Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 22 Aug 2013 21:49:18 -0300 Subject: [PATCH 49/62] added quantal & raring to support ubuntu releases --- docs/topics/ubuntu.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 5b8476119..14498f7f4 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -23,7 +23,15 @@ with command:: lsb_release -cs Supported Ubuntu releases are: ``karmic``, ``lucid``, ``maverick``, ``natty``, -``oneiric``, ``precise``. +``oneiric``, ``precise``, ``quantal``, ``raring``. + +For Ubuntu Raring (13.04):: + + deb http://archive.scrapy.org/ubuntu raring main + +For Ubuntu Quantal (12.10):: + + deb http://archive.scrapy.org/ubuntu quantal main For Ubuntu Precise (12.04):: From 19ff9ac4f925790b58b35536dea433797863df23 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 23 Aug 2013 12:43:22 -0300 Subject: [PATCH 50/62] url/body attributes of Request/Response objects are now immutable --- docs/news.rst | 6 ++++++ scrapy/http/common.py | 10 +++------- scrapy/http/request/__init__.py | 6 +++--- scrapy/http/response/__init__.py | 6 +++--- scrapy/tests/test_http_request.py | 5 +++++ scrapy/tests/test_http_response.py | 5 +++++ 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 04cfa6e8b..2ec7ea293 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,12 @@ Release notes ============= +0.20 (not released yet) +----------------------- + +- Request/Response url/body attributes are now immutable (modifying them had + been deprecated for a long time) + 0.18.0 (released 2013-08-09) ---------------------------- diff --git a/scrapy/http/common.py b/scrapy/http/common.py index 34d5389bf..ba6ab277c 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,10 +1,6 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning - -def deprecated_setter(setter, attrname): +def obsolete_setter(setter, attrname): def newsetter(self, value): c = self.__class__.__name__ - warnings.warn("Don't modify %s.%s attribute, use %s.replace() instead" % \ - (c, attrname, c), ScrapyDeprecationWarning, stacklevel=2) - return setter(self, value) + msg = "%s.%s is not modifiable, use %s.replace() instead" % (c, attrname, c) + raise AttributeError(msg) return newsetter diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1b14e2068..609a0d433 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -13,7 +13,7 @@ from scrapy.http.headers import Headers from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated from scrapy.utils.url import escape_ajax -from scrapy.http.common import deprecated_setter +from scrapy.http.common import obsolete_setter class Request(object_ref): @@ -60,7 +60,7 @@ class Request(object_ref): if ':' not in self._url: raise ValueError('Missing scheme in request url: %s' % self._url) - url = property(_get_url, deprecated_setter(_set_url, 'url')) + url = property(_get_url, obsolete_setter(_set_url, 'url')) def _get_body(self): return self._body @@ -78,7 +78,7 @@ class Request(object_ref): else: raise TypeError("Request body must either str or unicode. Got: '%s'" % type(body).__name__) - body = property(_get_body, deprecated_setter(_set_body, 'body')) + body = property(_get_body, obsolete_setter(_set_body, 'body')) @property def encoding(self): diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 46be1e362..7ff683eb6 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -9,7 +9,7 @@ import copy from scrapy.http.headers import Headers from scrapy.utils.trackref import object_ref -from scrapy.http.common import deprecated_setter +from scrapy.http.common import obsolete_setter class Response(object_ref): @@ -39,7 +39,7 @@ class Response(object_ref): raise TypeError('%s url must be str, got %s:' % (type(self).__name__, \ type(url).__name__)) - url = property(_get_url, deprecated_setter(_set_url, 'url')) + url = property(_get_url, obsolete_setter(_set_url, 'url')) def _get_body(self): return self._body @@ -56,7 +56,7 @@ class Response(object_ref): raise TypeError("Response body must either str or unicode. Got: '%s'" \ % type(body).__name__) - body = property(_get_body, deprecated_setter(_set_body, 'body')) + body = property(_get_body, obsolete_setter(_set_body, 'body')) def __str__(self): return "<%d %s>" % (self.status, self.url) diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 9d2f235cc..d3378148d 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -175,6 +175,11 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com", method=u"POST") assert isinstance(r.method, str) + def test_immutable_attributes(self): + r = self.request_class("http://example.com") + self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') + self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + class FormRequestTest(RequestTest): diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 26c66453a..0809340a1 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -107,6 +107,11 @@ class BaseResponseTest(unittest.TestCase): def _assert_response_encoding(self, response, encoding): self.assertEqual(response.encoding, resolve_encoding(encoding)) + def test_immutable_attributes(self): + r = self.response_class("http://example.com") + self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') + self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): From da1f6d31440dfacd7ef36c6213286a1b2a181b0d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 23 Aug 2013 13:03:28 -0300 Subject: [PATCH 51/62] remove unused imports and some assorted pylint-ing --- scrapy/__init__.py | 2 +- scrapy/commands/crawl.py | 2 +- scrapy/commands/shell.py | 1 - scrapy/contrib/exporter/__init__.py | 2 +- scrapy/contrib_exp/downloadermiddleware/decompression.py | 1 - scrapy/core/downloader/handlers/http11.py | 3 +-- scrapy/core/downloader/webclient.py | 2 -- scrapy/resolver.py | 2 -- scrapy/spider.py | 1 - scrapy/tests/test_http_request.py | 1 - scrapy/tests/test_utils_defer.py | 4 ++-- scrapy/tests/test_utils_misc/__init__.py | 4 ++-- 12 files changed, 8 insertions(+), 17 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 17bf64202..e32618ebb 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,7 +7,7 @@ version_info = tuple(__version__.split('.')[:3]) import sys, os, warnings -if sys.version_info < (2,6): +if sys.version_info < (2, 6): print "Scrapy %s requires Python 2.6 or above" % __version__ sys.exit(1) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index e1167cbc0..75e8e7720 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -34,7 +34,7 @@ class Command(ScrapyCommand): self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats)) + raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index fe45e7dba..7209ae16a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -8,7 +8,6 @@ from threading import Thread from scrapy.command import ScrapyCommand from scrapy.shell import Shell -from scrapy import log class Command(ScrapyCommand): diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index 584786a87..a20c92b54 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -198,7 +198,7 @@ class PickleItemExporter(BaseItemExporter): def __init__(self, file, protocol=2, **kwargs): self._configure(kwargs) - self.file =file + self.file = file self.protocol = protocol def export_item(self, item): diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index d18d51eb5..d67794d67 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -10,7 +10,6 @@ from cStringIO import StringIO from tempfile import mktemp from scrapy import log -from scrapy.http import Response from scrapy.responsetypes import responsetypes diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c74581991..6690bbe96 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,10 +8,9 @@ from zope.interface import implements from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer -from twisted.web.http import PotentialDataLoss from twisted.internet.error import TimeoutError from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, TCP4ClientEndpoint, ResponseFailed + HTTPConnectionPool, TCP4ClientEndpoint from scrapy.http import Headers from scrapy.responsetypes import responsetypes diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ea69dc5ca..3b239ddec 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,11 +1,9 @@ from time import time from urlparse import urlparse, urlunparse, urldefrag -from twisted.internet.ssl import ClientContextFactory from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient from twisted.internet import defer -from OpenSSL import SSL from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 6ba89ac94..7d9811727 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,5 +1,3 @@ -import socket - from twisted.internet import defer from twisted.internet.base import ThreadedResolver diff --git a/scrapy/spider.py b/scrapy/spider.py index 36e91d636..25a59036d 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -6,7 +6,6 @@ See documentation in docs/topics/spiders.rst from scrapy import log from scrapy.http import Request -from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index d3378148d..29358399e 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,7 +1,6 @@ import cgi import unittest import xmlrpclib -from cStringIO import StringIO from urlparse import urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse diff --git a/scrapy/tests/test_utils_defer.py b/scrapy/tests/test_utils_defer.py index 4473002dc..670116b34 100644 --- a/scrapy/tests/test_utils_defer.py +++ b/scrapy/tests/test_utils_defer.py @@ -14,7 +14,7 @@ class MustbeDeferredTest(unittest.TestCase): return steps dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred steps.append(2) # add another value, that should be catched by assertEqual return dfd @@ -27,7 +27,7 @@ class MustbeDeferredTest(unittest.TestCase): return dfd dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1,2]) # it is [1] with maybeDeferred + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred steps.append(2) # add another value, that should be catched by assertEqual return dfd diff --git a/scrapy/tests/test_utils_misc/__init__.py b/scrapy/tests/test_utils_misc/__init__.py index 916b5e25b..143c9b644 100644 --- a/scrapy/tests/test_utils_misc/__init__.py +++ b/scrapy/tests/test_utils_misc/__init__.py @@ -64,14 +64,14 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(None), '__iter__') assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') - assert hasattr(arg_to_iter([1,2,3]), '__iter__') + assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) - self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) + self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) From 6720acc1afdb10112a75c883da47ede36ced9731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 11:09:55 -0300 Subject: [PATCH 52/62] Treat responses without content-length or Transfer-Encoding as good responses There is not a way to determine if responses without Content-Length or Transfer-Encoding are complete, this change treat them as good responses but flags them as "partial". This is backout change only for this functionality of 3c64a989 --- scrapy/core/downloader/handlers/http11.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6690bbe96..bc25dbf28 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -147,5 +147,7 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) + elif reason.check(PotentialDataLoss): + self._finished.callback((self._txresponse, body, ['partial'])) else: self._finished.errback(reason) From caa0f90263af76119cfda23d468f902a0e877488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 11:53:18 -0300 Subject: [PATCH 53/62] test PotentiaDataLoss errors on unbound responses --- scrapy/core/downloader/handlers/http11.py | 1 + scrapy/tests/mockserver.py | 20 +++++++++++++++++ scrapy/tests/test_crawl.py | 27 +++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index bc25dbf28..04dd4952c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -9,6 +9,7 @@ from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer from twisted.internet.error import TimeoutError +from twisted.web.http import PotentialDataLoss from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ HTTPConnectionPool, TCP4ClientEndpoint diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index e91fd1194..56cbf3aef 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -20,6 +20,7 @@ class DeferMixin(object): # silence CancelledError d.addErrback(lambda _: None) d.cancel() + d = deferLater(reactor, delay, f, *a, **kw) request.notifyFinish().addErrback(_cancelrequest) return d @@ -84,6 +85,23 @@ class Status(Resource): return "" +class Raw(DeferMixin, Resource): + + isLeaf = True + + def render_GET(self, request): + request.startedWriting = 1 + self.deferRequest(request, 0, self._delayedRender, request) + return NOT_DONE_YET + + def _delayedRender(self, request): + raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n') + request.startedWriting = 1 + request.write(raw) + request.channel.transport.loseConnection() + request.finish() + + class Partial(DeferMixin, Resource): isLeaf = True @@ -92,6 +110,7 @@ class Partial(DeferMixin, Resource): request.setHeader("Content-Length", "1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET + render_POST = render_GET def _delayedRender(self, request): request.write("partial content\n") @@ -119,6 +138,7 @@ class Root(Resource): self.putChild("delay", Delay()) self.putChild("partial", Partial()) self.putChild("drop", Drop()) + self.putChild("raw", Raw()) def getChild(self, name, request): return self diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 7e5a80512..10219a602 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -90,6 +90,33 @@ class CrawlTestCase(TestCase): yield docrawl(spider) self._assert_retried() + @defer.inlineCallbacks + def test_unbounded_response(self): + # Completeness of responses without Content-Length or Transfer-Encoding + # can not be determined, we treat them as valid but flagged as "partial" + from urllib import urlencode + query = urlencode({'raw': '''\ +HTTP/1.1 200 OK +Server: Apache-Coyote/1.1 +X-Powered-By: Servlet 2.4; JBoss-4.2.3.GA (build: SVNTag=JBoss_4_2_3_GA date=200807181417)/JBossWeb-2.0 +Set-Cookie: JSESSIONID=08515F572832D0E659FD2B0D8031D75F; Path=/ +Pragma: no-cache +Expires: Thu, 01 Jan 1970 00:00:00 GMT +Cache-Control: no-cache +Cache-Control: no-store +Content-Type: text/html;charset=UTF-8 +Content-Language: en +Date: Tue, 27 Aug 2013 13:05:05 GMT +Connection: close + +foo body +with multiples lines +'''}) + spider = SimpleSpider("http://localhost:8998/raw?{}".format(query)) + yield docrawl(spider) + log = get_testlog() + self.assertEqual(log.count("Got response 200"), 1) + @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data From 92826586d5f2740598e7f8a71047decd44c531c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 12:12:50 -0300 Subject: [PATCH 54/62] py26 can not format zero length fields {} --- scrapy/tests/test_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index 10219a602..1878f7a1d 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -112,7 +112,7 @@ Connection: close foo body with multiples lines '''}) - spider = SimpleSpider("http://localhost:8998/raw?{}".format(query)) + spider = SimpleSpider("http://localhost:8998/raw?{0}".format(query)) yield docrawl(spider) log = get_testlog() self.assertEqual(log.count("Got response 200"), 1) From 588a262b73e73f319edede130882f2520350d968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 14:05:37 -0300 Subject: [PATCH 55/62] fix crawling tests under twisted pre 11.0.0 --- scrapy/core/downloader/webclient.py | 3 ++ scrapy/tests/mockserver.py | 63 ++++++++++++++++++----------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3b239ddec..135d8f293 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -58,12 +58,15 @@ class ScrapyHTTPPageGetter(HTTPClient): self.factory.gotHeaders(self.headers) def connectionLost(self, reason): + self._connection_lost_reason = reason HTTPClient.connectionLost(self, reason) self.factory.noPage(reason) def handleResponse(self, response): if self.factory.method.upper() == 'HEAD': self.factory.page('') + elif self.length is not None and self.length > 0: + self.factory.noPage(self._connection_lost_reason) else: self.factory.page(response) self.transport.loseConnection() diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 56cbf3aef..d165a6bdc 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -2,8 +2,29 @@ import sys, time, random, urllib from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource -from twisted.internet import reactor -from twisted.internet.task import deferLater +from twisted.internet import reactor, defer +from scrapy import twisted_version + + +if twisted_version < (11, 0, 0): + def deferLater(clock, delay, func, *args, **kw): + def _cancel_method(): + _cancel_cb(None) + d.errback(Exception()) + + def _cancel_cb(result): + if cl.active(): + cl.cancel() + return result + + d = defer.Deferred() + d.cancel = _cancel_method + d.addCallback(lambda ignored: func(*args, **kw)) + d.addBoth(_cancel_cb) + cl = clock.callLater(delay, d.callback, None) + return d +else: + from twisted.internet.task import deferLater def getarg(request, name, default=None, type=str): @@ -13,7 +34,9 @@ def getarg(request, name, default=None, type=str): return default -class DeferMixin(object): +class LeafResource(Resource): + + isLeaf = True def deferRequest(self, request, delay, f, *a, **kw): def _cancelrequest(_): @@ -26,9 +49,7 @@ class DeferMixin(object): return d -class Follow(DeferMixin, Resource): - - isLeaf = True +class Follow(LeafResource): def render(self, request): total = getarg(request, "total", 100, type=int) @@ -57,9 +78,7 @@ class Follow(DeferMixin, Resource): request.finish() -class Delay(DeferMixin, Resource): - - isLeaf = True +class Delay(LeafResource): def render_GET(self, request): n = getarg(request, "n", 1, type=float) @@ -75,9 +94,7 @@ class Delay(DeferMixin, Resource): request.finish() -class Status(Resource): - - isLeaf = True +class Status(LeafResource): def render_GET(self, request): n = getarg(request, "n", 200, type=int) @@ -85,14 +102,13 @@ class Status(Resource): return "" -class Raw(DeferMixin, Resource): - - isLeaf = True +class Raw(LeafResource): def render_GET(self, request): request.startedWriting = 1 self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET + render_POST = render_GET def _delayedRender(self, request): raw = getarg(request, 'raw', 'HTTP 1.1 200 OK\n') @@ -102,15 +118,12 @@ class Raw(DeferMixin, Resource): request.finish() -class Partial(DeferMixin, Resource): - - isLeaf = True +class Partial(LeafResource): def render_GET(self, request): request.setHeader("Content-Length", "1024") self.deferRequest(request, 0, self._delayedRender, request) return NOT_DONE_YET - render_POST = render_GET def _delayedRender(self, request): request.write("partial content\n") @@ -122,10 +135,13 @@ class Drop(Partial): def _delayedRender(self, request): abort = getarg(request, "abort", 0, type=int) request.write("this connection will be dropped\n") - if abort: - request.channel.transport.abortConnection() - else: - request.channel.transport.loseConnection() + tr = request.channel.transport + try: + if abort and hasattr(tr, 'abortConnection'): + tr.abortConnection() + else: + tr.loseConnection() + finally: request.finish() @@ -165,6 +181,7 @@ if __name__ == "__main__": root = Root() factory = Site(root) port = reactor.listenTCP(8998, factory) + def print_listening(): h = port.getHost() print "Mock server running at http://%s:%d" % (h.host, h.port) From 401e888ced2e8a0cc7091d2679b187fa572a1aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 15:30:10 -0300 Subject: [PATCH 56/62] test lucid and precise python packages under their respective python versions --- .travis.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10efe11ff..0ca1c2ed3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,25 @@ language: python python: - - 2.6 - - 2.7 + - "2.6" + - "2.7" env: - BUILDENV=lucid - BUILDENV=precise - BUILDENV=latest +matrix: + include: + - python: "2.6" + env: BUILDENV=lucid + - python: "2.7" + env: BUILDENV=precise + - python: "2.6" + env: BUILDENV=latest + - python: "2.7" + env: BUILDENV=latest + install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt - - pip install . + - pip install --use-mirrors . script: - trial scrapy notifications: From 7394969236ea10ee0d4bf6b290b29c7cfa4ecfbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 15:31:08 -0300 Subject: [PATCH 57/62] test pypy with travis --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0ca1c2ed3..31ff95072 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.6" - "2.7" + - "pypy" env: - BUILDENV=lucid - BUILDENV=precise @@ -16,6 +17,11 @@ matrix: env: BUILDENV=latest - python: "2.7" env: BUILDENV=latest + - python: "pypy" + env: BUILDENV=latest + allow_failures: + - python: "pypy" + env: BUILDENV=latest install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt From ad140a276355a18d85ae5bc0d2e175f40189967b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 16:53:54 -0300 Subject: [PATCH 58/62] another try to limit travis build matrix --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 31ff95072..1bc766a26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,4 @@ language: python -python: - - "2.6" - - "2.7" - - "pypy" -env: - - BUILDENV=lucid - - BUILDENV=precise - - BUILDENV=latest matrix: include: - python: "2.6" From c5222b4b8ab05867b343feca337c04fc0f88697f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 17:19:45 -0300 Subject: [PATCH 59/62] remove phanton build row from travis matrix due to travis-ci/travis-core#1027 --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1bc766a26..75c97fb14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: python +env: + TRAVISBUG="#1027" matrix: + exclude: + - env: TRAVISBUG="#1027" include: - python: "2.6" env: BUILDENV=lucid @@ -14,7 +18,6 @@ matrix: allow_failures: - python: "pypy" env: BUILDENV=latest - install: - pip install --use-mirrors -r .travis/requirements-$BUILDENV.txt - pip install --use-mirrors . From ac218ed4185c9d3a47f985c07d79c46918e7468e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 17:38:28 -0300 Subject: [PATCH 60/62] no need to test latest versions of dependencies on python 2.6 --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75c97fb14..e00a1596b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,6 @@ matrix: env: BUILDENV=lucid - python: "2.7" env: BUILDENV=precise - - python: "2.6" - env: BUILDENV=latest - python: "2.7" env: BUILDENV=latest - python: "pypy" From 0f00b1602a534ec40e89b3e37513930f8050d18b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 27 Aug 2013 18:45:21 -0300 Subject: [PATCH 61/62] merge 0.18 release notes --- docs/news.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 2ec7ea293..e3a7eda39 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,6 +9,25 @@ Release notes - Request/Response url/body attributes are now immutable (modifying them had been deprecated for a long time) +0.18.1 (released 2013-08-27) +---------------------------- + +- remove extra import added by cherry picked changes (:commit:`d20304e`) +- fix crawling tests under twisted pre 11.0.0 (:commit:`1994f38`) +- py26 can not format zero length fields {} (:commit:`abf756f`) +- test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`) +- Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`) +- do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`) +- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`) +- limit travis-ci build matrix (:commit:`3b01bb8`) +- Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`) +- Fixed so it refers to the correct folder (:commit:`3283809`) +- added quantal & raring to support ubuntu releases (:commit:`1411923`) +- fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`) +- fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`) +- minor updates to 0.18 release notes (:commit:`c45e5f1`) +- fix contributters list format (:commit:`0b60031`) + 0.18.0 (released 2013-08-09) ---------------------------- From 0a8bf2c9e4720d4276add52cc46fc601f9ce228a Mon Sep 17 00:00:00 2001 From: Lukasz Biedrycki Date: Wed, 28 Aug 2013 18:00:54 +0200 Subject: [PATCH 62/62] after additional tests with actual s3: k.set_contents_from_string is working --- scrapy/contrib/pipeline/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index b38a4c21b..f03bf69df 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -111,7 +111,7 @@ class S3FilesStore(object): if headers: h.update(headers) buf.seek(0) - return threads.deferToThread(k.set_contents_from_file, buf, + return threads.deferToThread(k.set_contents_from_string, buf.getvalue(), headers=h, policy=self.POLICY)