From 175a4b5957c82c814ef386f0e3276225ad86f3f4 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Thu, 1 Dec 2011 18:44:26 -0200 Subject: [PATCH 1/4] allow spider to set autothrottle max concurrency --- scrapy/contrib/throttle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py index a1afd2c5f..69edea1ca 100644 --- a/scrapy/contrib/throttle.py +++ b/scrapy/contrib/throttle.py @@ -76,6 +76,9 @@ class AutoThrottle(object): def spider_opened(self, spider): spider.download_delay = self.START_DELAY + if hasattr(spider, "max_concurrent_requests"): + self.MAX_CONCURRENCY = spider.max_concurrent_requests + # override in order to avoid to initialize slot with concurrency > 1 spider.max_concurrent_requests = 1 self.last_latencies = [self.START_DELAY] self.last_lat = self.START_DELAY, 0.0 From 98f3f87530034941a9f8313b2c2c2578f869ffa3 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Sun, 4 Dec 2011 23:42:41 -0400 Subject: [PATCH 2/4] Avoid _disconnectedDeferred AttributeError exception in Twisted>=11.1.0 --- scrapy/core/downloader/webclient.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 018e1d050..6579475a9 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -98,6 +98,14 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.start_time = time() self.deferred = defer.Deferred().addCallback(self._build_response, request) + # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected + # to have _disconnectedDeferred. See Twisted r32329. + # As Scrapy implements it's own logic to handle redirects is not + # needed to add the callback _waitForDisconnect. + # Specifically this avoids the AttributeError exception when + # clientConnectionFailed method is called. + self._disconnectedDeferred = defer.Deferred() + self._set_connection_attributes(request) # set Host header based on url From bcb31988f2acecfc5fefa7be0c3a44a7b31e1d77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 9 Nov 2011 09:50:59 -0200 Subject: [PATCH 3/4] change tutorial to follow changes on dmoz site --- docs/intro/tutorial.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 840515763..c395dbfa0 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -268,19 +268,19 @@ The shell also instantiates two selectors, one for HTML (in the ``hxs`` variable) and one for XML (in the ``xxs`` variable) with this response. So let's try them:: - In [1]: hxs.select('/html/head/title') - Out[1]: [] + In [1]: hxs.select('//title') + Out[1]: [] - In [2]: hxs.select('/html/head/title').extract() + In [2]: hxs.select('//title').extract() Out[2]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [3]: hxs.select('/html/head/title/text()') - Out[3]: [] + In [3]: hxs.select('//title/text()') + Out[3]: [] - In [4]: hxs.select('/html/head/title/text()').extract() + In [4]: hxs.select('//title/text()').extract() Out[4]: [u'Open Directory - Computers: Programming: Languages: Python: Books'] - In [5]: hxs.select('/html/head/title/text()').re('(\w+):') + In [5]: hxs.select('//title/text()').re('(\w+):') Out[5]: [u'Computers', u'Programming', u'Languages', u'Python'] Extracting the data From 63d583d9be12ae52dfcb60d960f4657c7241a1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 8 Dec 2011 13:54:16 -0200 Subject: [PATCH 4/4] SSL handshaking hangs when doing too many parallel connections to S3 --- scrapy/contrib/pipeline/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index c7333a542..aa0a7b0b4 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -102,7 +102,7 @@ class S3ImagesStore(object): def _get_boto_bucket(self): from boto.s3.connection import S3Connection - c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY) + c = S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) return c.get_bucket(self.bucket, validate=False) def _get_boto_key(self, key):