From dc8701ea429d4ded2f66d6b7c8fbce0bbcd0041a Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 12:56:42 -0200 Subject: [PATCH 1/8] Add test for already failed deferreds when downloading page in robots.txt middleware. --- tests/test_downloadermiddleware_robotstxt.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 5f45dcb82..f2e94e171 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -123,6 +123,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) return deferred + def test_robotstxt_immediate_error(self): + self.crawler.settings.set('ROBOTSTXT_OBEY', True) + err = error.DNSLookupError('Robotstxt address not found') + def immediate_failure(request, spider): + deferred = Deferred() + deferred.errback(failure.Failure(err)) + return deferred + self.crawler.engine.download.side_effect = immediate_failure + + middleware = RobotsTxtMiddleware(self.crawler) + return self.assertNotIgnored(Request('http://site.local'), middleware) + def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) def ignore_request(request, spider): From b2beb3e85d2e82977d259eea71402809d00d197e Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 13:09:08 -0200 Subject: [PATCH 2/8] Fix handling of already failed deferreds when downloading page in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c061c2407..7f6f0d012 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,7 +8,9 @@ import logging from six.moves.urllib import robotparser +from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred +from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -57,7 +59,13 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - dfd = self.crawler.engine.download(robotsreq, spider) + # engine.download() can return an already-called deferred, e.g. if a + # middleware returns a response in process_request(). Using + # deferLater() ensures that the error callback isn't called + # immediately upon being added, so that it doesn't remove the key + # before we check for it. + dfd = deferLater(reactor, 0, self.crawler.engine.download, + robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) From 90e3ae1c580875e4e68c9d7238d0fb4642306bf9 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 27 Jan 2016 21:00:35 -0200 Subject: [PATCH 3/8] Do not forget failed requests in robots.txt middleware. --- scrapy/downloadermiddlewares/robotstxt.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7f6f0d012..6fdba90cc 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -8,9 +8,7 @@ import logging from six.moves.urllib import robotparser -from twisted.internet import reactor from twisted.internet.defer import Deferred, maybeDeferred -from twisted.internet.task import deferLater from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -59,13 +57,7 @@ class RobotsTxtMiddleware(object): priority=self.DOWNLOAD_PRIORITY, meta={'dont_obey_robotstxt': True} ) - # engine.download() can return an already-called deferred, e.g. if a - # middleware returns a response in process_request(). Using - # deferLater() ensures that the error callback isn't called - # immediately upon being added, so that it doesn't remove the key - # before we check for it. - dfd = deferLater(reactor, 0, self.crawler.engine.download, - robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) @@ -109,4 +101,6 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): - self._parsers.pop(netloc).callback(None) + rp_dfd = self._parsers[netloc] + self._parsers[netloc] = None + rp_dfd.callback(None) From a1ebff83d39e65cf4ed34281a1e81ea6cd108fe0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 18:39:34 +0100 Subject: [PATCH 4/8] Remove __str__ and __repr__ from settings, introduce copy_to_dict() instead Settings instances as dict's are easier to print or pretty print in the shell Fixes #1732 --- scrapy/settings/__init__.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 342d2585e..918bfc1e5 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -368,11 +368,25 @@ class BaseSettings(MutableMapping): def __len__(self): return len(self.attributes) - def __str__(self): - return str(self.attributes) + def _to_dict(self): + return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) + for k, v in six.iteritems(self)} - def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.attributes) + def copy_to_dict(self): + """ + Make a copy of current settings and convert to a dict. + + This method returns a new dict populated with the same values + and their priorities as the current settings. + + Modifications to the returned dict won't be reflected on the original + settings. + + This method can be useful for example for printing settings + in Scrapy shell. + """ + settings = self.copy() + return settings._to_dict() @property def overrides(self): From aa78758bc744b3264c330cc18e58a8b15315517d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 18:59:12 +0100 Subject: [PATCH 5/8] Update tests for settings copy_to_dict() --- tests/test_settings/__init__.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 44b9b6df3..4acf22cba 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -302,6 +302,21 @@ class BaseSettingsTest(unittest.TestCase): self.assertListEqual(copy.get('TEST_LIST_OF_LISTS')[0], ['first_one', 'first_two']) + def test_copy_to_dict(self): + s = BaseSettings({'TEST_STRING': 'a string', + 'TEST_LIST': [1, 2], + 'TEST_BOOLEAN': False, + 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), + 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), + 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) + self.assertDictEqual(s.copy_to_dict(), + {'HASNOBASE': {3: 3000}, + 'TEST': {1: 10, 3: 30}, + 'TEST_BASE': {1: 1, 2: 2}, + 'TEST_BOOLEAN': False, + 'TEST_LIST': [1, 2], + 'TEST_STRING': 'a string'}) + def test_freeze(self): self.settings.freeze() with self.assertRaises(TypeError) as cm: @@ -343,14 +358,6 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(self.settings.defaults.get('BAR'), 'foo') self.assertIn('BAR', self.settings.defaults) - def test_repr(self): - settings = BaseSettings() - self.assertEqual(repr(settings), "") - attr = SettingsAttribute('testval', 15) - settings['testkey'] = attr - self.assertEqual(repr(settings), - "" % repr(attr)) - class SettingsTest(unittest.TestCase): From d843a0aae862a84b54f18f9d43ae957c182197b6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 29 Jan 2016 21:12:03 +0100 Subject: [PATCH 6/8] Amend "settings" command to output JSON for dict settings --- scrapy/commands/settings.py | 9 ++++++++- tests/test_cmdline/__init__.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 0e73f4f58..bce4e6086 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,8 @@ from __future__ import print_function +import json + from scrapy.commands import ScrapyCommand +from scrapy.settings import BaseSettings class Command(ScrapyCommand): @@ -28,7 +31,11 @@ class Command(ScrapyCommand): def run(self, args, opts): settings = self.crawler_process.settings if opts.get: - print(settings.get(opts.get)) + s = settings.get(opts.get) + if isinstance(s, BaseSettings): + print(json.dumps(s.copy_to_dict())) + else: + print(s) elif opts.getbool: print(settings.getbool(opts.getbool)) elif opts.getint: diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index c2de4fbc8..7733e7180 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) - self.assertIn('value=200', settingsdict[EXT_PATH]) + self.assertEquals(200, settingsdict[EXT_PATH]) From 268e912273dcb7bacc5a7102d5a3f90a868f035f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 1 Feb 2016 12:43:27 +0100 Subject: [PATCH 7/8] Add pretty-printting of settings as dict if using IPython shell Suggested by @digenis see http://ipython.readthedocs.org/en/stable/api/generated/IPython.lib.pretty.html?#extending --- scrapy/settings/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 918bfc1e5..7b7808959 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -4,6 +4,7 @@ import copy import warnings from collections import MutableMapping from importlib import import_module +from pprint import pformat from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning @@ -388,6 +389,12 @@ class BaseSettings(MutableMapping): settings = self.copy() return settings._to_dict() + def _repr_pretty_(self, p, cycle): + if cycle: + p.text(repr(self)) + else: + p.text(pformat(self.copy_to_dict())) + @property def overrides(self): warnings.warn("`Settings.overrides` attribute is deprecated and won't " From 51dc741fa2958700b3ab24c8abecde0bf91c7292 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 3 Feb 2016 16:08:46 +0100 Subject: [PATCH 8/8] Add 1.0.5 release notes --- docs/news.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 4d7dc4d41..3e60b2b4f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,14 @@ Release notes ============= +1.0.5 (2016-02-04) +------------------ + +- FIX: [Backport] Ignore bogus links in LinkExtractors (fixes :issue:`907`, :commit:`108195e`) +- TST: Changed buildbot makefile to use 'pytest' (:commit:`1f3d90a`) +- DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`) +- DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`) + 1.0.4 (2015-12-30) ------------------ @@ -590,7 +598,7 @@ Enhancements - Document `request_scheduled` signal (:issue:`746`) - Add a note about reporting security issues (:issue:`697`) - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`) -- Sort spider list output of `scrapy list` command (:issue:`742`) +- Sort spider list output of `scrapy list` command (:issue:`742`) - Multiple documentation enhancemens and fixes (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`, :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,