From 36f47a4aec6d6d0dedb26848ffec8112fb189e26 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 21 Aug 2012 17:27:45 -0300 Subject: [PATCH] Removed per-spider settings concept, and scrapy.conf.settings singleton from many extensions and middlewares. There are some still remaining, that will be removed in future commits --- docs/news.rst | 3 ++ docs/topics/commands.rst | 8 ++--- scrapy/contrib/closespider.py | 11 +++---- .../contrib/downloadermiddleware/cookies.py | 13 +++++--- .../downloadermiddleware/defaultheaders.py | 10 ++++-- .../downloadermiddleware/downloadtimeout.py | 9 ++++-- .../contrib/downloadermiddleware/httpcache.py | 9 ++++-- .../contrib/downloadermiddleware/redirect.py | 7 +++-- scrapy/contrib/downloadermiddleware/retry.py | 7 +++-- scrapy/contrib/downloadermiddleware/stats.py | 7 +++-- .../contrib/downloadermiddleware/useragent.py | 9 ++++-- scrapy/contrib/feedexport.py | 13 +++++--- scrapy/contrib/httpcache.py | 3 +- scrapy/contrib/logstats.py | 14 ++++++--- scrapy/contrib/memdebug.py | 15 +++++---- scrapy/contrib/spiders/crawl.py | 8 +++-- scrapy/contrib/statsmailer.py | 12 ++++--- scrapy/core/scraper.py | 1 + scrapy/settings/__init__.py | 22 ------------- scrapy/spider.py | 5 +-- ...est_downloadermiddleware_defaultheaders.py | 16 +--------- ...st_downloadermiddleware_downloadtimeout.py | 6 ++-- .../test_downloadermiddleware_redirect.py | 4 ++- .../tests/test_downloadermiddleware_retry.py | 4 ++- .../test_downloadermiddleware_useragent.py | 10 +++--- scrapy/tests/test_settings.py | 31 +------------------ 26 files changed, 119 insertions(+), 138 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7b71691b5..8b4fdc1b0 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -18,6 +18,9 @@ Scrapy changes: - added :setting:`REFERER_ENABLED` setting, to control referer middleware - changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)`` - removed (undocumented) ``HTMLImageLinkExtractor`` class from ``scrapy.contrib.linkextractors.image`` +- removed per-spider settings (to be replaced by instantiating multiple crawler objects) +- ``USER_AGENT`` spider attribute will no longer work, use ``user_agent`` attribute instead +- ``DOWNLOAD_TIMEOUT`` spider attribute will no longer work, use ``download_timeout`` attribute instead Scrapyd changes: diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index bac238fb8..8e2150e5f 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -103,10 +103,10 @@ information on which commands must be run from inside projects, and which not. Also keep in mind that some commands may have slightly different behaviours when running them from inside projects. For example, the fetch command will use -spider-overridden behaviours (such as custom :setting:`USER_AGENT` per-spider -setting) if the url being fetched is associated with some specific spider. This -is intentional, as the ``fetch`` command is meant to be used to check how -spiders are downloading pages. +spider-overridden behaviours (such as the ``user_agent`` attribute to override +the user-agent) if the url being fetched is associated with some specific +spider. This is intentional, as the ``fetch`` command is meant to be used to +check how spiders are downloading pages. .. _topics-commands-ref: diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index a92d92dff..f4611f4fc 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -4,7 +4,6 @@ conditions are met. See documentation in docs/topics/extensions.rst """ -import warnings from collections import defaultdict from twisted.internet import reactor @@ -12,17 +11,15 @@ from twisted.python import log as txlog from scrapy.xlib.pydispatch import dispatcher from scrapy import signals, log -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.conf import settings class CloseSpider(object): def __init__(self, crawler): self.crawler = crawler - self.timeout = settings.getint('CLOSESPIDER_TIMEOUT') - self.itemcount = settings.getint('CLOSESPIDER_ITEMCOUNT') - self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT') - self.errorcount = settings.getint('CLOSESPIDER_ERRORCOUNT') + self.timeout = crawler.settings.getint('CLOSESPIDER_TIMEOUT') + self.itemcount = crawler.settings.getint('CLOSESPIDER_ITEMCOUNT') + self.pagecount = crawler.settings.getint('CLOSESPIDER_PAGECOUNT') + self.errorcount = crawler.settings.getint('CLOSESPIDER_ERRORCOUNT') self.errorcounts = defaultdict(int) self.pagecounts = defaultdict(int) diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py index ed61a89f9..bfb1490fd 100644 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ b/scrapy/contrib/downloadermiddleware/cookies.py @@ -4,18 +4,21 @@ from collections import defaultdict from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.conf import settings from scrapy import log class CookiesMiddleware(object): """This middleware enables working with sites that need cookies""" - debug = settings.getbool('COOKIES_DEBUG') - def __init__(self): - if not settings.getbool('COOKIES_ENABLED'): - raise NotConfigured + def __init__(self, debug=False): self.jars = defaultdict(CookieJar) + self.debug = debug + + @classmethod + def from_crawler(cls, crawler): + if not crawler.settings.getbool('COOKIES_ENABLED'): + raise NotConfigured + return cls(crawler.settings.getbool('COOKIES_DEBUG')) def process_request(self, request, spider): if 'dont_merge_cookies' in request.meta: diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py index 61a05fc50..57eed1a0a 100644 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -3,17 +3,21 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -from scrapy import conf from scrapy.utils.python import WeakKeyCache class DefaultHeadersMiddleware(object): - def __init__(self, settings=conf.settings): + def __init__(self, settings): self._headers = WeakKeyCache(self._default_headers) + self._settings = settings + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings) def _default_headers(self, spider): - return spider.settings.get('DEFAULT_REQUEST_HEADERS').items() + return self._settings.get('DEFAULT_REQUEST_HEADERS').items() def process_request(self, request, spider): for k, v in self._headers[spider]: diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py index fe3754c61..ac75186cf 100644 --- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py +++ b/scrapy/contrib/downloadermiddleware/downloadtimeout.py @@ -8,13 +8,18 @@ from scrapy.utils.python import WeakKeyCache class DownloadTimeoutMiddleware(object): - def __init__(self): + def __init__(self, timeout=180): self._cache = WeakKeyCache(self._download_timeout) + self._timeout = timeout + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings['DOWNLOAD_TIMEOUT']) def _download_timeout(self, spider): if hasattr(spider, 'download_timeout'): return spider.download_timeout - return spider.settings.getint('DOWNLOAD_TIMEOUT') + return self._timeout def process_request(self, request, spider): timeout = self._cache[spider] diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 9eba51fdf..642f9bda7 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -15,12 +15,11 @@ from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.project import data_path -from scrapy import conf class HttpCacheMiddleware(object): - def __init__(self, settings=conf.settings): + def __init__(self, settings): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) @@ -30,6 +29,10 @@ class HttpCacheMiddleware(object): dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings) + def spider_opened(self, spider): self.storage.open_spider(spider) @@ -66,7 +69,7 @@ class HttpCacheMiddleware(object): class FilesystemCacheStorage(object): - def __init__(self, settings=conf.settings): + def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR']) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 02e402483..ea1247532 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -4,19 +4,22 @@ from scrapy import log from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured -from scrapy.conf import settings class RedirectMiddleware(object): """Handle redirection of requests based on response status and meta-refresh html tag""" - def __init__(self): + def __init__(self, settings): if not settings.getbool('REDIRECT_ENABLED'): raise NotConfigured self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY') self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES') self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST') + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings) + def process_response(self, request, response, spider): if 'dont_redirect' in request.meta: return response diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index 807c30622..38dc5988b 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -26,7 +26,6 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError from scrapy import log from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message -from scrapy.conf import settings class RetryMiddleware(object): @@ -37,13 +36,17 @@ class RetryMiddleware(object): ConnectionLost, TCPTimedOutError, IOError) - def __init__(self): + def __init__(self, settings): if not settings.getbool('RETRY_ENABLED'): raise NotConfigured self.max_retry_times = settings.getint('RETRY_TIMES') self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings) + def process_response(self, request, response, spider): if 'dont_retry' in request.meta: return response diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py index 0b8062d60..32bad0bb3 100644 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ b/scrapy/contrib/downloadermiddleware/stats.py @@ -2,13 +2,14 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.request import request_httprepr from scrapy.utils.response import response_httprepr from scrapy.stats import stats -from scrapy.conf import settings class DownloaderStats(object): - def __init__(self): - if not settings.getbool('DOWNLOADER_STATS'): + @classmethod + def from_crawler(cls, crawler): + if not crawler.settings.getbool('DOWNLOADER_STATS'): raise NotConfigured + return cls() def process_request(self, request, spider): stats.inc_value('downloader/request_count', spider=spider) diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py index 42d60f7cf..8084ab1ad 100644 --- a/scrapy/contrib/downloadermiddleware/useragent.py +++ b/scrapy/contrib/downloadermiddleware/useragent.py @@ -6,13 +6,18 @@ from scrapy.utils.python import WeakKeyCache class UserAgentMiddleware(object): """This middleware allows spiders to override the user_agent""" - def __init__(self): + def __init__(self, user_agent='Scrapy'): self.cache = WeakKeyCache(self._user_agent) + self.user_agent = user_agent + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings['USER_AGENT']) def _user_agent(self, spider): if hasattr(spider, 'user_agent'): return spider.user_agent - return spider.settings['USER_AGENT'] + return self.user_agent def process_request(self, request, spider): ua = self.cache[spider] diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index effe197de..cdda804f7 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -19,7 +19,6 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object -from scrapy.conf import settings class IFeedStorage(Interface): @@ -82,6 +81,7 @@ class FileFeedStorage(object): class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri): + from scrapy.conf import settings try: import boto except ImportError: @@ -133,7 +133,8 @@ class SpiderSlot(object): class FeedExporter(object): - def __init__(self): + def __init__(self, settings): + self.settings = settings self.urifmt = settings['FEED_URI'] if not self.urifmt: raise NotConfigured @@ -152,6 +153,10 @@ class FeedExporter(object): dispatcher.connect(self.close_spider, signals.spider_closed) dispatcher.connect(self.item_scraped, signals.item_scraped) + @classmethod + def from_crawler(cls, crawler): + return FeedExporter(crawler.settings) + def open_spider(self, spider): uri = self.urifmt % self._get_uri_params(spider) storage = self._get_storage(uri) @@ -179,8 +184,8 @@ class FeedExporter(object): return item def _load_components(self, setting_prefix): - conf = dict(settings['%s_BASE' % setting_prefix]) - conf.update(settings[setting_prefix]) + conf = dict(self.settings['%s_BASE' % setting_prefix]) + conf.update(self.settings[setting_prefix]) d = {} for k, v in conf.items(): try: diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 3d05e0900..87f980db2 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -6,12 +6,11 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path -from scrapy import conf class DbmCacheStorage(object): - def __init__(self, settings=conf.settings): + def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR']) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE']) diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py index 9ce599bff..a22c1cb2a 100644 --- a/scrapy/contrib/logstats.py +++ b/scrapy/contrib/logstats.py @@ -2,7 +2,6 @@ from twisted.internet import task from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured -from scrapy.conf import settings from scrapy import log, signals class Slot(object): @@ -16,10 +15,8 @@ class Slot(object): class LogStats(object): """Log basic scraping stats periodically""" - def __init__(self): - self.interval = settings.getfloat('LOGSTATS_INTERVAL') - if not self.interval: - raise NotConfigured + def __init__(self, interval=60.0): + self.interval = interval self.slots = {} self.multiplier = 60.0 / self.interval dispatcher.connect(self.item_scraped, signal=signals.item_scraped) @@ -29,6 +26,13 @@ class LogStats(object): dispatcher.connect(self.engine_started, signal=signals.engine_started) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + @classmethod + def from_crawler(cls, crawler): + interval = crawler.settings.getfloat('LOGSTATS_INTERVAL') + if not interval: + raise NotConfigured + return cls(interval) + def item_scraped(self, spider): self.slots[spider].items += 1 diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index a28aad2ef..cb9051fd2 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -10,24 +10,27 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.exceptions import NotConfigured -from scrapy.conf import settings from scrapy.stats import stats from scrapy.utils.trackref import live_refs class MemoryDebugger(object): - def __init__(self): + def __init__(self, trackrefs=False): try: import libxml2 self.libxml2 = libxml2 except ImportError: self.libxml2 = None - if not settings.getbool('MEMDEBUG_ENABLED'): - raise NotConfigured - + self.trackrefs = trackrefs dispatcher.connect(self.engine_started, signals.engine_started) dispatcher.connect(self.engine_stopped, signals.engine_stopped) + @classmethod + def from_crawler(cls, crawler): + if not crawler.settings.getbool('MEMDEBUG_ENABLED'): + raise NotConfigured + return cls(crawler.settings.getbool('TRACK_REFS')) + def engine_started(self): if self.libxml2: self.libxml2.debugMemory(1) @@ -38,7 +41,7 @@ class MemoryDebugger(object): stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) gc.collect() stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) - if settings.getbool('TRACK_REFS'): + if self.trackrefs: for cls, wdict in live_refs.iteritems(): if not wdict: continue diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 2dbb03199..3e98b9a61 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -10,7 +10,6 @@ import copy from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output from scrapy.spider import BaseSpider -from scrapy.conf import settings def identity(x): return x @@ -70,10 +69,9 @@ class CrawlSpider(BaseSpider): for requests_or_item in iterate_spider_output(cb_res): yield requests_or_item - if follow and settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True): + if follow and self._follow_links: for request_or_item in self._requests_to_follow(response): yield request_or_item - def _compile_rules(self): def get_method(method): @@ -87,3 +85,7 @@ class CrawlSpider(BaseSpider): rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) rule.process_request = get_method(rule.process_request) + + def set_crawler(self, crawler): + super(CrawlSpider, self).set_crawler(crawler) + self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index 21cce76fe..9dc5161ec 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -9,17 +9,21 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.stats import stats from scrapy import signals from scrapy.mail import MailSender -from scrapy.conf import settings from scrapy.exceptions import NotConfigured class StatsMailer(object): - def __init__(self): - self.recipients = settings.getlist("STATSMAILER_RCPTS") + def __init__(self, recipients): + self.recipients = recipients if not self.recipients: raise NotConfigured dispatcher.connect(self.stats_spider_closed, signal=signals.stats_spider_closed) - + + @classmethod + def from_crawler(cls, crawler): + recipients = crawler.settings.getlist("STATSMAILER_RCPTS") + return cls(recipients) + def stats_spider_closed(self, spider, spider_stats): mail = MailSender() body = "Global stats\n\n" diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ef6e17110..2e3b2d43e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -184,6 +184,7 @@ class Scraper(object): """ if spider_failure is download_failure: errmsg = spider_failure.getErrorMessage() + spider_failure.printTraceback() if errmsg: log.msg("Error downloading %s: %s" % (request, errmsg), log.ERROR, spider=spider) return diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 42cc35121..7075d4d9b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -57,25 +57,3 @@ class CrawlerSettings(Settings): def __str__(self): return "" % self.settings_module - - -class SpiderSettings(Settings): - - def __init__(self, spider, crawler_settings, **kw): - super(SpiderSettings, self).__init__(**kw) - self.spider = spider - self.cset = crawler_settings - - def __getitem__(self, opt_name): - if opt_name in self.cset.overrides: - return self.cset.overrides[opt_name] - if hasattr(self.spider, opt_name): - return getattr(self.spider, opt_name) - if self.cset.settings_module and hasattr(self.cset.settings_module, opt_name): - return getattr(self.cset.settings_module, opt_name) - if opt_name in self.cset.defaults: - return self.cset.defaults[opt_name] - return super(SpiderSettings, self).__getitem__(opt_name) - - def __str__(self): - return "" % self.spider.name diff --git a/scrapy/spider.py b/scrapy/spider.py index 1829a9d3b..5ac8dac07 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -5,7 +5,6 @@ See documentation in docs/topics/spiders.rst """ from scrapy import log -from scrapy.settings import SpiderSettings from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref @@ -45,9 +44,7 @@ class BaseSpider(object_ref): @property def settings(self): - if not hasattr(self, '_settings'): - self._settings = SpiderSettings(self, self.crawler.settings) - return self._settings + return self.crawler.settings def start_requests(self): for url in self.start_urls: diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index cc227e20c..4784d75cc 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -1,6 +1,5 @@ from unittest import TestCase -from scrapy.conf import settings from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spider import BaseSpider @@ -15,7 +14,7 @@ class TestDefaultHeadersMiddleware(TestCase): spider.set_crawler(crawler) defaults = dict([(k, [v]) for k, v in \ crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) - return defaults, spider, DefaultHeadersMiddleware() + return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) def test_process_request(self): defaults, spider, mw = self.get_defaults_spider_mw() @@ -23,19 +22,6 @@ class TestDefaultHeadersMiddleware(TestCase): mw.process_request(req, spider) self.assertEquals(req.headers, defaults) - def test_spider_default_request_headers(self): - defaults, spider, mw = self.get_defaults_spider_mw() - spider_headers = {'Unexistant-Header': ['value']} - # override one of the global default headers by spider - if defaults: - k = set(defaults).pop() - spider_headers[k] = ['__newvalue__'] - spider.DEFAULT_REQUEST_HEADERS = spider_headers - - req = Request('http://www.scrapytest.org') - mw.process_request(req, spider) - self.assertEquals(req.headers, dict(spider_headers)) - def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} diff --git a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py b/scrapy/tests/test_downloadermiddleware_downloadtimeout.py index fbe371996..5066becd3 100644 --- a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py +++ b/scrapy/tests/test_downloadermiddleware_downloadtimeout.py @@ -13,7 +13,7 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): spider = BaseSpider('foo') spider.set_crawler(crawler) request = Request('http://scrapytest.org/') - return request, spider, DownloadTimeoutMiddleware() + return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) def test_default_download_timeout(self): req, spider, mw = self.get_request_spider_mw() @@ -22,13 +22,13 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() - spider.DOWNLOAD_TIMEOUT = 2 + spider.download_timeout = 2 assert mw.process_request(req, spider) is None self.assertEquals(req.meta.get('download_timeout'), 2) def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() - spider.DOWNLOAD_TIMEOUT = 2 + spider.download_timeout = 2 req.meta['download_timeout'] = 1 assert mw.process_request(req, spider) is None self.assertEquals(req.meta.get('download_timeout'), 1) diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index 8be2e8194..0df82e02a 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -4,12 +4,14 @@ from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import BaseSpider from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response, HtmlResponse, Headers +from scrapy.utils.test import get_crawler class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): + crawler = get_crawler() self.spider = BaseSpider('foo') - self.mw = RedirectMiddleware() + self.mw = RedirectMiddleware.from_crawler(crawler) def test_priority_adjust(self): req = Request('http://a.com') diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index 11a3ccaba..d9bf1a543 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -7,11 +7,13 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware from scrapy.spider import BaseSpider from scrapy.http import Request, Response +from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): + crawler = get_crawler() self.spider = BaseSpider('foo') - self.mw = RetryMiddleware() + self.mw = RetryMiddleware.from_crawler(crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index 44ac74c8b..eced125eb 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -12,7 +12,7 @@ class UserAgentMiddlewareTest(TestCase): crawler = get_crawler({'USER_AGENT': default_useragent}) spider = BaseSpider('foo') spider.set_crawler(crawler) - return spider, UserAgentMiddleware() + return spider, UserAgentMiddleware.from_crawler(crawler) def test_default_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') @@ -23,28 +23,28 @@ class UserAgentMiddlewareTest(TestCase): def test_remove_agent(self): # settings UESR_AGENT to None should remove the user agent spider, mw = self.get_spider_and_mw('default_useragent') - spider.USER_AGENT = None + spider.user_agent = None req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None assert req.headers.get('User-Agent') is None def test_spider_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') - spider.USER_AGENT = 'spider_useragent' + spider.user_agent = 'spider_useragent' req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None self.assertEquals(req.headers['User-Agent'], 'spider_useragent') def test_header_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') - spider.USER_AGENT = 'spider_useragent' + spider.user_agent = 'spider_useragent' req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'}) assert mw.process_request(req, spider) is None self.assertEquals(req.headers['User-Agent'], 'header_useragent') def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) - spider.USER_AGENT = None + spider.user_agent = None req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None assert 'User-Agent' not in req.headers diff --git a/scrapy/tests/test_settings.py b/scrapy/tests/test_settings.py index 9da67652c..608afb045 100644 --- a/scrapy/tests/test_settings.py +++ b/scrapy/tests/test_settings.py @@ -1,6 +1,6 @@ import unittest -from scrapy.settings import Settings, SpiderSettings +from scrapy.settings import Settings from scrapy.utils.test import get_crawler from scrapy.spider import BaseSpider @@ -68,35 +68,6 @@ class CrawlerSettingsTest(unittest.TestCase): crawler.settings.overrides['DOWNLOAD_TIMEOUT'] = '15' self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 15) -class SpiderSettingsTest(unittest.TestCase): - - def test_global_defaults(self): - crawler = get_crawler() - settings = SpiderSettings(BaseSpider('name'), crawler.settings) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180) - - def test_defaults(self): - crawler = get_crawler() - crawler.settings.defaults['DOWNLOAD_TIMEOUT'] = '99' - settings = SpiderSettings(BaseSpider('name'), crawler.settings) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99) - - def test_crawler_defaults(self): - crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'}) - settings = SpiderSettings(BaseSpider('name'), crawler.settings) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3) - - def test_spider_overrides_crawler(self): - crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'}) - crawler.settings.defaults['DOWNLOAD_TIMEOUT'] = '99' - settings = SpiderSettings(BaseSpider('name', DOWNLOAD_TIMEOUT='12'), crawler.settings) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 12) - - def test_overrides_most_precedence(self): - crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'}) - crawler.settings.overrides['DOWNLOAD_TIMEOUT'] = '15' - settings = SpiderSettings(BaseSpider('name', DOWNLOAD_TIMEOUT='12'), crawler.settings) - self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15) if __name__ == "__main__": unittest.main()