mirror of https://github.com/scrapy/scrapy.git
Removed per-spider settings concept, and scrapy.conf.settings singleton from many extensions and middlewares. There are some still remaining, that will be removed in future commits
This commit is contained in:
Pablo Hoffman
parent
19bcb44c25
commit
36f47a4aec
|
|
@ -18,6 +18,9 @@ Scrapy changes:
|
|||
- added :setting:`REFERER_ENABLED` setting, to control referer middleware
|
||||
- changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)``
|
||||
- removed (undocumented) ``HTMLImageLinkExtractor`` class from ``scrapy.contrib.linkextractors.image``
|
||||
- removed per-spider settings (to be replaced by instantiating multiple crawler objects)
|
||||
- ``USER_AGENT`` spider attribute will no longer work, use ``user_agent`` attribute instead
|
||||
- ``DOWNLOAD_TIMEOUT`` spider attribute will no longer work, use ``download_timeout`` attribute instead
|
||||
|
||||
Scrapyd changes:
|
||||
|
||||
|
|
|
|||
|
|
@ -103,10 +103,10 @@ information on which commands must be run from inside projects, and which not.
|
|||
|
||||
Also keep in mind that some commands may have slightly different behaviours
|
||||
when running them from inside projects. For example, the fetch command will use
|
||||
spider-overridden behaviours (such as custom :setting:`USER_AGENT` per-spider
|
||||
setting) if the url being fetched is associated with some specific spider. This
|
||||
is intentional, as the ``fetch`` command is meant to be used to check how
|
||||
spiders are downloading pages.
|
||||
spider-overridden behaviours (such as the ``user_agent`` attribute to override
|
||||
the user-agent) if the url being fetched is associated with some specific
|
||||
spider. This is intentional, as the ``fetch`` command is meant to be used to
|
||||
check how spiders are downloading pages.
|
||||
|
||||
.. _topics-commands-ref:
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ conditions are met.
|
|||
See documentation in docs/topics/extensions.rst
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -12,17 +11,15 @@ from twisted.python import log as txlog
|
|||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy import signals, log
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CloseSpider(object):
|
||||
|
||||
def __init__(self, crawler):
|
||||
self.crawler = crawler
|
||||
self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
|
||||
self.itemcount = settings.getint('CLOSESPIDER_ITEMCOUNT')
|
||||
self.pagecount = settings.getint('CLOSESPIDER_PAGECOUNT')
|
||||
self.errorcount = settings.getint('CLOSESPIDER_ERRORCOUNT')
|
||||
self.timeout = crawler.settings.getint('CLOSESPIDER_TIMEOUT')
|
||||
self.itemcount = crawler.settings.getint('CLOSESPIDER_ITEMCOUNT')
|
||||
self.pagecount = crawler.settings.getint('CLOSESPIDER_PAGECOUNT')
|
||||
self.errorcount = crawler.settings.getint('CLOSESPIDER_ERRORCOUNT')
|
||||
|
||||
self.errorcounts = defaultdict(int)
|
||||
self.pagecounts = defaultdict(int)
|
||||
|
|
|
|||
|
|
@ -4,18 +4,21 @@ from collections import defaultdict
|
|||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.conf import settings
|
||||
from scrapy import log
|
||||
|
||||
|
||||
class CookiesMiddleware(object):
|
||||
"""This middleware enables working with sites that need cookies"""
|
||||
debug = settings.getbool('COOKIES_DEBUG')
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('COOKIES_ENABLED'):
|
||||
raise NotConfigured
|
||||
def __init__(self, debug=False):
|
||||
self.jars = defaultdict(CookieJar)
|
||||
self.debug = debug
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool('COOKIES_ENABLED'):
|
||||
raise NotConfigured
|
||||
return cls(crawler.settings.getbool('COOKIES_DEBUG'))
|
||||
|
||||
def process_request(self, request, spider):
|
||||
if 'dont_merge_cookies' in request.meta:
|
||||
|
|
|
|||
|
|
@ -3,17 +3,21 @@ DefaultHeaders downloader middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
from scrapy import conf
|
||||
from scrapy.utils.python import WeakKeyCache
|
||||
|
||||
|
||||
class DefaultHeadersMiddleware(object):
|
||||
|
||||
def __init__(self, settings=conf.settings):
|
||||
def __init__(self, settings):
|
||||
self._headers = WeakKeyCache(self._default_headers)
|
||||
self._settings = settings
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings)
|
||||
|
||||
def _default_headers(self, spider):
|
||||
return spider.settings.get('DEFAULT_REQUEST_HEADERS').items()
|
||||
return self._settings.get('DEFAULT_REQUEST_HEADERS').items()
|
||||
|
||||
def process_request(self, request, spider):
|
||||
for k, v in self._headers[spider]:
|
||||
|
|
|
|||
|
|
@ -8,13 +8,18 @@ from scrapy.utils.python import WeakKeyCache
|
|||
|
||||
class DownloadTimeoutMiddleware(object):
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, timeout=180):
|
||||
self._cache = WeakKeyCache(self._download_timeout)
|
||||
self._timeout = timeout
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings['DOWNLOAD_TIMEOUT'])
|
||||
|
||||
def _download_timeout(self, spider):
|
||||
if hasattr(spider, 'download_timeout'):
|
||||
return spider.download_timeout
|
||||
return spider.settings.getint('DOWNLOAD_TIMEOUT')
|
||||
return self._timeout
|
||||
|
||||
def process_request(self, request, spider):
|
||||
timeout = self._cache[spider]
|
||||
|
|
|
|||
|
|
@ -15,12 +15,11 @@ from scrapy.utils.request import request_fingerprint
|
|||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy import conf
|
||||
|
||||
|
||||
class HttpCacheMiddleware(object):
|
||||
|
||||
def __init__(self, settings=conf.settings):
|
||||
def __init__(self, settings):
|
||||
if not settings.getbool('HTTPCACHE_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
|
||||
|
|
@ -30,6 +29,10 @@ class HttpCacheMiddleware(object):
|
|||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings)
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.storage.open_spider(spider)
|
||||
|
||||
|
|
@ -66,7 +69,7 @@ class HttpCacheMiddleware(object):
|
|||
|
||||
class FilesystemCacheStorage(object):
|
||||
|
||||
def __init__(self, settings=conf.settings):
|
||||
def __init__(self, settings):
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
|
||||
|
|
|
|||
|
|
@ -4,19 +4,22 @@ from scrapy import log
|
|||
from scrapy.http import HtmlResponse
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class RedirectMiddleware(object):
|
||||
"""Handle redirection of requests based on response status and meta-refresh html tag"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, settings):
|
||||
if not settings.getbool('REDIRECT_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY')
|
||||
self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
|
||||
self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if 'dont_redirect' in request.meta:
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError
|
|||
from scrapy import log
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.response import response_status_message
|
||||
from scrapy.conf import settings
|
||||
|
||||
class RetryMiddleware(object):
|
||||
|
||||
|
|
@ -37,13 +36,17 @@ class RetryMiddleware(object):
|
|||
ConnectionLost, TCPTimedOutError,
|
||||
IOError)
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, settings):
|
||||
if not settings.getbool('RETRY_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.max_retry_times = settings.getint('RETRY_TIMES')
|
||||
self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
|
||||
self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if 'dont_retry' in request.meta:
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ from scrapy.exceptions import NotConfigured
|
|||
from scrapy.utils.request import request_httprepr
|
||||
from scrapy.utils.response import response_httprepr
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
class DownloaderStats(object):
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('DOWNLOADER_STATS'):
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool('DOWNLOADER_STATS'):
|
||||
raise NotConfigured
|
||||
return cls()
|
||||
|
||||
def process_request(self, request, spider):
|
||||
stats.inc_value('downloader/request_count', spider=spider)
|
||||
|
|
|
|||
|
|
@ -6,13 +6,18 @@ from scrapy.utils.python import WeakKeyCache
|
|||
class UserAgentMiddleware(object):
|
||||
"""This middleware allows spiders to override the user_agent"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, user_agent='Scrapy'):
|
||||
self.cache = WeakKeyCache(self._user_agent)
|
||||
self.user_agent = user_agent
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls(crawler.settings['USER_AGENT'])
|
||||
|
||||
def _user_agent(self, spider):
|
||||
if hasattr(spider, 'user_agent'):
|
||||
return spider.user_agent
|
||||
return spider.settings['USER_AGENT']
|
||||
return self.user_agent
|
||||
|
||||
def process_request(self, request, spider):
|
||||
ua = self.cache[spider]
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ from scrapy.xlib.pydispatch import dispatcher
|
|||
from scrapy.utils.ftp import ftp_makedirs_cwd
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class IFeedStorage(Interface):
|
||||
|
|
@ -82,6 +81,7 @@ class FileFeedStorage(object):
|
|||
class S3FeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri):
|
||||
from scrapy.conf import settings
|
||||
try:
|
||||
import boto
|
||||
except ImportError:
|
||||
|
|
@ -133,7 +133,8 @@ class SpiderSlot(object):
|
|||
|
||||
class FeedExporter(object):
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.urifmt = settings['FEED_URI']
|
||||
if not self.urifmt:
|
||||
raise NotConfigured
|
||||
|
|
@ -152,6 +153,10 @@ class FeedExporter(object):
|
|||
dispatcher.connect(self.close_spider, signals.spider_closed)
|
||||
dispatcher.connect(self.item_scraped, signals.item_scraped)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return FeedExporter(crawler.settings)
|
||||
|
||||
def open_spider(self, spider):
|
||||
uri = self.urifmt % self._get_uri_params(spider)
|
||||
storage = self._get_storage(uri)
|
||||
|
|
@ -179,8 +184,8 @@ class FeedExporter(object):
|
|||
return item
|
||||
|
||||
def _load_components(self, setting_prefix):
|
||||
conf = dict(settings['%s_BASE' % setting_prefix])
|
||||
conf.update(settings[setting_prefix])
|
||||
conf = dict(self.settings['%s_BASE' % setting_prefix])
|
||||
conf.update(self.settings[setting_prefix])
|
||||
d = {}
|
||||
for k, v in conf.items():
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -6,12 +6,11 @@ from scrapy.http import Headers
|
|||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy import conf
|
||||
|
||||
|
||||
class DbmCacheStorage(object):
|
||||
|
||||
def __init__(self, settings=conf.settings):
|
||||
def __init__(self, settings):
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'])
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'])
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from twisted.internet import task
|
|||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
from scrapy import log, signals
|
||||
|
||||
class Slot(object):
|
||||
|
|
@ -16,10 +15,8 @@ class Slot(object):
|
|||
class LogStats(object):
|
||||
"""Log basic scraping stats periodically"""
|
||||
|
||||
def __init__(self):
|
||||
self.interval = settings.getfloat('LOGSTATS_INTERVAL')
|
||||
if not self.interval:
|
||||
raise NotConfigured
|
||||
def __init__(self, interval=60.0):
|
||||
self.interval = interval
|
||||
self.slots = {}
|
||||
self.multiplier = 60.0 / self.interval
|
||||
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
|
||||
|
|
@ -29,6 +26,13 @@ class LogStats(object):
|
|||
dispatcher.connect(self.engine_started, signal=signals.engine_started)
|
||||
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
interval = crawler.settings.getfloat('LOGSTATS_INTERVAL')
|
||||
if not interval:
|
||||
raise NotConfigured
|
||||
return cls(interval)
|
||||
|
||||
def item_scraped(self, spider):
|
||||
self.slots[spider].items += 1
|
||||
|
||||
|
|
|
|||
|
|
@ -10,24 +10,27 @@ from scrapy.xlib.pydispatch import dispatcher
|
|||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.conf import settings
|
||||
from scrapy.stats import stats
|
||||
from scrapy.utils.trackref import live_refs
|
||||
|
||||
class MemoryDebugger(object):
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, trackrefs=False):
|
||||
try:
|
||||
import libxml2
|
||||
self.libxml2 = libxml2
|
||||
except ImportError:
|
||||
self.libxml2 = None
|
||||
if not settings.getbool('MEMDEBUG_ENABLED'):
|
||||
raise NotConfigured
|
||||
|
||||
self.trackrefs = trackrefs
|
||||
dispatcher.connect(self.engine_started, signals.engine_started)
|
||||
dispatcher.connect(self.engine_stopped, signals.engine_stopped)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
|
||||
raise NotConfigured
|
||||
return cls(crawler.settings.getbool('TRACK_REFS'))
|
||||
|
||||
def engine_started(self):
|
||||
if self.libxml2:
|
||||
self.libxml2.debugMemory(1)
|
||||
|
|
@ -38,7 +41,7 @@ class MemoryDebugger(object):
|
|||
stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1))
|
||||
gc.collect()
|
||||
stats.set_value('memdebug/gc_garbage_count', len(gc.garbage))
|
||||
if settings.getbool('TRACK_REFS'):
|
||||
if self.trackrefs:
|
||||
for cls, wdict in live_refs.iteritems():
|
||||
if not wdict:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import copy
|
|||
from scrapy.http import Request, HtmlResponse
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.conf import settings
|
||||
|
||||
def identity(x):
|
||||
return x
|
||||
|
|
@ -70,10 +69,9 @@ class CrawlSpider(BaseSpider):
|
|||
for requests_or_item in iterate_spider_output(cb_res):
|
||||
yield requests_or_item
|
||||
|
||||
if follow and settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True):
|
||||
if follow and self._follow_links:
|
||||
for request_or_item in self._requests_to_follow(response):
|
||||
yield request_or_item
|
||||
|
||||
|
||||
def _compile_rules(self):
|
||||
def get_method(method):
|
||||
|
|
@ -87,3 +85,7 @@ class CrawlSpider(BaseSpider):
|
|||
rule.callback = get_method(rule.callback)
|
||||
rule.process_links = get_method(rule.process_links)
|
||||
rule.process_request = get_method(rule.process_request)
|
||||
|
||||
def set_crawler(self, crawler):
|
||||
super(CrawlSpider, self).set_crawler(crawler)
|
||||
self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
|
||||
|
|
|
|||
|
|
@ -9,17 +9,21 @@ from scrapy.xlib.pydispatch import dispatcher
|
|||
from scrapy.stats import stats
|
||||
from scrapy import signals
|
||||
from scrapy.mail import MailSender
|
||||
from scrapy.conf import settings
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
class StatsMailer(object):
|
||||
|
||||
def __init__(self):
|
||||
self.recipients = settings.getlist("STATSMAILER_RCPTS")
|
||||
def __init__(self, recipients):
|
||||
self.recipients = recipients
|
||||
if not self.recipients:
|
||||
raise NotConfigured
|
||||
dispatcher.connect(self.stats_spider_closed, signal=signals.stats_spider_closed)
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
|
||||
return cls(recipients)
|
||||
|
||||
def stats_spider_closed(self, spider, spider_stats):
|
||||
mail = MailSender()
|
||||
body = "Global stats\n\n"
|
||||
|
|
|
|||
|
|
@ -184,6 +184,7 @@ class Scraper(object):
|
|||
"""
|
||||
if spider_failure is download_failure:
|
||||
errmsg = spider_failure.getErrorMessage()
|
||||
spider_failure.printTraceback()
|
||||
if errmsg:
|
||||
log.msg("Error downloading %s: %s" % (request, errmsg), log.ERROR, spider=spider)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -57,25 +57,3 @@ class CrawlerSettings(Settings):
|
|||
|
||||
def __str__(self):
|
||||
return "<CrawlerSettings module=%r>" % self.settings_module
|
||||
|
||||
|
||||
class SpiderSettings(Settings):
|
||||
|
||||
def __init__(self, spider, crawler_settings, **kw):
|
||||
super(SpiderSettings, self).__init__(**kw)
|
||||
self.spider = spider
|
||||
self.cset = crawler_settings
|
||||
|
||||
def __getitem__(self, opt_name):
|
||||
if opt_name in self.cset.overrides:
|
||||
return self.cset.overrides[opt_name]
|
||||
if hasattr(self.spider, opt_name):
|
||||
return getattr(self.spider, opt_name)
|
||||
if self.cset.settings_module and hasattr(self.cset.settings_module, opt_name):
|
||||
return getattr(self.cset.settings_module, opt_name)
|
||||
if opt_name in self.cset.defaults:
|
||||
return self.cset.defaults[opt_name]
|
||||
return super(SpiderSettings, self).__getitem__(opt_name)
|
||||
|
||||
def __str__(self):
|
||||
return "<SpiderSettings spider=%r>" % self.spider.name
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ See documentation in docs/topics/spiders.rst
|
|||
"""
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.settings import SpiderSettings
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
|
@ -45,9 +44,7 @@ class BaseSpider(object_ref):
|
|||
|
||||
@property
|
||||
def settings(self):
|
||||
if not hasattr(self, '_settings'):
|
||||
self._settings = SpiderSettings(self, self.crawler.settings)
|
||||
return self._settings
|
||||
return self.crawler.settings
|
||||
|
||||
def start_requests(self):
|
||||
for url in self.start_urls:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from unittest import TestCase
|
||||
|
||||
from scrapy.conf import settings
|
||||
from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import BaseSpider
|
||||
|
|
@ -15,7 +14,7 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
spider.set_crawler(crawler)
|
||||
defaults = dict([(k, [v]) for k, v in \
|
||||
crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])
|
||||
return defaults, spider, DefaultHeadersMiddleware()
|
||||
return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_process_request(self):
|
||||
defaults, spider, mw = self.get_defaults_spider_mw()
|
||||
|
|
@ -23,19 +22,6 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
mw.process_request(req, spider)
|
||||
self.assertEquals(req.headers, defaults)
|
||||
|
||||
def test_spider_default_request_headers(self):
|
||||
defaults, spider, mw = self.get_defaults_spider_mw()
|
||||
spider_headers = {'Unexistant-Header': ['value']}
|
||||
# override one of the global default headers by spider
|
||||
if defaults:
|
||||
k = set(defaults).pop()
|
||||
spider_headers[k] = ['__newvalue__']
|
||||
spider.DEFAULT_REQUEST_HEADERS = spider_headers
|
||||
|
||||
req = Request('http://www.scrapytest.org')
|
||||
mw.process_request(req, spider)
|
||||
self.assertEquals(req.headers, dict(spider_headers))
|
||||
|
||||
def test_update_headers(self):
|
||||
defaults, spider, mw = self.get_defaults_spider_mw()
|
||||
headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
|
|||
spider = BaseSpider('foo')
|
||||
spider.set_crawler(crawler)
|
||||
request = Request('http://scrapytest.org/')
|
||||
return request, spider, DownloadTimeoutMiddleware()
|
||||
return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_default_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw()
|
||||
|
|
@ -22,13 +22,13 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
|
|||
|
||||
def test_spider_has_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw()
|
||||
spider.DOWNLOAD_TIMEOUT = 2
|
||||
spider.download_timeout = 2
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta.get('download_timeout'), 2)
|
||||
|
||||
def test_request_has_download_timeout(self):
|
||||
req, spider, mw = self.get_request_spider_mw()
|
||||
spider.DOWNLOAD_TIMEOUT = 2
|
||||
spider.download_timeout = 2
|
||||
req.meta['download_timeout'] = 1
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta.get('download_timeout'), 1)
|
||||
|
|
|
|||
|
|
@ -4,12 +4,14 @@ from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware
|
|||
from scrapy.spider import BaseSpider
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.http import Request, Response, HtmlResponse, Headers
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
class RedirectMiddlewareTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
crawler = get_crawler()
|
||||
self.spider = BaseSpider('foo')
|
||||
self.mw = RedirectMiddleware()
|
||||
self.mw = RedirectMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_priority_adjust(self):
|
||||
req = Request('http://a.com')
|
||||
|
|
|
|||
|
|
@ -7,11 +7,13 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup
|
|||
from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
class RetryTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
crawler = get_crawler()
|
||||
self.spider = BaseSpider('foo')
|
||||
self.mw = RetryMiddleware()
|
||||
self.mw = RetryMiddleware.from_crawler(crawler)
|
||||
self.mw.max_retry_times = 2
|
||||
|
||||
def test_priority_adjust(self):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ class UserAgentMiddlewareTest(TestCase):
|
|||
crawler = get_crawler({'USER_AGENT': default_useragent})
|
||||
spider = BaseSpider('foo')
|
||||
spider.set_crawler(crawler)
|
||||
return spider, UserAgentMiddleware()
|
||||
return spider, UserAgentMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_default_agent(self):
|
||||
spider, mw = self.get_spider_and_mw('default_useragent')
|
||||
|
|
@ -23,28 +23,28 @@ class UserAgentMiddlewareTest(TestCase):
|
|||
def test_remove_agent(self):
|
||||
# settings UESR_AGENT to None should remove the user agent
|
||||
spider, mw = self.get_spider_and_mw('default_useragent')
|
||||
spider.USER_AGENT = None
|
||||
spider.user_agent = None
|
||||
req = Request('http://scrapytest.org/')
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert req.headers.get('User-Agent') is None
|
||||
|
||||
def test_spider_agent(self):
|
||||
spider, mw = self.get_spider_and_mw('default_useragent')
|
||||
spider.USER_AGENT = 'spider_useragent'
|
||||
spider.user_agent = 'spider_useragent'
|
||||
req = Request('http://scrapytest.org/')
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.headers['User-Agent'], 'spider_useragent')
|
||||
|
||||
def test_header_agent(self):
|
||||
spider, mw = self.get_spider_and_mw('default_useragent')
|
||||
spider.USER_AGENT = 'spider_useragent'
|
||||
spider.user_agent = 'spider_useragent'
|
||||
req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.headers['User-Agent'], 'header_useragent')
|
||||
|
||||
def test_no_agent(self):
|
||||
spider, mw = self.get_spider_and_mw(None)
|
||||
spider.USER_AGENT = None
|
||||
spider.user_agent = None
|
||||
req = Request('http://scrapytest.org/')
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'User-Agent' not in req.headers
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.settings import Settings, SpiderSettings
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.test import get_crawler
|
||||
from scrapy.spider import BaseSpider
|
||||
|
||||
|
|
@ -68,35 +68,6 @@ class CrawlerSettingsTest(unittest.TestCase):
|
|||
crawler.settings.overrides['DOWNLOAD_TIMEOUT'] = '15'
|
||||
self.assertEqual(crawler.settings.getint('DOWNLOAD_TIMEOUT'), 15)
|
||||
|
||||
class SpiderSettingsTest(unittest.TestCase):
|
||||
|
||||
def test_global_defaults(self):
|
||||
crawler = get_crawler()
|
||||
settings = SpiderSettings(BaseSpider('name'), crawler.settings)
|
||||
self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 180)
|
||||
|
||||
def test_defaults(self):
|
||||
crawler = get_crawler()
|
||||
crawler.settings.defaults['DOWNLOAD_TIMEOUT'] = '99'
|
||||
settings = SpiderSettings(BaseSpider('name'), crawler.settings)
|
||||
self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 99)
|
||||
|
||||
def test_crawler_defaults(self):
|
||||
crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
|
||||
settings = SpiderSettings(BaseSpider('name'), crawler.settings)
|
||||
self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 3)
|
||||
|
||||
def test_spider_overrides_crawler(self):
|
||||
crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
|
||||
crawler.settings.defaults['DOWNLOAD_TIMEOUT'] = '99'
|
||||
settings = SpiderSettings(BaseSpider('name', DOWNLOAD_TIMEOUT='12'), crawler.settings)
|
||||
self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 12)
|
||||
|
||||
def test_overrides_most_precedence(self):
|
||||
crawler = get_crawler({'DOWNLOAD_TIMEOUT': '3'})
|
||||
crawler.settings.overrides['DOWNLOAD_TIMEOUT'] = '15'
|
||||
settings = SpiderSettings(BaseSpider('name', DOWNLOAD_TIMEOUT='12'), crawler.settings)
|
||||
self.assertEqual(settings.getint('DOWNLOAD_TIMEOUT'), 15)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue