From 5f1b1c05f8fa98fca544e02eeda8b5cdba526cbb Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 8 Sep 2011 15:18:10 -0300 Subject: [PATCH] Do not filter requests with dont_filter attribute set in OffsiteMiddleware --- docs/topics/spider-middleware.rst | 6 +++++- scrapy/contrib/spidermiddleware/offsite.py | 2 +- scrapy/tests/test_spidermiddleware_offsite.py | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 66059e6af..ea3466ecc 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -211,7 +211,7 @@ OffsiteMiddleware ----------------- .. module:: scrapy.contrib.spidermiddleware.offsite - :synopsis: Offiste Spider Middleware + :synopsis: Offsite Spider Middleware .. class:: OffsiteMiddleware @@ -236,6 +236,10 @@ OffsiteMiddleware :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. + If the request has the :attr:`~scrapy.http.Request.dont_filter` attribute + set, the offsite middleware will allow the request even if its domain is not + listed in allowed domains. + RefererMiddleware ----------------- diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index ac583c5a5..acb616043 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -23,7 +23,7 @@ class OffsiteMiddleware(object): def process_spider_output(self, response, result, spider): for x in result: if isinstance(x, Request): - if self.should_follow(x, spider): + if x.dont_filter or self.should_follow(x, spider): yield x else: domain = urlparse_cached(x).hostname diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 8d9c173b4..44433ee57 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -20,8 +20,10 @@ class TestOffsiteMiddleware(TestCase): onsite_reqs = [Request('http://scrapytest.org/1'), Request('http://scrapy.org/1'), - Request('http://sub.scrapy.org/1')] - offsite_reqs = [Request('http://scrapy2.org')] + Request('http://sub.scrapy.org/1'), + Request('http://offsite.tld/letmepass', dont_filter=True)] + offsite_reqs = [Request('http://scrapy2.org'), + Request('http://offsite.tld/')] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider))