RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OffsiteMiddleware: isolate policy of urls belonging to spiders into a separate method

This commit is contained in:
Pablo Hoffman 2009-08-15 20:44:11 -03:00
parent bc87350133
commit 08424ee093
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/contrib/spidermiddleware/offsite.py
View File

@ -10,4 +10,9 @@ from scrapy.utils.url import url_is_from_spider
class OffsiteMiddleware(object):
def process_spider_output(self, response, result, spider):
return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider))
return (x for x in result if not isinstance(x, Request) or \
self.is_url_from_spider(x.url, spider))
def is_url_from_spider(self, url, spider):
# XXX: should we ask this to the spider manager?
return url_is_from_spider(url, spider)