RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

duplicatesfilter: add a singeton duplicates filter and adapt current middleware and redirection middleware 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40889
This commit is contained in:
Daniel Grana 2009-02-19 19:43:24 +00:00
parent ac735a185e
commit 0e98526daa
5 changed files with 28 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/trunk/scrapy/conf/default_settings.py
View File

@ -76,7 +76,7 @@ DOWNLOADER_MIDDLEWARES = [
DOWNLOADER_STATS = True
DUPLICATESFILTER_FILTERCLASS = 'scrapy.contrib.spidermiddleware.duplicatesfilter.SimplePerDomainFilter'
DUPLICATESFILTER_FILTERCLASS = 'scrapy.core.filters.SimplePerDomainFilter'
ENABLED_SPIDERS_FILE = ''

22
scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -1,10 +1,15 @@
import re
from pydispatch import dispatcher
from scrapy.core import signals
from scrapy import log
from scrapy.http import Request, Response
from scrapy.core.exceptions import HttpException
from scrapy.core.filters import duplicatesfilter
from scrapy.utils.misc import load_object
from scrapy.utils.url import urljoin_rfc as urljoin
from scrapy.utils.response import get_meta_refresh
from scrapy.conf import settings
class RedirectLoop(Exception):
pass
@ -19,26 +24,31 @@ class RedirectMiddleware(object):
if not isinstance(exception, HttpException):
return
domain = spider.domain_name
status = exception.status
response = exception.response
if status in set([302, 303]):
redirected_url = urljoin(request.url, response.headers['location'][0])
redirected = request.replace(url=redirected_url, method='GET', body=None)
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
return self._redirect(redirected, request, spider, status)
if status in [301, 307]:
redirected_url = urljoin(request.url, response.headers['location'][0])
redirected = request.replace(url=redirected_url)
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
return self._redirect(redirected, request, spider, status)
def process_response(self, request, response, spider):
interval, url = get_meta_refresh(response)
if url and int(interval) < META_REFRESH_MAXSEC:
redirected = request.replace(url=urljoin(request.url, url))
log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
return self._redirect(redirected, request, spider, 'meta refresh') or response
return response
def _redirect(self, redirected, request, spider, reason):
domain = spider.domain_name
if duplicatesfilter.add(domain, redirected) or redirected.dont_filter:
log.msg("Redirecting (%s) to %s from %s" % (reason, redirected, request), level=log.DEBUG, domain=domain)
return redirected

55
scrapy/trunk/scrapy/contrib/spidermiddleware/duplicatesfilter.py
View File

@ -2,73 +2,26 @@
DuplicatesFilterMiddleware: Filter out already visited urls
"""
from pydispatch import dispatcher
from scrapy.core import signals
from scrapy.http import Request
from scrapy.core.exceptions import NotConfigured, IgnoreRequest
from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import load_object
from scrapy.conf import settings
from scrapy.core.filters import duplicatesfilter
from scrapy import log
class DuplicatesFilterMiddleware(object):
"""Filter out duplicate requests to avoid visiting same page more than once.
filter class (defined by DUPLICATESFILTER_FILTERCLASS setting) must
inplement a simple API:
* open(domain) called when a new domain starts
* close(domain) called when a domain is going to be closed.
* add(domain, request) called each time a new request needs to be tested
looking if it is was already seen or is a new one. This is the most
important method for a filtering class.
"""
def __init__(self):
clspath = settings.get('DUPLICATESFILTER_FILTERCLASS')
if not clspath:
raise NotConfigured
self.filter = load_object(clspath)()
dispatcher.connect(self.filter.open, signals.domain_open)
dispatcher.connect(self.filter.close, signals.domain_closed)
"""Filter out already seen requests to avoid visiting pages more than once."""
def process_spider_input(self, response, spider):
self.filter.add(spider.domain_name, response.request)
duplicatesfilter.add(spider.domain_name, response.request)
def process_spider_output(self, response, result, spider):
domain = spider.domain_name
for req in result:
if isinstance(req, Request):
added = self.filter.add(domain, req)
added = duplicatesfilter.add(domain, req)
if not (added or req.dont_filter):
log.msg('Skipped (already processed): %s' % req, log.TRACE, domain=domain)
continue
yield req
class SimplePerDomainFilter(dict):
"""Filter out a request if already seen for same domain"""
def open(self, domain):
"""Initialize the resources needed for filtering for this domain"""
self[domain] = set()
def close(self, domain):
"""Remove the resources reserved for filtering for this domain"""
del self[domain]
def add(self, domain, request):
"""Add the fingerprint of a request to the domain set if a equivalent fingerprint has not been added.
This method will return true if the fingerprint was added and false otherwise.
"""
fp = request_fingerprint(request)
if fp not in self[domain]:
self[domain].add(fp)
return True
return False

6
scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py
View File

@ -2,6 +2,7 @@ import unittest
from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware
from scrapy.core.exceptions import HttpException
from scrapy.core.filters import duplicatesfilter
from scrapy.spider import spiders
from scrapy.http import Request, Response, Headers
@ -11,6 +12,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
spiders.spider_modules = ['scrapy.tests.test_spiders']
spiders.reload()
self.spider = spiders.fromdomain('scrapytest.org')
duplicatesfilter.open('scrapytest.org')
def test_process_exception(self):
@ -28,7 +30,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
self.assertEqual(req2.url, url2)
url = 'http://www.example.com/302'
url2 = 'http://www.example.com/redirected'
url2 = 'http://www.example.com/redirected2'
req = Request(url, method='POST')
hdr = Headers({'Location': [url2]})
rsp = Response(url, headers=hdr)
@ -43,7 +45,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
def test_process_response(self):
mw = RedirectMiddleware()
body = """<html>
<head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
</html>"""

28
scrapy/trunk/scrapy/tests/test_spidermiddleware_duplicatesfilter.py
View File

@ -3,7 +3,8 @@ import unittest
from scrapy.spider import spiders
from scrapy.http import Request, Response
from scrapy.core.exceptions import IgnoreRequest
from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware, SimplePerDomainFilter
from scrapy.contrib.spidermiddleware.duplicatesfilter import DuplicatesFilterMiddleware
from scrapy.core.filters import duplicatesfilter
class DuplicatesFilterMiddlewareTest(unittest.TestCase):
@ -11,10 +12,10 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
spiders.spider_modules = ['scrapy.tests.test_spiders']
spiders.reload()
self.spider = spiders.fromdomain('scrapytest.org')
duplicatesfilter.open('scrapytest.org')
def test_process_spider_output(self):
mw = DuplicatesFilterMiddleware()
mw.filter.open('scrapytest.org')
response = Response('http://scrapytest.org/')
response.request = Request('http://scrapytest.org/')
@ -31,26 +32,3 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
assert r1 in filtered
assert r2 in filtered
assert r3 not in filtered
mw.filter.close('scrapytest.org')
class SimplePerDomainFilterTest(unittest.TestCase):
def test_filter(self):
domain = 'scrapytest.org'
filter = SimplePerDomainFilter()
filter.open(domain)
assert domain in filter
r1 = Request('http://scrapytest.org/1')
r2 = Request('http://scrapytest.org/2')
r3 = Request('http://scrapytest.org/2')
assert filter.add(domain, r1)
assert filter.add(domain, r2)
assert not filter.add(domain, r3)
filter.close(domain)
assert domain not in filter