mirror of https://github.com/scrapy/scrapy.git
removed experimental crawlspider v2
This commit is contained in:
Pablo Hoffman
parent
5bf733b6f6
commit
03ae481cad
|
|
@ -1,128 +0,0 @@
|
|||
.. _topics-crawlspider-v2:
|
||||
|
||||
==============
|
||||
CrawlSpider v2
|
||||
==============
|
||||
|
||||
Introduction
|
||||
============
|
||||
|
||||
TODO: introduction
|
||||
|
||||
Rules Matching
|
||||
==============
|
||||
|
||||
TODO: describe purpose of rules
|
||||
|
||||
Request Extractors & Processors
|
||||
===============================
|
||||
|
||||
TODO: describe purpose of extractors & processors
|
||||
|
||||
Examples
|
||||
========
|
||||
|
||||
TODO: plenty of examples
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.spider
|
||||
:synopsis: CrawlSpider
|
||||
|
||||
|
||||
Reference
|
||||
=========
|
||||
|
||||
CrawlSpider
|
||||
-----------
|
||||
|
||||
TODO: describe crawlspider
|
||||
|
||||
.. class:: CrawlSpider
|
||||
|
||||
TODO: describe class
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.rules
|
||||
:synopsis: Rules
|
||||
|
||||
Rules
|
||||
-----
|
||||
|
||||
TODO: describe spider rules
|
||||
|
||||
.. class:: Rule
|
||||
|
||||
TODO: describe Rules class
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.reqext
|
||||
:synopsis: Request Extractors
|
||||
|
||||
Request Extractors
|
||||
------------------
|
||||
|
||||
TODO: describe extractors purpose
|
||||
|
||||
.. class:: BaseSgmlRequestExtractor
|
||||
|
||||
TODO: describe base extractor
|
||||
|
||||
.. class:: SgmlRequestExtractor
|
||||
|
||||
TODO: describe sgml extractor
|
||||
|
||||
.. class:: XPathRequestExtractor
|
||||
|
||||
TODO: describe xpath request extractor
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.reqproc
|
||||
:synopsis: Request Processors
|
||||
|
||||
Request Processors
|
||||
------------------
|
||||
|
||||
TODO: describe request processors
|
||||
|
||||
.. class:: Canonicalize
|
||||
|
||||
TODO: describe proc
|
||||
|
||||
.. class:: Unique
|
||||
|
||||
TODO: describe unique
|
||||
|
||||
.. class:: FilterDomain
|
||||
|
||||
TODO: describe filter domain
|
||||
|
||||
.. class:: FilterUrl
|
||||
|
||||
TODO: describe filter url
|
||||
|
||||
|
||||
.. module:: scrapy.contrib_exp.crawlspider.matchers
|
||||
:synopsis: Matchers
|
||||
|
||||
Request/Response Matchers
|
||||
-------------------------
|
||||
|
||||
TODO: describe matchers
|
||||
|
||||
.. class:: BaseMatcher
|
||||
|
||||
TODO: describe base matcher
|
||||
|
||||
.. class:: UrlMatcher
|
||||
|
||||
TODO: describe url matcher
|
||||
|
||||
.. class:: UrlRegexMatcher
|
||||
|
||||
TODO: describe UrlListMatcher
|
||||
|
||||
.. class:: UrlListMatcher
|
||||
|
||||
TODO: describe url list matcher
|
||||
|
||||
|
||||
|
|
@ -20,4 +20,3 @@ it's properly merged) . Use at your own risk.
|
|||
:maxdepth: 1
|
||||
|
||||
djangoitems
|
||||
crawlspider-v2
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
"""CrawlSpider v2"""
|
||||
|
||||
from .rules import Rule
|
||||
from .spider import CrawlSpider
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
"""
|
||||
Request/Response Matchers
|
||||
|
||||
Perform evaluation to Request or Response attributes
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
class BaseMatcher(object):
|
||||
"""Base matcher. Returns True by default."""
|
||||
|
||||
def matches_request(self, request):
|
||||
"""Performs Request Matching"""
|
||||
return True
|
||||
|
||||
def matches_response(self, response):
|
||||
"""Performs Response Matching"""
|
||||
return True
|
||||
|
||||
|
||||
class UrlMatcher(BaseMatcher):
|
||||
"""Matches URL attribute"""
|
||||
|
||||
def __init__(self, url):
|
||||
"""Initialize url attribute"""
|
||||
self._url = url
|
||||
|
||||
def matches_url(self, url):
|
||||
"""Returns True if given url is equal to matcher's url"""
|
||||
return self._url == url
|
||||
|
||||
def matches_request(self, request):
|
||||
"""Returns True if Request's url matches initial url"""
|
||||
return self.matches_url(request.url)
|
||||
|
||||
def matches_response(self, response):
|
||||
"""Returns True if Response's url matches initial url"""
|
||||
return self.matches_url(response.url)
|
||||
|
||||
|
||||
class UrlRegexMatcher(UrlMatcher):
|
||||
"""Matches URL using regular expression"""
|
||||
|
||||
def __init__(self, regex, flags=0):
|
||||
"""Initialize regular expression"""
|
||||
self._regex = re.compile(regex, flags)
|
||||
|
||||
def matches_url(self, url):
|
||||
"""Returns True if url matches regular expression"""
|
||||
return self._regex.search(url) is not None
|
||||
|
||||
|
||||
class UrlListMatcher(UrlMatcher):
|
||||
"""Matches if URL is in List"""
|
||||
|
||||
def __init__(self, urls):
|
||||
self._urls = urls
|
||||
|
||||
def matches_url(self, url):
|
||||
"""Returns True if url is in urls list"""
|
||||
return url in self._urls
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
"""Request Extractors"""
|
||||
from w3lib.url import safe_url_string, urljoin_rfc
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.python import FixedSGMLParser, str_to_unicode
|
||||
|
||||
from itertools import ifilter
|
||||
|
||||
|
||||
class BaseSgmlRequestExtractor(FixedSGMLParser):
|
||||
"""Base SGML Request Extractor"""
|
||||
|
||||
def __init__(self, tag='a', attr='href'):
|
||||
"""Initialize attributes"""
|
||||
FixedSGMLParser.__init__(self)
|
||||
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.current_request = None
|
||||
|
||||
def extract_requests(self, response):
|
||||
"""Returns list of requests extracted from response"""
|
||||
return self._extract_requests(response.body, response.url,
|
||||
response.encoding)
|
||||
|
||||
def _extract_requests(self, response_text, response_url, response_encoding):
|
||||
"""Extract requests with absolute urls"""
|
||||
self.reset()
|
||||
self.feed(response_text)
|
||||
self.close()
|
||||
|
||||
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
|
||||
self._make_absolute_urls(base_url, response_encoding)
|
||||
self._fix_link_text_encoding(response_encoding)
|
||||
|
||||
return self.requests
|
||||
|
||||
def _make_absolute_urls(self, base_url, encoding):
|
||||
"""Makes all request's urls absolute"""
|
||||
self.requests = [x.replace(url=safe_url_string(urljoin_rfc(base_url, \
|
||||
x.url, encoding), encoding)) for x in self.requests]
|
||||
|
||||
def _fix_link_text_encoding(self, encoding):
|
||||
"""Convert link_text to unicode for each request"""
|
||||
for req in self.requests:
|
||||
req.meta.setdefault('link_text', '')
|
||||
req.meta['link_text'] = str_to_unicode(req.meta['link_text'],
|
||||
encoding)
|
||||
|
||||
def reset(self):
|
||||
"""Reset state"""
|
||||
FixedSGMLParser.reset(self)
|
||||
self.requests = []
|
||||
self.base_url = None
|
||||
|
||||
def unknown_starttag(self, tag, attrs):
|
||||
"""Process unknown start tag"""
|
||||
if 'base' == tag:
|
||||
self.base_url = dict(attrs).get('href')
|
||||
|
||||
_matches = lambda (attr, value): self.scan_attr(attr) \
|
||||
and value is not None
|
||||
if self.scan_tag(tag):
|
||||
for attr, value in ifilter(_matches, attrs):
|
||||
req = Request(url=value)
|
||||
self.requests.append(req)
|
||||
self.current_request = req
|
||||
|
||||
def unknown_endtag(self, tag):
|
||||
"""Process unknown end tag"""
|
||||
self.current_request = None
|
||||
|
||||
def handle_data(self, data):
|
||||
"""Process data"""
|
||||
current = self.current_request
|
||||
if current and not 'link_text' in current.meta:
|
||||
current.meta['link_text'] = data.strip()
|
||||
|
||||
|
||||
class SgmlRequestExtractor(BaseSgmlRequestExtractor):
|
||||
"""SGML Request Extractor"""
|
||||
|
||||
def __init__(self, tags=None, attrs=None):
|
||||
"""Initialize with custom tag & attribute function checkers"""
|
||||
# defaults
|
||||
tags = tuple(tags) if tags else ('a', 'area')
|
||||
attrs = tuple(attrs) if attrs else ('href', )
|
||||
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func)
|
||||
|
||||
# TODO: move to own file
|
||||
class XPathRequestExtractor(SgmlRequestExtractor):
|
||||
"""SGML Request Extractor with XPath restriction"""
|
||||
|
||||
def __init__(self, restrict_xpaths, tags=None, attrs=None):
|
||||
"""Initialize XPath restrictions"""
|
||||
self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
|
||||
SgmlRequestExtractor.__init__(self, tags, attrs)
|
||||
|
||||
def extract_requests(self, response):
|
||||
"""Restrict to XPath regions"""
|
||||
hxs = HtmlXPathSelector(response)
|
||||
fragments = (''.join(
|
||||
html_frag for html_frag in hxs.select(xpath).extract()
|
||||
) for xpath in self.restrict_xpaths)
|
||||
html_slice = ''.join(html_frag for html_frag in fragments)
|
||||
return self._extract_requests(html_slice, response.url,
|
||||
response.encoding)
|
||||
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
"""Request Generator"""
|
||||
from itertools import imap
|
||||
|
||||
class RequestGenerator(object):
|
||||
"""Extracto and process requests from response"""
|
||||
|
||||
def __init__(self, req_extractors, req_processors, callback, spider=None):
|
||||
"""Initialize attributes"""
|
||||
self._request_extractors = req_extractors
|
||||
self._request_processors = req_processors
|
||||
#TODO: resolve callback?
|
||||
self._callback = callback
|
||||
|
||||
def generate_requests(self, response):
|
||||
"""Extract and process new requests from response.
|
||||
Attach callback to each request as default callback."""
|
||||
requests = []
|
||||
for ext in self._request_extractors:
|
||||
requests.extend(ext.extract_requests(response))
|
||||
|
||||
for proc in self._request_processors:
|
||||
requests = proc(requests)
|
||||
|
||||
# return iterator
|
||||
# @@@ creates new Request object with callback
|
||||
return imap(lambda r: r.replace(callback=self._callback), requests)
|
||||
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
"""Request Processors"""
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain
|
||||
|
||||
from itertools import ifilter
|
||||
|
||||
import re
|
||||
|
||||
class Canonicalize(object):
|
||||
"""Canonicalize Request Processor"""
|
||||
|
||||
def __call__(self, requests):
|
||||
"""Canonicalize all requests' urls"""
|
||||
return (x.replace(url=canonicalize_url(x.url)) for x in requests)
|
||||
|
||||
|
||||
class FilterDupes(object):
|
||||
"""Filter duplicate Requests"""
|
||||
|
||||
def __init__(self, *attributes):
|
||||
"""Initialize comparison attributes"""
|
||||
self._attributes = tuple(attributes) if attributes \
|
||||
else tuple(['url'])
|
||||
|
||||
def _equal_attr(self, obj1, obj2, attr):
|
||||
return getattr(obj1, attr) == getattr(obj2, attr)
|
||||
|
||||
def _requests_equal(self, req1, req2):
|
||||
"""Attribute comparison helper"""
|
||||
# look for not equal attribute
|
||||
_not_equal = lambda attr: not self._equal_attr(req1, req2, attr)
|
||||
for attr in ifilter(_not_equal, self._attributes):
|
||||
return False
|
||||
# all attributes equal
|
||||
return True
|
||||
|
||||
def _request_in(self, request, requests_seen):
|
||||
"""Check if request is in given requests seen list"""
|
||||
_req_seen = lambda r: self._requests_equal(r, request)
|
||||
for seen in ifilter(_req_seen, requests_seen):
|
||||
return True
|
||||
# request not seen
|
||||
return False
|
||||
|
||||
def __call__(self, requests):
|
||||
"""Filter seen requests"""
|
||||
# per-call duplicates filter
|
||||
self.requests_seen = set()
|
||||
_not_seen = lambda r: not self._request_in(r, self.requests_seen)
|
||||
for req in ifilter(_not_seen, requests):
|
||||
yield req
|
||||
# registry seen request
|
||||
self.requests_seen.add(req)
|
||||
|
||||
|
||||
class FilterDomain(object):
|
||||
"""Filter request's domain"""
|
||||
|
||||
def __init__(self, allow=(), deny=()):
|
||||
"""Initialize allow/deny attributes"""
|
||||
self.allow = tuple(arg_to_iter(allow))
|
||||
self.deny = tuple(arg_to_iter(deny))
|
||||
|
||||
def __call__(self, requests):
|
||||
"""Filter domains"""
|
||||
processed = (req for req in requests)
|
||||
|
||||
if self.allow:
|
||||
processed = (req for req in requests
|
||||
if url_is_from_any_domain(req.url, self.allow))
|
||||
if self.deny:
|
||||
processed = (req for req in requests
|
||||
if not url_is_from_any_domain(req.url, self.deny))
|
||||
|
||||
return processed
|
||||
|
||||
|
||||
class FilterUrl(object):
|
||||
"""Filter request's url"""
|
||||
|
||||
def __init__(self, allow=(), deny=()):
|
||||
"""Initialize allow/deny attributes"""
|
||||
_re_type = type(re.compile('', 0))
|
||||
|
||||
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x)
|
||||
for x in arg_to_iter(allow)]
|
||||
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x)
|
||||
for x in arg_to_iter(deny)]
|
||||
|
||||
def __call__(self, requests):
|
||||
"""Filter request's url based on allow/deny rules"""
|
||||
#TODO: filter valid urls here?
|
||||
processed = (req for req in requests)
|
||||
|
||||
if self.allow_res:
|
||||
processed = (req for req in requests
|
||||
if self._matches(req.url, self.allow_res))
|
||||
if self.deny_res:
|
||||
processed = (req for req in requests
|
||||
if not self._matches(req.url, self.deny_res))
|
||||
|
||||
return processed
|
||||
|
||||
def _matches(self, url, regexs):
|
||||
"""Returns True if url matches any regex in given list"""
|
||||
return any(r.search(url) for r in regexs)
|
||||
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
"""Crawler Rules"""
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Response
|
||||
|
||||
from functools import partial
|
||||
from itertools import ifilter
|
||||
|
||||
from .matchers import BaseMatcher
|
||||
# default strint-to-matcher class
|
||||
from .matchers import UrlRegexMatcher
|
||||
|
||||
class CompiledRule(object):
|
||||
"""Compiled version of Rule"""
|
||||
def __init__(self, matcher, callback=None, follow=False):
|
||||
"""Initialize attributes checking type"""
|
||||
assert isinstance(matcher, BaseMatcher)
|
||||
assert callback is None or callable(callback)
|
||||
assert isinstance(follow, bool)
|
||||
|
||||
self.matcher = matcher
|
||||
self.callback = callback
|
||||
self.follow = follow
|
||||
|
||||
|
||||
class Rule(object):
|
||||
"""Crawler Rule"""
|
||||
def __init__(self, matcher=None, callback=None, follow=False, **kwargs):
|
||||
"""Store attributes"""
|
||||
self.matcher = matcher
|
||||
self.callback = callback
|
||||
self.cb_kwargs = kwargs if kwargs else {}
|
||||
self.follow = True if follow else False
|
||||
|
||||
if self.callback is None and self.follow is False:
|
||||
raise ValueError("Rule must either have a callback or "
|
||||
"follow=True: %r" % self)
|
||||
|
||||
def __repr__(self):
|
||||
return "Rule(matcher=%r, callback=%r, follow=%r, **%r)" \
|
||||
% (self.matcher, self.callback, self.follow, self.cb_kwargs)
|
||||
|
||||
|
||||
class RulesManager(object):
|
||||
"""Rules Manager"""
|
||||
def __init__(self, rules, spider, default_matcher=UrlRegexMatcher):
|
||||
"""Initialize rules using spider and default matcher"""
|
||||
self._rules = tuple()
|
||||
|
||||
# compile absolute/relative-to-spider callbacks"""
|
||||
for rule in rules:
|
||||
# prepare matcher
|
||||
if rule.matcher is None:
|
||||
# instance BaseMatcher by default
|
||||
matcher = BaseMatcher()
|
||||
elif isinstance(rule.matcher, BaseMatcher):
|
||||
matcher = rule.matcher
|
||||
else:
|
||||
# matcher not BaseMatcher, check for string
|
||||
if isinstance(rule.matcher, basestring):
|
||||
# instance default matcher
|
||||
matcher = default_matcher(rule.matcher)
|
||||
else:
|
||||
raise ValueError('Not valid matcher given %r in %r' \
|
||||
% (rule.matcher, rule))
|
||||
|
||||
# prepare callback
|
||||
if callable(rule.callback):
|
||||
callback = rule.callback
|
||||
elif not rule.callback is None:
|
||||
# callback from spider
|
||||
callback = getattr(spider, rule.callback)
|
||||
|
||||
if not callable(callback):
|
||||
raise AttributeError('Invalid callback %r can not be resolved' \
|
||||
% callback)
|
||||
else:
|
||||
callback = None
|
||||
|
||||
if rule.cb_kwargs:
|
||||
# build partial callback
|
||||
callback = partial(callback, **rule.cb_kwargs)
|
||||
|
||||
# append compiled rule to rules list
|
||||
crule = CompiledRule(matcher, callback, follow=rule.follow)
|
||||
self._rules += (crule, )
|
||||
|
||||
def get_rule_from_request(self, request):
|
||||
"""Returns first rule that matches given Request"""
|
||||
_matches = lambda r: r.matcher.matches_request(request)
|
||||
for rule in ifilter(_matches, self._rules):
|
||||
# return first match of iterator
|
||||
return rule
|
||||
|
||||
def get_rule_from_response(self, response):
|
||||
"""Returns first rule that matches given Response"""
|
||||
_matches = lambda r: r.matcher.matches_response(response)
|
||||
for rule in ifilter(_matches, self._rules):
|
||||
# return first match of iterator
|
||||
return rule
|
||||
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
"""CrawlSpider v2"""
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
from .matchers import UrlListMatcher
|
||||
from .rules import Rule, RulesManager
|
||||
from .reqext import SgmlRequestExtractor
|
||||
from .reqgen import RequestGenerator
|
||||
from .reqproc import Canonicalize, FilterDupes
|
||||
|
||||
class CrawlSpider(BaseSpider):
|
||||
"""CrawlSpider v2"""
|
||||
|
||||
request_extractors = None
|
||||
request_processors = None
|
||||
rules = []
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
"""Initialize dispatcher"""
|
||||
super(CrawlSpider, self).__init__(*a, **kw)
|
||||
|
||||
# auto follow start urls
|
||||
if self.start_urls:
|
||||
_matcher = UrlListMatcher(self.start_urls)
|
||||
# append new rule using type from current self.rules
|
||||
rules = self.rules + type(self.rules)([
|
||||
Rule(_matcher, follow=True)
|
||||
])
|
||||
else:
|
||||
rules = self.rules
|
||||
|
||||
# set defaults if not set
|
||||
if self.request_extractors is None:
|
||||
# default link extractor. Extracts all links from response
|
||||
self.request_extractors = [ SgmlRequestExtractor() ]
|
||||
|
||||
if self.request_processors is None:
|
||||
# default proccessor. Filter duplicates requests
|
||||
self.request_processors = [ FilterDupes() ]
|
||||
|
||||
|
||||
# wrap rules
|
||||
self._rulesman = RulesManager(rules, spider=self)
|
||||
# generates new requests with given callback
|
||||
self._reqgen = RequestGenerator(self.request_extractors,
|
||||
self.request_processors,
|
||||
callback=self.parse)
|
||||
|
||||
def parse(self, response):
|
||||
"""Dispatch callback and generate requests"""
|
||||
# get rule for response
|
||||
rule = self._rulesman.get_rule_from_response(response)
|
||||
|
||||
if rule:
|
||||
# dispatch callback if set
|
||||
if rule.callback:
|
||||
output = iterate_spider_output(rule.callback(response))
|
||||
for req_or_item in output:
|
||||
yield req_or_item
|
||||
|
||||
if rule.follow:
|
||||
for req in self._reqgen.generate_requests(response):
|
||||
# only dispatch request if has matching rule
|
||||
if self._rulesman.get_rule_from_request(req):
|
||||
yield req
|
||||
else:
|
||||
self.log("No rule for response %s" % response, level=log.WARNING)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue