diff --git a/docs/experimental/crawlspider-v2.rst b/docs/experimental/crawlspider-v2.rst deleted file mode 100644 index a1ea09c48..000000000 --- a/docs/experimental/crawlspider-v2.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _topics-crawlspider-v2: - -============== -CrawlSpider v2 -============== - -Introduction -============ - -TODO: introduction - -Rules Matching -============== - -TODO: describe purpose of rules - -Request Extractors & Processors -=============================== - -TODO: describe purpose of extractors & processors - -Examples -======== - -TODO: plenty of examples - - -.. module:: scrapy.contrib_exp.crawlspider.spider - :synopsis: CrawlSpider - - -Reference -========= - -CrawlSpider ------------ - -TODO: describe crawlspider - -.. class:: CrawlSpider - - TODO: describe class - - -.. module:: scrapy.contrib_exp.crawlspider.rules - :synopsis: Rules - -Rules ------ - -TODO: describe spider rules - -.. class:: Rule - - TODO: describe Rules class - - -.. module:: scrapy.contrib_exp.crawlspider.reqext - :synopsis: Request Extractors - -Request Extractors ------------------- - -TODO: describe extractors purpose - -.. class:: BaseSgmlRequestExtractor - - TODO: describe base extractor - -.. class:: SgmlRequestExtractor - - TODO: describe sgml extractor - -.. class:: XPathRequestExtractor - - TODO: describe xpath request extractor - - -.. module:: scrapy.contrib_exp.crawlspider.reqproc - :synopsis: Request Processors - -Request Processors ------------------- - -TODO: describe request processors - -.. class:: Canonicalize - - TODO: describe proc - -.. class:: Unique - - TODO: describe unique - -.. class:: FilterDomain - - TODO: describe filter domain - -.. class:: FilterUrl - - TODO: describe filter url - - -.. module:: scrapy.contrib_exp.crawlspider.matchers - :synopsis: Matchers - -Request/Response Matchers -------------------------- - -TODO: describe matchers - -.. class:: BaseMatcher - - TODO: describe base matcher - -.. class:: UrlMatcher - - TODO: describe url matcher - -.. class:: UrlRegexMatcher - - TODO: describe UrlListMatcher - -.. class:: UrlListMatcher - - TODO: describe url list matcher - - diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index f159e8c51..c28cdd9ed 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -20,4 +20,3 @@ it's properly merged) . Use at your own risk. :maxdepth: 1 djangoitems - crawlspider-v2 diff --git a/scrapy/contrib_exp/crawlspider/__init__.py b/scrapy/contrib_exp/crawlspider/__init__.py deleted file mode 100644 index 03173eb38..000000000 --- a/scrapy/contrib_exp/crawlspider/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""CrawlSpider v2""" - -from .rules import Rule -from .spider import CrawlSpider diff --git a/scrapy/contrib_exp/crawlspider/matchers.py b/scrapy/contrib_exp/crawlspider/matchers.py deleted file mode 100644 index 3ef259c67..000000000 --- a/scrapy/contrib_exp/crawlspider/matchers.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -Request/Response Matchers - -Perform evaluation to Request or Response attributes -""" - -import re - -class BaseMatcher(object): - """Base matcher. Returns True by default.""" - - def matches_request(self, request): - """Performs Request Matching""" - return True - - def matches_response(self, response): - """Performs Response Matching""" - return True - - -class UrlMatcher(BaseMatcher): - """Matches URL attribute""" - - def __init__(self, url): - """Initialize url attribute""" - self._url = url - - def matches_url(self, url): - """Returns True if given url is equal to matcher's url""" - return self._url == url - - def matches_request(self, request): - """Returns True if Request's url matches initial url""" - return self.matches_url(request.url) - - def matches_response(self, response): - """Returns True if Response's url matches initial url""" - return self.matches_url(response.url) - - -class UrlRegexMatcher(UrlMatcher): - """Matches URL using regular expression""" - - def __init__(self, regex, flags=0): - """Initialize regular expression""" - self._regex = re.compile(regex, flags) - - def matches_url(self, url): - """Returns True if url matches regular expression""" - return self._regex.search(url) is not None - - -class UrlListMatcher(UrlMatcher): - """Matches if URL is in List""" - - def __init__(self, urls): - self._urls = urls - - def matches_url(self, url): - """Returns True if url is in urls list""" - return url in self._urls diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py deleted file mode 100644 index eb6e32f2e..000000000 --- a/scrapy/contrib_exp/crawlspider/reqext.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Request Extractors""" -from w3lib.url import safe_url_string, urljoin_rfc - -from scrapy.http import Request -from scrapy.selector import HtmlXPathSelector -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import FixedSGMLParser, str_to_unicode - -from itertools import ifilter - - -class BaseSgmlRequestExtractor(FixedSGMLParser): - """Base SGML Request Extractor""" - - def __init__(self, tag='a', attr='href'): - """Initialize attributes""" - FixedSGMLParser.__init__(self) - - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.current_request = None - - def extract_requests(self, response): - """Returns list of requests extracted from response""" - return self._extract_requests(response.body, response.url, - response.encoding) - - def _extract_requests(self, response_text, response_url, response_encoding): - """Extract requests with absolute urls""" - self.reset() - self.feed(response_text) - self.close() - - base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url - self._make_absolute_urls(base_url, response_encoding) - self._fix_link_text_encoding(response_encoding) - - return self.requests - - def _make_absolute_urls(self, base_url, encoding): - """Makes all request's urls absolute""" - self.requests = [x.replace(url=safe_url_string(urljoin_rfc(base_url, \ - x.url, encoding), encoding)) for x in self.requests] - - def _fix_link_text_encoding(self, encoding): - """Convert link_text to unicode for each request""" - for req in self.requests: - req.meta.setdefault('link_text', '') - req.meta['link_text'] = str_to_unicode(req.meta['link_text'], - encoding) - - def reset(self): - """Reset state""" - FixedSGMLParser.reset(self) - self.requests = [] - self.base_url = None - - def unknown_starttag(self, tag, attrs): - """Process unknown start tag""" - if 'base' == tag: - self.base_url = dict(attrs).get('href') - - _matches = lambda (attr, value): self.scan_attr(attr) \ - and value is not None - if self.scan_tag(tag): - for attr, value in ifilter(_matches, attrs): - req = Request(url=value) - self.requests.append(req) - self.current_request = req - - def unknown_endtag(self, tag): - """Process unknown end tag""" - self.current_request = None - - def handle_data(self, data): - """Process data""" - current = self.current_request - if current and not 'link_text' in current.meta: - current.meta['link_text'] = data.strip() - - -class SgmlRequestExtractor(BaseSgmlRequestExtractor): - """SGML Request Extractor""" - - def __init__(self, tags=None, attrs=None): - """Initialize with custom tag & attribute function checkers""" - # defaults - tags = tuple(tags) if tags else ('a', 'area') - attrs = tuple(attrs) if attrs else ('href', ) - - tag_func = lambda x: x in tags - attr_func = lambda x: x in attrs - BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) - -# TODO: move to own file -class XPathRequestExtractor(SgmlRequestExtractor): - """SGML Request Extractor with XPath restriction""" - - def __init__(self, restrict_xpaths, tags=None, attrs=None): - """Initialize XPath restrictions""" - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - SgmlRequestExtractor.__init__(self, tags, attrs) - - def extract_requests(self, response): - """Restrict to XPath regions""" - hxs = HtmlXPathSelector(response) - fragments = (''.join( - html_frag for html_frag in hxs.select(xpath).extract() - ) for xpath in self.restrict_xpaths) - html_slice = ''.join(html_frag for html_frag in fragments) - return self._extract_requests(html_slice, response.url, - response.encoding) - diff --git a/scrapy/contrib_exp/crawlspider/reqgen.py b/scrapy/contrib_exp/crawlspider/reqgen.py deleted file mode 100644 index 3858fbcf7..000000000 --- a/scrapy/contrib_exp/crawlspider/reqgen.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Request Generator""" -from itertools import imap - -class RequestGenerator(object): - """Extracto and process requests from response""" - - def __init__(self, req_extractors, req_processors, callback, spider=None): - """Initialize attributes""" - self._request_extractors = req_extractors - self._request_processors = req_processors - #TODO: resolve callback? - self._callback = callback - - def generate_requests(self, response): - """Extract and process new requests from response. - Attach callback to each request as default callback.""" - requests = [] - for ext in self._request_extractors: - requests.extend(ext.extract_requests(response)) - - for proc in self._request_processors: - requests = proc(requests) - - # return iterator - # @@@ creates new Request object with callback - return imap(lambda r: r.replace(callback=self._callback), requests) - diff --git a/scrapy/contrib_exp/crawlspider/reqproc.py b/scrapy/contrib_exp/crawlspider/reqproc.py deleted file mode 100644 index adc07005a..000000000 --- a/scrapy/contrib_exp/crawlspider/reqproc.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Request Processors""" -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import canonicalize_url, url_is_from_any_domain - -from itertools import ifilter - -import re - -class Canonicalize(object): - """Canonicalize Request Processor""" - - def __call__(self, requests): - """Canonicalize all requests' urls""" - return (x.replace(url=canonicalize_url(x.url)) for x in requests) - - -class FilterDupes(object): - """Filter duplicate Requests""" - - def __init__(self, *attributes): - """Initialize comparison attributes""" - self._attributes = tuple(attributes) if attributes \ - else tuple(['url']) - - def _equal_attr(self, obj1, obj2, attr): - return getattr(obj1, attr) == getattr(obj2, attr) - - def _requests_equal(self, req1, req2): - """Attribute comparison helper""" - # look for not equal attribute - _not_equal = lambda attr: not self._equal_attr(req1, req2, attr) - for attr in ifilter(_not_equal, self._attributes): - return False - # all attributes equal - return True - - def _request_in(self, request, requests_seen): - """Check if request is in given requests seen list""" - _req_seen = lambda r: self._requests_equal(r, request) - for seen in ifilter(_req_seen, requests_seen): - return True - # request not seen - return False - - def __call__(self, requests): - """Filter seen requests""" - # per-call duplicates filter - self.requests_seen = set() - _not_seen = lambda r: not self._request_in(r, self.requests_seen) - for req in ifilter(_not_seen, requests): - yield req - # registry seen request - self.requests_seen.add(req) - - -class FilterDomain(object): - """Filter request's domain""" - - def __init__(self, allow=(), deny=()): - """Initialize allow/deny attributes""" - self.allow = tuple(arg_to_iter(allow)) - self.deny = tuple(arg_to_iter(deny)) - - def __call__(self, requests): - """Filter domains""" - processed = (req for req in requests) - - if self.allow: - processed = (req for req in requests - if url_is_from_any_domain(req.url, self.allow)) - if self.deny: - processed = (req for req in requests - if not url_is_from_any_domain(req.url, self.deny)) - - return processed - - -class FilterUrl(object): - """Filter request's url""" - - def __init__(self, allow=(), deny=()): - """Initialize allow/deny attributes""" - _re_type = type(re.compile('', 0)) - - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) - for x in arg_to_iter(deny)] - - def __call__(self, requests): - """Filter request's url based on allow/deny rules""" - #TODO: filter valid urls here? - processed = (req for req in requests) - - if self.allow_res: - processed = (req for req in requests - if self._matches(req.url, self.allow_res)) - if self.deny_res: - processed = (req for req in requests - if not self._matches(req.url, self.deny_res)) - - return processed - - def _matches(self, url, regexs): - """Returns True if url matches any regex in given list""" - return any(r.search(url) for r in regexs) - diff --git a/scrapy/contrib_exp/crawlspider/rules.py b/scrapy/contrib_exp/crawlspider/rules.py deleted file mode 100644 index ff1691ad0..000000000 --- a/scrapy/contrib_exp/crawlspider/rules.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Crawler Rules""" -from scrapy.http import Request -from scrapy.http import Response - -from functools import partial -from itertools import ifilter - -from .matchers import BaseMatcher -# default strint-to-matcher class -from .matchers import UrlRegexMatcher - -class CompiledRule(object): - """Compiled version of Rule""" - def __init__(self, matcher, callback=None, follow=False): - """Initialize attributes checking type""" - assert isinstance(matcher, BaseMatcher) - assert callback is None or callable(callback) - assert isinstance(follow, bool) - - self.matcher = matcher - self.callback = callback - self.follow = follow - - -class Rule(object): - """Crawler Rule""" - def __init__(self, matcher=None, callback=None, follow=False, **kwargs): - """Store attributes""" - self.matcher = matcher - self.callback = callback - self.cb_kwargs = kwargs if kwargs else {} - self.follow = True if follow else False - - if self.callback is None and self.follow is False: - raise ValueError("Rule must either have a callback or " - "follow=True: %r" % self) - - def __repr__(self): - return "Rule(matcher=%r, callback=%r, follow=%r, **%r)" \ - % (self.matcher, self.callback, self.follow, self.cb_kwargs) - - -class RulesManager(object): - """Rules Manager""" - def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): - """Initialize rules using spider and default matcher""" - self._rules = tuple() - - # compile absolute/relative-to-spider callbacks""" - for rule in rules: - # prepare matcher - if rule.matcher is None: - # instance BaseMatcher by default - matcher = BaseMatcher() - elif isinstance(rule.matcher, BaseMatcher): - matcher = rule.matcher - else: - # matcher not BaseMatcher, check for string - if isinstance(rule.matcher, basestring): - # instance default matcher - matcher = default_matcher(rule.matcher) - else: - raise ValueError('Not valid matcher given %r in %r' \ - % (rule.matcher, rule)) - - # prepare callback - if callable(rule.callback): - callback = rule.callback - elif not rule.callback is None: - # callback from spider - callback = getattr(spider, rule.callback) - - if not callable(callback): - raise AttributeError('Invalid callback %r can not be resolved' \ - % callback) - else: - callback = None - - if rule.cb_kwargs: - # build partial callback - callback = partial(callback, **rule.cb_kwargs) - - # append compiled rule to rules list - crule = CompiledRule(matcher, callback, follow=rule.follow) - self._rules += (crule, ) - - def get_rule_from_request(self, request): - """Returns first rule that matches given Request""" - _matches = lambda r: r.matcher.matches_request(request) - for rule in ifilter(_matches, self._rules): - # return first match of iterator - return rule - - def get_rule_from_response(self, response): - """Returns first rule that matches given Response""" - _matches = lambda r: r.matcher.matches_response(response) - for rule in ifilter(_matches, self._rules): - # return first match of iterator - return rule - diff --git a/scrapy/contrib_exp/crawlspider/spider.py b/scrapy/contrib_exp/crawlspider/spider.py deleted file mode 100644 index e2bf1a34b..000000000 --- a/scrapy/contrib_exp/crawlspider/spider.py +++ /dev/null @@ -1,69 +0,0 @@ -"""CrawlSpider v2""" -from scrapy.spider import BaseSpider -from scrapy.utils.spider import iterate_spider_output - -from .matchers import UrlListMatcher -from .rules import Rule, RulesManager -from .reqext import SgmlRequestExtractor -from .reqgen import RequestGenerator -from .reqproc import Canonicalize, FilterDupes - -class CrawlSpider(BaseSpider): - """CrawlSpider v2""" - - request_extractors = None - request_processors = None - rules = [] - - def __init__(self, *a, **kw): - """Initialize dispatcher""" - super(CrawlSpider, self).__init__(*a, **kw) - - # auto follow start urls - if self.start_urls: - _matcher = UrlListMatcher(self.start_urls) - # append new rule using type from current self.rules - rules = self.rules + type(self.rules)([ - Rule(_matcher, follow=True) - ]) - else: - rules = self.rules - - # set defaults if not set - if self.request_extractors is None: - # default link extractor. Extracts all links from response - self.request_extractors = [ SgmlRequestExtractor() ] - - if self.request_processors is None: - # default proccessor. Filter duplicates requests - self.request_processors = [ FilterDupes() ] - - - # wrap rules - self._rulesman = RulesManager(rules, spider=self) - # generates new requests with given callback - self._reqgen = RequestGenerator(self.request_extractors, - self.request_processors, - callback=self.parse) - - def parse(self, response): - """Dispatch callback and generate requests""" - # get rule for response - rule = self._rulesman.get_rule_from_response(response) - - if rule: - # dispatch callback if set - if rule.callback: - output = iterate_spider_output(rule.callback(response)) - for req_or_item in output: - yield req_or_item - - if rule.follow: - for req in self._reqgen.generate_requests(response): - # only dispatch request if has matching rule - if self._rulesman.get_rule_from_request(req): - yield req - else: - self.log("No rule for response %s" % response, level=log.WARNING) - -