From 34971ea6e7a356fb6b5ca86828287356b05e74f0 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 12 Dec 2009 18:15:18 -0200 Subject: [PATCH 01/68] bumped version to 0.9-dev --- scrapy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index bf6d46e17..fffaed1dc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 8, 0, '', 0) -__version__ = "0.8" +version_info = (0, 9, 0, 'dev') +__version__ = "0.9-dev" import sys, os From 7bbc14dd1523ba6e9267396b8555ef063ce8aa26 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Mon, 11 Jan 2010 12:26:43 -0400 Subject: [PATCH 02/68] templates: updated code --- scrapy/templates/project/module/pipelines.py.tmpl | 2 +- scrapy/templates/spiders/crawl.tmpl | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index fa6f5ea6f..e3f89342d 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -4,5 +4,5 @@ # See: http://doc.scrapy.org/topics/item-pipeline.html class ${ProjectName}Pipeline(object): - def process_item(self, domain, item): + def process_item(self, spider, item): return item diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 2d4f7fce6..18fd0ec42 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -10,15 +10,15 @@ class $classname(CrawlSpider): start_urls = ['http://www.$site/'] rules = ( - Rule(SgmlLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), + Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), ) def parse_item(self, response): - xs = HtmlXPathSelector(response) + hxs = HtmlXPathSelector(response) i = ${ProjectName}Item() - #i['site_id'] = xs.select('//input[@id="sid"]/@value').extract() - #i['name'] = xs.select('//div[@id="name"]').extract() - #i['description'] = xs.select('//div[@id="description"]').extract() + #i['site_id'] = hxs.select('//input[@id="sid"]/@value').extract() + #i['name'] = hxs.select('//div[@id="name"]').extract() + #i['description'] = hxs.select('//div[@id="description"]').extract() return i SPIDER = $classname() From 1402da31c5aedde0d339d7b9dd5191cfd37a1316 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Mon, 11 Jan 2010 12:28:22 -0400 Subject: [PATCH 03/68] docs: fixed typos and updated code examples --- docs/intro/tutorial.rst | 4 ++-- docs/topics/item-pipeline.rst | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3fb965110..6d7c3c64a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -420,8 +420,8 @@ scraped so far, the code for our Spider should be like this:: Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: - [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'Text Processing in Python'], 'link': [u'http://gnosis.cx/TPiP/'], 'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n']}) in - [dmoz.org] DEBUG: Scraped DmozItem({'title': [u'XML Processing with Python'], 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], 'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n']}) in + [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in + [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n'], link=[u'http://www.informit.com/store/product.aspx?isbn=0130211192'], title=[u'XML Processing with Python']) in Storing the data (using an Item Pipeline) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 0c325756b..0a4205b8c 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -98,10 +98,10 @@ spider returns multiples items with the same id:: del self.duplicates[spider] def process_item(self, spider, item): - if item.id in self.duplicates[spider]: + if item['id'] in self.duplicates[spider]: raise DropItem("Duplicate item found: %s" % item) else: - self.duplicates[spider].add(item.id) + self.duplicates[spider].add(item['id']) return item Built-in Item Pipelines reference @@ -178,7 +178,7 @@ on the respective Item Exporter to get more info. the first line. This format requires you to specify the fields to export using the :setting:`EXPORT_FIELDS` setting. -* ``jsonlines``: uses a :class:`~jsonlines.JsonLinesItemExporter` +* ``json``: uses a :class:`~jsonlines.JsonLinesItemExporter` * ``pickle``: uses a :class:`PickleItemExporter` From 7ddd4441e3a422207fecf385c8feac3495027118 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Fri, 19 Feb 2010 17:57:48 -0400 Subject: [PATCH 04/68] utils.python: added equal_attributes() to compare two objects arbitrary attributes Sign-Off: Rolando Espinoza La fuente --- scrapy/tests/test_utils_python.py | 50 ++++++++++++++++++++++++++++++- scrapy/utils/python.py | 25 ++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index 13d8be30c..dc46e1f91 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -1,7 +1,8 @@ +import operator import unittest from scrapy.utils.python import str_to_unicode, unicode_to_str, \ - memoizemethod_noargs, isbinarytext + memoizemethod_noargs, isbinarytext, equal_attributes class UtilsPythonTestCase(unittest.TestCase): def test_str_to_unicode(self): @@ -61,5 +62,52 @@ class UtilsPythonTestCase(unittest.TestCase): # finally some real binary bytes assert isbinarytext("\x02\xa3") + def test_equal_attributes(self): + class Obj: + pass + + a = Obj() + b = Obj() + # no attributes given return False + self.failIf(equal_attributes(a, b, [])) + # not existent attributes + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + a.x = 1 + b.x = 1 + # equal attribute + self.failUnless(equal_attributes(a, b, ['x'])) + + b.y = 2 + # obj1 has no attribute y + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + a.y = 2 + # equal attributes + self.failUnless(equal_attributes(a, b, ['x', 'y'])) + + a.y = 1 + # differente attributes + self.failIf(equal_attributes(a, b, ['x', 'y'])) + + # test callable + a.meta = {} + b.meta = {} + self.failUnless(equal_attributes(a, b, ['meta'])) + + # compare ['meta']['a'] + a.meta['z'] = 1 + b.meta['z'] = 1 + + get_z = operator.itemgetter('z') + get_meta = operator.attrgetter('meta') + compare_z = lambda obj: get_z(get_meta(obj)) + + self.failUnless(equal_attributes(a, b, [compare_z, 'x'])) + # fail z equality + a.meta['z'] = 2 + self.failIf(equal_attributes(a, b, [compare_z, 'x'])) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7d43c4566..319acdc05 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -216,3 +216,28 @@ def get_func_args(func): else: raise TypeError('%s is not callable' % type(func)) return func_args + + +def equal_attributes(obj1, obj2, attributes): + """Compare two objects attributes""" + # not attributes given return False by default + if not attributes: + return False + + for attr in attributes: + # support callables like itemgetter + if callable(attr): + if not attr(obj1) == attr(obj2): + return False + else: + # check that objects has attribute + if not hasattr(obj1, attr): + return False + if not hasattr(obj2, attr): + return False + # compare object attributes + if not getattr(obj1, attr) == getattr(obj2, attr): + return False + # all attributes equal + return True + From 17d154392937a78a6d9667180874eaf01b708ff4 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Fri, 19 Feb 2010 18:19:01 -0400 Subject: [PATCH 05/68] contrib_exp: added crawlspider v2 package + tests Sign-Off: Rolando Espinoza La fuente --- scrapy/contrib_exp/crawlspider/__init__.py | 4 + scrapy/contrib_exp/crawlspider/matchers.py | 61 ++++ scrapy/contrib_exp/crawlspider/reqext.py | 117 ++++++++ scrapy/contrib_exp/crawlspider/reqgen.py | 27 ++ scrapy/contrib_exp/crawlspider/reqproc.py | 111 ++++++++ scrapy/contrib_exp/crawlspider/rules.py | 100 +++++++ scrapy/contrib_exp/crawlspider/spider.py | 69 +++++ .../test_contrib_exp_crawlspider_matchers.py | 94 +++++++ .../test_contrib_exp_crawlspider_reqext.py | 137 +++++++++ .../test_contrib_exp_crawlspider_reqgen.py | 128 +++++++++ .../test_contrib_exp_crawlspider_reqproc.py | 144 ++++++++++ .../test_contrib_exp_crawlspider_rules.py | 262 ++++++++++++++++++ .../test_contrib_exp_crawlspider_spider.py | 222 +++++++++++++++ 13 files changed, 1476 insertions(+) create mode 100644 scrapy/contrib_exp/crawlspider/__init__.py create mode 100644 scrapy/contrib_exp/crawlspider/matchers.py create mode 100644 scrapy/contrib_exp/crawlspider/reqext.py create mode 100644 scrapy/contrib_exp/crawlspider/reqgen.py create mode 100644 scrapy/contrib_exp/crawlspider/reqproc.py create mode 100644 scrapy/contrib_exp/crawlspider/rules.py create mode 100644 scrapy/contrib_exp/crawlspider/spider.py create mode 100644 scrapy/tests/test_contrib_exp_crawlspider_matchers.py create mode 100644 scrapy/tests/test_contrib_exp_crawlspider_reqext.py create mode 100644 scrapy/tests/test_contrib_exp_crawlspider_reqgen.py create mode 100644 scrapy/tests/test_contrib_exp_crawlspider_reqproc.py create mode 100644 scrapy/tests/test_contrib_exp_crawlspider_rules.py create mode 100644 scrapy/tests/test_contrib_exp_crawlspider_spider.py diff --git a/scrapy/contrib_exp/crawlspider/__init__.py b/scrapy/contrib_exp/crawlspider/__init__.py new file mode 100644 index 000000000..03173eb38 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/__init__.py @@ -0,0 +1,4 @@ +"""CrawlSpider v2""" + +from .rules import Rule +from .spider import CrawlSpider diff --git a/scrapy/contrib_exp/crawlspider/matchers.py b/scrapy/contrib_exp/crawlspider/matchers.py new file mode 100644 index 000000000..3ef259c67 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/matchers.py @@ -0,0 +1,61 @@ +""" +Request/Response Matchers + +Perform evaluation to Request or Response attributes +""" + +import re + +class BaseMatcher(object): + """Base matcher. Returns True by default.""" + + def matches_request(self, request): + """Performs Request Matching""" + return True + + def matches_response(self, response): + """Performs Response Matching""" + return True + + +class UrlMatcher(BaseMatcher): + """Matches URL attribute""" + + def __init__(self, url): + """Initialize url attribute""" + self._url = url + + def matches_url(self, url): + """Returns True if given url is equal to matcher's url""" + return self._url == url + + def matches_request(self, request): + """Returns True if Request's url matches initial url""" + return self.matches_url(request.url) + + def matches_response(self, response): + """Returns True if Response's url matches initial url""" + return self.matches_url(response.url) + + +class UrlRegexMatcher(UrlMatcher): + """Matches URL using regular expression""" + + def __init__(self, regex, flags=0): + """Initialize regular expression""" + self._regex = re.compile(regex, flags) + + def matches_url(self, url): + """Returns True if url matches regular expression""" + return self._regex.search(url) is not None + + +class UrlListMatcher(UrlMatcher): + """Matches if URL is in List""" + + def __init__(self, urls): + self._urls = urls + + def matches_url(self, url): + """Returns True if url is in urls list""" + return url in self._urls diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py new file mode 100644 index 000000000..bb7318f79 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqext.py @@ -0,0 +1,117 @@ +"""Request Extractors""" +from scrapy.http import Request +from scrapy.selector import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import FixedSGMLParser, str_to_unicode +from scrapy.utils.url import safe_url_string, urljoin_rfc + +from itertools import ifilter + + +class BaseSgmlRequestExtractor(FixedSGMLParser): + """Base SGML Request Extractor""" + + def __init__(self, tag='a', attr='href'): + """Initialize attributes""" + FixedSGMLParser.__init__(self) + + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.current_request = None + + def extract_requests(self, response): + """Returns list of requests extracted from response""" + return self._extract_requests(response.body, response.url, + response.encoding) + + def _extract_requests(self, response_text, response_url, response_encoding): + """Extract requests with absolute urls""" + self.reset() + self.feed(response_text) + self.close() + + base_url = self.base_url if self.base_url else response_url + self._make_absolute_urls(base_url, response_encoding) + self._fix_link_text_encoding(response_encoding) + + return self.requests + + def _make_absolute_urls(self, base_url, encoding): + """Makes all request's urls absolute""" + for req in self.requests: + url = req.url + # make absolute url + url = urljoin_rfc(base_url, url, encoding) + url = safe_url_string(url, encoding) + # replace in-place request's url + req.url = url + + def _fix_link_text_encoding(self, encoding): + """Convert link_text to unicode for each request""" + for req in self.requests: + req.meta.setdefault('link_text', '') + req.meta['link_text'] = str_to_unicode(req.meta['link_text'], + encoding) + + def reset(self): + """Reset state""" + FixedSGMLParser.reset(self) + self.requests = [] + self.base_url = None + + def unknown_starttag(self, tag, attrs): + """Process unknown start tag""" + if 'base' == tag: + self.base_url = dict(attrs).get('href') + + _matches = lambda (attr, value): self.scan_attr(attr) \ + and value is not None + if self.scan_tag(tag): + for attr, value in ifilter(_matches, attrs): + req = Request(url=value) + self.requests.append(req) + self.current_request = req + + def unknown_endtag(self, tag): + """Process unknown end tag""" + self.current_request = None + + def handle_data(self, data): + """Process data""" + current = self.current_request + if current and not 'link_text' in current.meta: + current.meta['link_text'] = data.strip() + + +class SgmlRequestExtractor(BaseSgmlRequestExtractor): + """SGML Request Extractor""" + + def __init__(self, tags=None, attrs=None): + """Initialize with custom tag & attribute function checkers""" + # defaults + tags = tuple(tags) if tags else ('a', 'area') + attrs = tuple(attrs) if attrs else ('href', ) + + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + BaseSgmlRequestExtractor.__init__(self, tag=tag_func, attr=attr_func) + +# TODO: move to own file +class XPathRequestExtractor(SgmlRequestExtractor): + """SGML Request Extractor with XPath restriction""" + + def __init__(self, restrict_xpaths, tags=None, attrs=None): + """Initialize XPath restrictions""" + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + SgmlRequestExtractor.__init__(self, tags, attrs) + + def extract_requests(self, response): + """Restrict to XPath regions""" + hxs = HtmlXPathSelector(response) + fragments = (''.join( + html_frag for html_frag in hxs.select(xpath).extract() + ) for xpath in self.restrict_xpaths) + html_slice = ''.join(html_frag for html_frag in fragments) + return self._extract_requests(html_slice, response.url, + response.encoding) + diff --git a/scrapy/contrib_exp/crawlspider/reqgen.py b/scrapy/contrib_exp/crawlspider/reqgen.py new file mode 100644 index 000000000..3858fbcf7 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqgen.py @@ -0,0 +1,27 @@ +"""Request Generator""" +from itertools import imap + +class RequestGenerator(object): + """Extracto and process requests from response""" + + def __init__(self, req_extractors, req_processors, callback, spider=None): + """Initialize attributes""" + self._request_extractors = req_extractors + self._request_processors = req_processors + #TODO: resolve callback? + self._callback = callback + + def generate_requests(self, response): + """Extract and process new requests from response. + Attach callback to each request as default callback.""" + requests = [] + for ext in self._request_extractors: + requests.extend(ext.extract_requests(response)) + + for proc in self._request_processors: + requests = proc(requests) + + # return iterator + # @@@ creates new Request object with callback + return imap(lambda r: r.replace(callback=self._callback), requests) + diff --git a/scrapy/contrib_exp/crawlspider/reqproc.py b/scrapy/contrib_exp/crawlspider/reqproc.py new file mode 100644 index 000000000..d39399b20 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/reqproc.py @@ -0,0 +1,111 @@ +"""Request Processors""" +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.url import canonicalize_url, url_is_from_any_domain + +from itertools import ifilter, imap + +import re + +class Canonicalize(object): + """Canonicalize Request Processor""" + def _replace_url(self, req): + # replace in-place + req.url = canonicalize_url(req.url) + return req + + def __call__(self, requests): + """Canonicalize all requests' urls""" + return imap(self._replace_url, requests) + + +class FilterDupes(object): + """Filter duplicate Requests""" + + def __init__(self, *attributes): + """Initialize comparison attributes""" + self._attributes = tuple(attributes) if attributes \ + else tuple(['url']) + + def _equal_attr(self, obj1, obj2, attr): + return getattr(obj1, attr) == getattr(obj2, attr) + + def _requests_equal(self, req1, req2): + """Attribute comparison helper""" + # look for not equal attribute + _not_equal = lambda attr: not self._equal_attr(req1, req2, attr) + for attr in ifilter(_not_equal, self._attributes): + return False + # all attributes equal + return True + + def _request_in(self, request, requests_seen): + """Check if request is in given requests seen list""" + _req_seen = lambda r: self._requests_equal(r, request) + for seen in ifilter(_req_seen, requests_seen): + return True + # request not seen + return False + + def __call__(self, requests): + """Filter seen requests""" + # per-call duplicates filter + self.requests_seen = set() + _not_seen = lambda r: not self._request_in(r, self.requests_seen) + for req in ifilter(_not_seen, requests): + yield req + # registry seen request + self.requests_seen.add(req) + + +class FilterDomain(object): + """Filter request's domain""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + self.allow = tuple(arg_to_iter(allow)) + self.deny = tuple(arg_to_iter(deny)) + + def __call__(self, requests): + """Filter domains""" + processed = (req for req in requests) + + if self.allow: + processed = (req for req in requests + if url_is_from_any_domain(req.url, self.allow)) + if self.deny: + processed = (req for req in requests + if not url_is_from_any_domain(req.url, self.deny)) + + return processed + + +class FilterUrl(object): + """Filter request's url""" + + def __init__(self, allow=(), deny=()): + """Initialize allow/deny attributes""" + _re_type = type(re.compile('', 0)) + + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] + + def __call__(self, requests): + """Filter request's url based on allow/deny rules""" + #TODO: filter valid urls here? + processed = (req for req in requests) + + if self.allow_res: + processed = (req for req in requests + if self._matches(req.url, self.allow_res)) + if self.deny_res: + processed = (req for req in requests + if not self._matches(req.url, self.deny_res)) + + return processed + + def _matches(self, url, regexs): + """Returns True if url matches any regex in given list""" + return any(r.search(url) for r in regexs) + diff --git a/scrapy/contrib_exp/crawlspider/rules.py b/scrapy/contrib_exp/crawlspider/rules.py new file mode 100644 index 000000000..ff1691ad0 --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/rules.py @@ -0,0 +1,100 @@ +"""Crawler Rules""" +from scrapy.http import Request +from scrapy.http import Response + +from functools import partial +from itertools import ifilter + +from .matchers import BaseMatcher +# default strint-to-matcher class +from .matchers import UrlRegexMatcher + +class CompiledRule(object): + """Compiled version of Rule""" + def __init__(self, matcher, callback=None, follow=False): + """Initialize attributes checking type""" + assert isinstance(matcher, BaseMatcher) + assert callback is None or callable(callback) + assert isinstance(follow, bool) + + self.matcher = matcher + self.callback = callback + self.follow = follow + + +class Rule(object): + """Crawler Rule""" + def __init__(self, matcher=None, callback=None, follow=False, **kwargs): + """Store attributes""" + self.matcher = matcher + self.callback = callback + self.cb_kwargs = kwargs if kwargs else {} + self.follow = True if follow else False + + if self.callback is None and self.follow is False: + raise ValueError("Rule must either have a callback or " + "follow=True: %r" % self) + + def __repr__(self): + return "Rule(matcher=%r, callback=%r, follow=%r, **%r)" \ + % (self.matcher, self.callback, self.follow, self.cb_kwargs) + + +class RulesManager(object): + """Rules Manager""" + def __init__(self, rules, spider, default_matcher=UrlRegexMatcher): + """Initialize rules using spider and default matcher""" + self._rules = tuple() + + # compile absolute/relative-to-spider callbacks""" + for rule in rules: + # prepare matcher + if rule.matcher is None: + # instance BaseMatcher by default + matcher = BaseMatcher() + elif isinstance(rule.matcher, BaseMatcher): + matcher = rule.matcher + else: + # matcher not BaseMatcher, check for string + if isinstance(rule.matcher, basestring): + # instance default matcher + matcher = default_matcher(rule.matcher) + else: + raise ValueError('Not valid matcher given %r in %r' \ + % (rule.matcher, rule)) + + # prepare callback + if callable(rule.callback): + callback = rule.callback + elif not rule.callback is None: + # callback from spider + callback = getattr(spider, rule.callback) + + if not callable(callback): + raise AttributeError('Invalid callback %r can not be resolved' \ + % callback) + else: + callback = None + + if rule.cb_kwargs: + # build partial callback + callback = partial(callback, **rule.cb_kwargs) + + # append compiled rule to rules list + crule = CompiledRule(matcher, callback, follow=rule.follow) + self._rules += (crule, ) + + def get_rule_from_request(self, request): + """Returns first rule that matches given Request""" + _matches = lambda r: r.matcher.matches_request(request) + for rule in ifilter(_matches, self._rules): + # return first match of iterator + return rule + + def get_rule_from_response(self, response): + """Returns first rule that matches given Response""" + _matches = lambda r: r.matcher.matches_response(response) + for rule in ifilter(_matches, self._rules): + # return first match of iterator + return rule + diff --git a/scrapy/contrib_exp/crawlspider/spider.py b/scrapy/contrib_exp/crawlspider/spider.py new file mode 100644 index 000000000..730ad0e8d --- /dev/null +++ b/scrapy/contrib_exp/crawlspider/spider.py @@ -0,0 +1,69 @@ +"""CrawlSpider v2""" +from scrapy.spider import BaseSpider +from scrapy.utils.spider import iterate_spider_output + +from .matchers import UrlListMatcher +from .rules import Rule, RulesManager +from .reqext import SgmlRequestExtractor +from .reqgen import RequestGenerator +from .reqproc import Canonicalize, FilterDupes + +class CrawlSpider(BaseSpider): + """CrawlSpider v2""" + + request_extractors = None + request_processors = None + rules = [] + + def __init__(self): + """Initialize dispatcher""" + super(CrawlSpider, self).__init__() + + # auto follow start urls + if self.start_urls: + _matcher = UrlListMatcher(self.start_urls) + # append new rule using type from current self.rules + rules = self.rules + type(self.rules)([ + Rule(_matcher, follow=True) + ]) + else: + rules = self.rules + + # set defaults if not set + if self.request_extractors is None: + # default link extractor. Extracts all links from response + self.request_extractors = [ SgmlRequestExtractor() ] + + if self.request_processors is None: + # default proccessor. Filter duplicates requests + self.request_processors = [ FilterDupes() ] + + + # wrap rules + self._rulesman = RulesManager(rules, spider=self) + # generates new requests with given callback + self._reqgen = RequestGenerator(self.request_extractors, + self.request_processors, + callback=self.parse) + + def parse(self, response): + """Dispatch callback and generate requests""" + # get rule for response + rule = self._rulesman.get_rule_from_response(response) + + if rule: + # dispatch callback if set + if rule.callback: + output = iterate_spider_output(rule.callback(response)) + for req_or_item in output: + yield req_or_item + + if rule.follow: + for req in self._reqgen.generate_requests(response): + # only dispatch request if has matching rule + if self._rulesman.get_rule_from_request(req): + yield req + else: + self.log("No rule for response %s" % response, level=log.WARNING) + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_matchers.py b/scrapy/tests/test_contrib_exp_crawlspider_matchers.py new file mode 100644 index 000000000..4cb832aa1 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_matchers.py @@ -0,0 +1,94 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import Response + +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlListMatcher + +import re + +class MatchersTest(unittest.TestCase): + + def setUp(self): + pass + + def test_base_matcher(self): + matcher = BaseMatcher() + + request = Request('http://example.com') + response = Response('http://example.com') + + self.assertTrue(matcher.matches_request(request)) + self.assertTrue(matcher.matches_response(response)) + + def test_url_matcher(self): + matcher = UrlMatcher('http://example.com') + + request = Request('http://example.com') + response = Response('http://example.com') + + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_request(response)) + + request = Request('http://example2.com') + response = Response('http://example2.com') + + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_request(response)) + + def test_url_regex_matcher(self): + matcher = UrlRegexMatcher(r'sample') + urls = ( + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample4.html', + ) + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + matcher = UrlRegexMatcher(r'sample_fail') + for url in urls: + request, response = Request(url), Response(url) + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_response(response)) + + matcher = UrlRegexMatcher(r'SAMPLE\d+', re.IGNORECASE) + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + def test_url_list_matcher(self): + urls = ( + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample4.html', + ) + urls2 = ( + 'http://example.com/sample5.html', + 'http://example.com/sample6.html', + 'http://example.com/sample7.html', + 'http://example.com/sample8.html', + 'http://example.com/', + ) + matcher = UrlListMatcher(urls) + + # match urls + for url in urls: + request, response = Request(url), Response(url) + self.failUnless(matcher.matches_request(request)) + self.failUnless(matcher.matches_response(response)) + + # non-match urls + for url in urls2: + request, response = Request(url), Response(url) + self.failIf(matcher.matches_request(request)) + self.failIf(matcher.matches_response(response)) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py new file mode 100644 index 000000000..d0d0d1b5f --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py @@ -0,0 +1,137 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.tests import get_testdata + +from scrapy.contrib_exp.crawlspider.reqext import BaseSgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqext import XPathRequestExtractor + +class AbstractRequestExtractorTest(unittest.TestCase): + + def _requests_equals(self, list1, list2): + """Compares request's urls and link_text""" + for (r1, r2) in zip(list1, list2): + if r1.url != r2.url: + return False + if r1.meta['link_text'] != r2.meta['link_text']: + return False + # all equal + return True + + +class RequestExtractorTest(AbstractRequestExtractorTest): + + def test_basic(self): + base_url = 'http://example.org/somepage/index.html' + html = """Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + requests = [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/about.html', + meta={'link_text': 'About us'}), + Request('http://example.org/othercat.html', + meta={'link_text': 'Other category'}), + Request('http://example.org/', + meta={'link_text': ''}), + ] + + response = HtmlResponse(base_url, body=html) + reqx = BaseSgmlRequestExtractor() # default: tag=a, attr=href + + self.failUnless( + self._requests_equals(requests, reqx.extract_requests(response)) + ) + + def test_base_url(self): + html = """<html><head><title>Page title<title> + <base href="http://otherdomain.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", + body=html) + reqx = BaseSgmlRequestExtractor() + + self.failUnless( + self._requests_equals(reqx.extract_requests(response), + [ Request('http://otherdomain.com/base/item/12.html', + meta={'link_text': 'Item 12'}) ] + ) + ) + + def test_extraction_encoding(self): + #TODO: use own fixtures + body = get_testdata('link_extractor', 'linkextractor_noenc.html') + response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, + headers={'Content-Type': ['text/html; charset=utf-8']}) + response_noenc = HtmlResponse(url='http://example.com/noenc', + body=body) + body = get_testdata('link_extractor', 'linkextractor_latin1.html') + response_latin1 = HtmlResponse(url='http://example.com/latin1', + body=body) + + reqx = BaseSgmlRequestExtractor() + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_utf8), + [ Request(url='http://example.com/sample_%C3%B1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E2%82%AC.html', + meta={'link_text': + 'sample \xe2\x82\xac text'.decode('utf-8')}) ] + ) + ) + + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_noenc), + [ Request(url='http://example.com/sample_%C3%B1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E2%82%AC.html', + meta={'link_text': + 'sample \xe2\x82\xac text'.decode('utf-8')}) ] + ) + ) + + self.failUnless( + self._requests_equals( + reqx.extract_requests(response_latin1), + [ Request(url='http://example.com/sample_%F1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample_%E1.html', + meta={'link_text': + 'sample \xe1 text'.decode('latin1')}) ] + ) + ) + + +class SgmlRequestExtractorTest(AbstractRequestExtractorTest): + pass + + +class XPathRequestExtractorTest(AbstractRequestExtractorTest): + + def setUp(self): + # TODO: use own fixtures + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + + def test_restrict_xpaths(self): + reqx = XPathRequestExtractor('//div[@id="subwrapper"]') + self.failUnless( + self._requests_equals( + reqx.extract_requests(self.response), + [ Request(url='http://example.com/sample1.html', + meta={'link_text': ''}), + Request(url='http://example.com/sample2.html', + meta={'link_text': 'sample 2'}) ] + ) + ) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py b/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py new file mode 100644 index 000000000..67aca2387 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqgen.py @@ -0,0 +1,128 @@ +from twisted.internet import defer +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.utils.python import equal_attributes + +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqgen import RequestGenerator +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDomain +from scrapy.contrib_exp.crawlspider.reqproc import FilterUrl +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + +class RequestGeneratorTest(unittest.TestCase): + + def setUp(self): + url = 'http://example.org/somepage/index.html' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + self.response = HtmlResponse(url, body=html) + self.deferred = defer.Deferred() + self.requests = [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/about.html', + meta={'link_text': 'About us'}), + Request('http://example.org/othercat.html', + meta={'link_text': 'Other category'}), + Request('http://example.org/', + meta={'link_text': ''}), + ] + + def _equal_requests_list(self, list1, list2): + list1 = list(list1) + list2 = list(list2) + if not len(list1) == len(list2): + return False + + for (req1, req2) in zip(list1, list2): + if not equal_attributes(req1, req2, ['url']): + return False + return True + + def test_basic(self): + reqgen = RequestGenerator([], [], callback=self.deferred) + # returns generator + requests = reqgen.generate_requests(self.response) + self.failUnlessEqual(list(requests), []) + + def test_request_extractor(self): + extractors = [ + SgmlRequestExtractor() + ] + + # extract all requests + reqgen = RequestGenerator(extractors, [], callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnless(self._equal_requests_list(requests, self.requests)) + + for req in requests: + # check callback + self.failUnlessEqual(req.deferred, self.deferred) + + def test_request_processor(self): + extractors = [ + SgmlRequestExtractor() + ] + + processors = [ + Canonicalize(), + FilterDupes(), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnless(self._equal_requests_list(requests, self.requests)) + + # filter domain + processors = [ + Canonicalize(), + FilterDupes(), + FilterDomain(deny='example.org'), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + self.failUnlessEqual(list(requests), []) + + # filter url + processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(deny=(r'about', r'othercat')), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + + self.failUnless(self._equal_requests_list(requests, [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + Request('http://example.org/', + meta={'link_text': ''}), + ])) + + processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(allow=r'/somepage/'), + ] + + reqgen = RequestGenerator(extractors, processors, callback=self.deferred) + requests = reqgen.generate_requests(self.response) + + self.failUnless(self._equal_requests_list(requests, [ + Request('http://example.org/somepage/item/12.html', + meta={'link_text': 'Item 12'}), + ])) + + + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py b/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py new file mode 100644 index 000000000..da5db67b2 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqproc.py @@ -0,0 +1,144 @@ +from twisted.trial import unittest + +from scrapy.http import Request + +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDomain +from scrapy.contrib_exp.crawlspider.reqproc import FilterUrl +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + +import copy + +class RequestProcessorsTest(unittest.TestCase): + + def test_canonicalize_requests(self): + urls = [ + 'http://example.com/do?&b=1&a=2&c=3', + 'http://example.com/do?123,&q=a space', + ] + urls_after = [ + 'http://example.com/do?a=2&b=1&c=3', + 'http://example.com/do?123%2C=&q=a+space', + ] + + proc = Canonicalize() + results = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(results, urls_after) + + def test_unique_requests(self): + urls = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + ] + urls_unique = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + ] + + proc = FilterDupes() + results = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(results, urls_unique) + + # Check custom attributes + requests = [ + Request('http://example.com', method='GET'), + Request('http://example.com', method='POST'), + ] + proc = FilterDupes('url', 'method') + self.failUnlessEqual(len(list(proc(requests))), 2) + + proc = FilterDupes('url') + self.failUnlessEqual(len(list(proc(requests))), 1) + + def test_filter_domain(self): + urls = [ + 'http://blah1.com/index', + 'http://blah2.com/index', + 'http://blah1.com/section', + 'http://blah2.com/section', + ] + + proc = FilterDomain(allow=('blah1.com'), deny=('blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, [ + 'http://blah1.com/index', + 'http://blah1.com/section', + ]) + + proc = FilterDomain(deny=('blah1.com', 'blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, []) + + proc = FilterDomain(allow=('blah1.com', 'blah2.com')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, urls) + + def test_filter_url(self): + urls = [ + 'http://blah1.com/index', + 'http://blah2.com/index', + 'http://blah1.com/section', + 'http://blah2.com/section', + ] + + proc = FilterUrl(allow=(r'blah1'), deny=(r'blah2')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, [ + 'http://blah1.com/index', + 'http://blah1.com/section', + ]) + + proc = FilterUrl(deny=('blah1', 'blah2')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, []) + + proc = FilterUrl(allow=('index$', 'section$')) + filtered = [req.url for req in proc(Request(url) for url in urls)] + self.failUnlessEquals(filtered, urls) + + + + def test_all_processors(self): + urls = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/do?&b=1&a=2&c=3', + 'http://example.com/do?123,&q=a space', + ] + urls_processed = [ + 'http://example.com/sample1.html', + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/do?a=2&b=1&c=3', + 'http://example.com/do?123%2C=&q=a+space', + ] + + processors = [ + Canonicalize(), + FilterDupes(), + ] + + def _process(requests): + """Apply all processors""" + # copy list + processed = [copy.copy(req) for req in requests] + for proc in processors: + processed = proc(processed) + return processed + + # empty requests + results1 = [r.url for r in _process([])] + self.failUnlessEquals(results1, []) + + # try urls + requests = (Request(url) for url in urls) + results2 = [r.url for r in _process(requests)] + self.failUnlessEquals(results2, urls_processed) + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_rules.py b/scrapy/tests/test_contrib_exp_crawlspider_rules.py new file mode 100644 index 000000000..0fbe52415 --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_rules.py @@ -0,0 +1,262 @@ +from twisted.trial import unittest + +from scrapy.http import HtmlResponse +from scrapy.spider import BaseSpider +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher + +from scrapy.contrib_exp.crawlspider.rules import CompiledRule +from scrapy.contrib_exp.crawlspider.rules import Rule +from scrapy.contrib_exp.crawlspider.rules import RulesManager + +from functools import partial + +class RuleInitializationTest(unittest.TestCase): + + def test_fail_if_rule_null(self): + # fail on empty rule + self.failUnlessRaises(ValueError, Rule) + self.failUnlessRaises(ValueError, Rule, + **dict(callback=None, follow=None)) + self.failUnlessRaises(ValueError, Rule, + **dict(callback=None, follow=False)) + + def test_minimal_arguments_to_instantiation(self): + # not fail if callback set + self.failUnless(Rule(callback=lambda: True)) + # not fail if follow set + self.failUnless(Rule(follow=True)) + + def test_validate_default_attributes(self): + # test null Rule + rule = Rule(follow=True) + self.failUnlessEqual(None, rule.matcher) + self.failUnlessEqual(None, rule.callback) + self.failUnlessEqual({}, rule.cb_kwargs) + # follow default False + self.failUnlessEqual(True, rule.follow) + + def test_validate_attributes_set(self): + matcher = BaseMatcher() + callback = lambda: True + rule = Rule(matcher, callback, True, a=1) + # test attributes + self.failUnlessEqual(matcher, rule.matcher) + self.failUnlessEqual(callback, rule.callback) + self.failUnlessEqual({'a': 1}, rule.cb_kwargs) + self.failUnlessEqual(True, rule.follow) + +class CompiledRuleInitializationTest(unittest.TestCase): + + def test_fail_on_invalid_matcher(self): + # pass with valid matcher + self.failUnless(CompiledRule(BaseMatcher()), + "Failed CompiledRule instantiation") + + # at least needs valid matcher + self.assertRaises(AssertionError, CompiledRule, None) + self.assertRaises(AssertionError, CompiledRule, False) + self.assertRaises(AssertionError, CompiledRule, True) + + def test_fail_on_invalid_callback(self): + # pass with valid callback + callback = lambda: True + self.failUnless(CompiledRule(BaseMatcher(), callback)) + # pass with callback none + self.failUnless(CompiledRule(BaseMatcher(), None)) + + # assert on invalid callback + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + 'myfunc') + + # numeric variable + var = 123 + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + var) + + class A: + pass + + # random instance + self.assertRaises(AssertionError, CompiledRule, BaseMatcher(), + A()) + + + def test_fail_on_invalid_follow_value(self): + callback = lambda: True + matcher = BaseMatcher() + # pass bool + self.failUnless(CompiledRule(matcher, callback, True)) + self.failUnless(CompiledRule(matcher, callback, False)) + + # assert with non-bool + self.assertRaises(AssertionError, CompiledRule, matcher, + callback, None) + self.assertRaises(AssertionError, CompiledRule, matcher, + callback, 1) + + def test_validate_default_attributes(self): + callback = lambda: True + matcher = BaseMatcher() + rule = CompiledRule(matcher, callback, True) + + # test attributes + self.failUnlessEqual(matcher, rule.matcher) + self.failUnlessEqual(callback, rule.callback) + self.failUnlessEqual(True, rule.follow) + + +class RulesTest(unittest.TestCase): + def test_rules_manager_basic(self): + spider = BaseSpider() + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + rulesman = RulesManager([], spider) + + # should return none + self.failIf(rulesman.get_rule_from_response(response1)) + self.failIf(rulesman.get_rule_from_response(response2)) + + # rules manager with match-all rule + rulesman = RulesManager([ + Rule(BaseMatcher(), follow=True), + ], spider) + + # returns CompiledRule + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnless(isinstance(rule1, CompiledRule)) + self.failUnless(isinstance(rule2, CompiledRule)) + self.assert_(rule1 is rule2) + self.failUnlessEqual(rule1.callback, None) + self.failUnlessEqual(rule1.follow, True) + + def test_rules_manager_empty_rule(self): + spider = BaseSpider() + response = HtmlResponse('http://example.org') + + rulesman = RulesManager([Rule(follow=True)], spider) + + rule = rulesman.get_rule_from_response(response) + # default matcher if None: BaseMatcher + self.failUnless(isinstance(rule.matcher, BaseMatcher)) + + def test_rules_manager_default_matcher(self): + spider = BaseSpider() + response = HtmlResponse('http://example.org') + callback = lambda x: None + + rulesman = RulesManager([ + Rule('http://example.org', callback), + ], spider, default_matcher=UrlMatcher) + + rule = rulesman.get_rule_from_response(response) + self.failUnless(isinstance(rule.matcher, UrlMatcher)) + + def test_rules_manager_matchers(self): + spider = BaseSpider() + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + + urlmatcher = UrlMatcher('http://example.org') + basematcher = BaseMatcher() + # callback needed for Rule + callback = lambda x: None + + # test fail matcher resolve + self.assertRaises(ValueError, RulesManager, + [Rule(False, callback)], spider) + self.assertRaises(ValueError, RulesManager, + [Rule(spider, callback)], spider) + + rulesman = RulesManager([ + Rule(urlmatcher, callback), + Rule(basematcher, callback), + ], spider) + + # response1 matches example.org + rule1 = rulesman.get_rule_from_response(response1) + # response2 is catch by BaseMatcher() + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.matcher, urlmatcher) + self.failUnlessEqual(rule2.matcher, basematcher) + + # reverse order. BaseMatcher should match all + rulesman = RulesManager([ + Rule(basematcher, callback), + Rule(urlmatcher, callback), + ], spider) + + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.matcher, basematcher) + self.failUnlessEqual(rule2.matcher, basematcher) + self.failUnless(rule1 is rule2) + + def test_rules_manager_callbacks(self): + mycallback = lambda: True + + spider = BaseSpider() + spider.parse_item = lambda: True + + response1 = HtmlResponse('http://example.org') + response2 = HtmlResponse('http://othersite.org') + + rulesman = RulesManager([ + Rule('example', mycallback), + Rule('othersite', 'parse_item'), + ], spider, default_matcher=UrlRegexMatcher) + + rule1 = rulesman.get_rule_from_response(response1) + rule2 = rulesman.get_rule_from_response(response2) + + self.failUnlessEqual(rule1.callback, mycallback) + self.failUnlessEqual(rule2.callback, spider.parse_item) + + # fail unknown callback + self.assertRaises(AttributeError, RulesManager, [ + Rule(BaseMatcher(), 'mycallback') + ], spider) + # fail not callable + spider.not_callable = True + self.assertRaises(AttributeError, RulesManager, [ + Rule(BaseMatcher(), 'not_callable') + ], spider) + + + def test_rules_manager_callback_with_arguments(self): + spider = BaseSpider() + response = HtmlResponse('http://example.org') + + kwargs = {'a': 1} + + def myfunc(**mykwargs): + return mykwargs + + # verify return validation + self.failUnlessEquals(kwargs, myfunc(**kwargs)) + + # test callback w/o arguments + rulesman = RulesManager([ + Rule(BaseMatcher(), myfunc), + ], spider) + rule = rulesman.get_rule_from_response(response) + + # without arguments should return same callback + self.failUnlessEqual(rule.callback, myfunc) + + # test callback w/ arguments + rulesman = RulesManager([ + Rule(BaseMatcher(), myfunc, **kwargs), + ], spider) + rule = rulesman.get_rule_from_response(response) + + # with argument should return partial applied callback + self.failUnless(isinstance(rule.callback, partial)) + self.failUnlessEquals(kwargs, rule.callback()) + + diff --git a/scrapy/tests/test_contrib_exp_crawlspider_spider.py b/scrapy/tests/test_contrib_exp_crawlspider_spider.py new file mode 100644 index 000000000..5e067508a --- /dev/null +++ b/scrapy/tests/test_contrib_exp_crawlspider_spider.py @@ -0,0 +1,222 @@ +from twisted.trial import unittest + +from scrapy.http import Request +from scrapy.http import HtmlResponse +from scrapy.item import BaseItem +from scrapy.utils.spider import iterate_spider_output + +# basics +from scrapy.contrib_exp.crawlspider import CrawlSpider +from scrapy.contrib_exp.crawlspider import Rule + +# matchers +from scrapy.contrib_exp.crawlspider.matchers import BaseMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlRegexMatcher +from scrapy.contrib_exp.crawlspider.matchers import UrlListMatcher + +# extractors +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor + +# processors +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize +from scrapy.contrib_exp.crawlspider.reqproc import FilterDupes + + +# mock items +class Item1(BaseItem): + pass + +class Item2(BaseItem): + pass + +class Item3(BaseItem): + pass + + +class CrawlSpiderTest(unittest.TestCase): + + def spider_factory(self, rules=[], + extractors=[], processors=[], + start_urls=[]): + # mock spider + class Spider(CrawlSpider): + def parse_item1(self, response): + return Item1() + + def parse_item2(self, response): + return Item2() + + def parse_item3(self, response): + return Item3() + + def parse_request1(self, response): + return Request('http://example.org/request1') + + def parse_request2(self, response): + return Request('http://example.org/request2') + + Spider.start_urls = start_urls + Spider.rules = rules + Spider.request_extractors = extractors + Spider.request_processors = processors + + return Spider() + + def test_start_url_auto_rule(self): + spider = self.spider_factory() + # zero spider rules + self.failUnlessEqual(len(spider.rules), 0) + self.failUnlessEqual(len(spider._rulesman._rules), 0) + + spider = self.spider_factory(start_urls=['http://example.org']) + + self.failUnlessEqual(len(spider.rules), 0) + self.failUnlessEqual(len(spider._rulesman._rules), 1) + + def test_start_url_matcher(self): + url = 'http://example.org' + spider = self.spider_factory(start_urls=[url]) + + response = HtmlResponse(url) + + rule = spider._rulesman.get_rule_from_response(response) + self.failUnless(isinstance(rule.matcher, UrlListMatcher)) + + response = HtmlResponse(url + '/item.html') + + rule = spider._rulesman.get_rule_from_response(response) + self.failUnless(rule is None) + + # TODO: remove this block + # in previous version get_rule returns rule from response.request + response.request = Request(url) + rule = spider._rulesman.get_rule_from_response(response.request) + self.failUnless(isinstance(rule.matcher, UrlListMatcher)) + self.failUnlessEqual(rule.follow, True) + + def test_parse_callback(self): + response = HtmlResponse('http://example.org') + rules = ( + Rule(BaseMatcher(), 'parse_item1'), + ) + spider = self.spider_factory(rules) + + result = list(spider.parse(response)) + self.failUnlessEqual(len(result), 1) + self.failUnless(isinstance(result[0], Item1)) + + def test_crawling_start_url(self): + url = 'http://example.org/' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + response = HtmlResponse(url, body=html) + + extractors = (SgmlRequestExtractor(), ) + spider = self.spider_factory(start_urls=[url], + extractors=extractors) + result = list(spider.parse(response)) + + # 1 request extracted: example.org/ + # because requests returns only matching + self.failUnlessEqual(len(result), 1) + + # we will add catch-all rule to extract all + callback = lambda x: None + rules = [Rule(r'\.html$', callback=callback)] + spider = self.spider_factory(rules, start_urls=[url], + extractors=extractors) + result = list(spider.parse(response)) + + # 4 requests extracted + # 3 of .html pattern + # 1 of start url patter + self.failUnlessEqual(len(result), 4) + + def test_crawling_simple_rule(self): + url = 'http://example.org/somepage/index.html' + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + response = HtmlResponse(url, body=html) + + rules = ( + # first response callback + Rule(r'index\.html', 'parse_item1'), + ) + spider = self.spider_factory(rules) + result = list(spider.parse(response)) + + # should return Item1 + self.failUnlessEqual(len(result), 1) + self.failUnless(isinstance(result[0], Item1)) + + # test request generation + rules = ( + # first response without callback and follow flag + Rule(r'index\.html', follow=True), + Rule(r'(\.html|/)$', 'parse_item1'), + ) + spider = self.spider_factory(rules) + result = list(spider.parse(response)) + + # 0 because spider does not have extractors + self.failUnlessEqual(len(result), 0) + + extractors = (SgmlRequestExtractor(), ) + + # instance spider with extractor + spider = self.spider_factory(rules, extractors) + result = list(spider.parse(response)) + # 4 requests extracted + self.failUnlessEqual(len(result), 4) + + def test_crawling_multiple_rules(self): + html = """<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p></body></html>""" + + response = HtmlResponse('http://example.org/index.html', body=html) + response1 = HtmlResponse('http://example.org/1.html') + response2 = HtmlResponse('http://example.org/othercat.html') + + rules = ( + Rule(r'\d+\.html$', 'parse_item1'), + Rule(r'othercat\.html$', 'parse_item2'), + # follow-only rules + Rule(r'index\.html', 'parse_item3', follow=True) + ) + extractors = [SgmlRequestExtractor()] + spider = self.spider_factory(rules, extractors) + + result = list(spider.parse(response)) + # 1 Item 2 Requests + self.failUnlessEqual(len(result), 3) + # parse_item3 + self.failUnless(isinstance(result[0], Item3)) + only_requests = lambda r: isinstance(r, Request) + requests = filter(only_requests, result[1:]) + self.failUnlessEqual(len(requests), 2) + self.failUnless(all(requests)) + + result1 = list(spider.parse(response1)) + # parse_item1 + self.failUnlessEqual(len(result1), 1) + self.failUnless(isinstance(result1[0], Item1)) + + result2 = list(spider.parse(response2)) + # parse_item2 + self.failUnlessEqual(len(result2), 1) + self.failUnless(isinstance(result2[0], Item2)) + + From a6a3f085a7a0c6aa23c69a6fe9f031ae9203681b Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente <darkrho@gmail.com> Date: Fri, 19 Feb 2010 18:22:38 -0400 Subject: [PATCH 06/68] docs: added crawlspider v2 outline documentation Sign-Off: Rolando Espinoza La fuente --- docs/experimental/crawlspider-v2.rst | 128 +++++++++++++++++++++++++++ docs/experimental/index.rst | 1 + 2 files changed, 129 insertions(+) create mode 100644 docs/experimental/crawlspider-v2.rst diff --git a/docs/experimental/crawlspider-v2.rst b/docs/experimental/crawlspider-v2.rst new file mode 100644 index 000000000..a1ea09c48 --- /dev/null +++ b/docs/experimental/crawlspider-v2.rst @@ -0,0 +1,128 @@ +.. _topics-crawlspider-v2: + +============== +CrawlSpider v2 +============== + +Introduction +============ + +TODO: introduction + +Rules Matching +============== + +TODO: describe purpose of rules + +Request Extractors & Processors +=============================== + +TODO: describe purpose of extractors & processors + +Examples +======== + +TODO: plenty of examples + + +.. module:: scrapy.contrib_exp.crawlspider.spider + :synopsis: CrawlSpider + + +Reference +========= + +CrawlSpider +----------- + +TODO: describe crawlspider + +.. class:: CrawlSpider + + TODO: describe class + + +.. module:: scrapy.contrib_exp.crawlspider.rules + :synopsis: Rules + +Rules +----- + +TODO: describe spider rules + +.. class:: Rule + + TODO: describe Rules class + + +.. module:: scrapy.contrib_exp.crawlspider.reqext + :synopsis: Request Extractors + +Request Extractors +------------------ + +TODO: describe extractors purpose + +.. class:: BaseSgmlRequestExtractor + + TODO: describe base extractor + +.. class:: SgmlRequestExtractor + + TODO: describe sgml extractor + +.. class:: XPathRequestExtractor + + TODO: describe xpath request extractor + + +.. module:: scrapy.contrib_exp.crawlspider.reqproc + :synopsis: Request Processors + +Request Processors +------------------ + +TODO: describe request processors + +.. class:: Canonicalize + + TODO: describe proc + +.. class:: Unique + + TODO: describe unique + +.. class:: FilterDomain + + TODO: describe filter domain + +.. class:: FilterUrl + + TODO: describe filter url + + +.. module:: scrapy.contrib_exp.crawlspider.matchers + :synopsis: Matchers + +Request/Response Matchers +------------------------- + +TODO: describe matchers + +.. class:: BaseMatcher + + TODO: describe base matcher + +.. class:: UrlMatcher + + TODO: describe url matcher + +.. class:: UrlRegexMatcher + + TODO: describe UrlListMatcher + +.. class:: UrlListMatcher + + TODO: describe url list matcher + + diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 47f4ee4ac..f63ceddd9 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -21,3 +21,4 @@ it's properly merged) . Use at your own risk. djangoitems scheduler-middleware + crawlspider-v2 From 4a053a762fe176693a42d8741ebe8c96af78e309 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente <darkrho@gmail.com> Date: Fri, 19 Feb 2010 18:28:16 -0400 Subject: [PATCH 07/68] examples/experimental: added gooledir crawler --- .../googledir/googledir/__init__.py | 1 + .../experimental/googledir/googledir/items.py | 16 ++++++++ .../googledir/googledir/pipelines.py | 22 ++++++++++ .../googledir/googledir/settings.py | 21 ++++++++++ .../googledir/googledir/spiders/__init__.py | 8 ++++ .../googledir/spiders/google_directory.py | 40 +++++++++++++++++++ examples/experimental/googledir/scrapy-ctl.py | 7 ++++ 7 files changed, 115 insertions(+) create mode 100644 examples/experimental/googledir/googledir/__init__.py create mode 100644 examples/experimental/googledir/googledir/items.py create mode 100644 examples/experimental/googledir/googledir/pipelines.py create mode 100644 examples/experimental/googledir/googledir/settings.py create mode 100644 examples/experimental/googledir/googledir/spiders/__init__.py create mode 100644 examples/experimental/googledir/googledir/spiders/google_directory.py create mode 100644 examples/experimental/googledir/scrapy-ctl.py diff --git a/examples/experimental/googledir/googledir/__init__.py b/examples/experimental/googledir/googledir/__init__.py new file mode 100644 index 000000000..3104ef709 --- /dev/null +++ b/examples/experimental/googledir/googledir/__init__.py @@ -0,0 +1 @@ +# googledir project diff --git a/examples/experimental/googledir/googledir/items.py b/examples/experimental/googledir/googledir/items.py new file mode 100644 index 000000000..decc2c9ba --- /dev/null +++ b/examples/experimental/googledir/googledir/items.py @@ -0,0 +1,16 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class GoogledirItem(Item): + + name = Field(default='') + url = Field(default='') + description = Field(default='') + + def __str__(self): + return "Google Category: name=%s url=%s" \ + % (self['name'], self['url']) diff --git a/examples/experimental/googledir/googledir/pipelines.py b/examples/experimental/googledir/googledir/pipelines.py new file mode 100644 index 000000000..f775b254c --- /dev/null +++ b/examples/experimental/googledir/googledir/pipelines.py @@ -0,0 +1,22 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +from scrapy.core.exceptions import DropItem + +class FilterWordsPipeline(object): + """ + A pipeline for filtering out items which contain certain + words in their description + """ + + # put all words in lowercase + words_to_filter = ['politics', 'religion'] + + def process_item(self, spider, item): + for word in self.words_to_filter: + if word in unicode(item['description']).lower(): + raise DropItem("Contains forbidden word: %s" % word) + else: + return item diff --git a/examples/experimental/googledir/googledir/settings.py b/examples/experimental/googledir/googledir/settings.py new file mode 100644 index 000000000..4e3c11163 --- /dev/null +++ b/examples/experimental/googledir/googledir/settings.py @@ -0,0 +1,21 @@ +# Scrapy settings for googledir project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'googledir' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['googledir.spiders'] +NEWSPIDER_MODULE = 'googledir.spiders' +DEFAULT_ITEM_CLASS = 'googledir.items.GoogledirItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + +ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline'] diff --git a/examples/experimental/googledir/googledir/spiders/__init__.py b/examples/experimental/googledir/googledir/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py new file mode 100644 index 000000000..fdbf3f24a --- /dev/null +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -0,0 +1,40 @@ +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule + +from googledir.items import GoogledirItem + +class GoogleDirectorySpider(CrawlSpider): + + domain_name = 'directory.google.com' + start_urls = ['http://directory.google.com/'] + + rules = ( + # search for categories pattern and follow links + Rule(r'/[A-Z][a-zA-Z_/]+$', 'parse_category', follow=True), + ) + + def parse_category(self, response): + # The main selector we're using to extract data from the page + main_selector = HtmlXPathSelector(response) + + # The XPath to website links in the directory page + xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font' + + # Get a list of (sub) selectors to each website node pointed by the XPath + sub_selectors = main_selector.select(xpath) + + # Iterate over the sub-selectors to extract data for each website + for selector in sub_selectors: + item = GoogledirItem() + + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('name', 'a/text()') + l.add_xpath('url', 'a/@href') + l.add_xpath('description', 'font[2]/text()') + + # Here we populate the item and yield it + yield l.load_item() + +SPIDER = GoogleDirectorySpider() + diff --git a/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py new file mode 100644 index 000000000..552421ac3 --- /dev/null +++ b/examples/experimental/googledir/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings') + +from scrapy.command.cmdline import execute +execute() From c1f8198639eab1b8ac3adffd68e5881ad020e4da Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Fri, 19 Feb 2010 21:53:18 -0200 Subject: [PATCH 08/68] Added RANDOMIZE_DOWNLOAD_DELAY setting --- docs/topics/settings.rst | 30 ++++++++++++++++++++++++++++++ scrapy/conf/default_settings.py | 2 ++ scrapy/core/downloader/manager.py | 24 +++++++++++++++++++----- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b3dbc78f..d7a7d7fcf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -418,6 +418,15 @@ supported. Example:: DOWNLOAD_DELAY = 0.25 # 250 ms of delay +This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting (which is enabled by default). By default, Scrapy doesn't wait a fixed +amount of time between requests, but uses a random interval between 0.5 and 1.5 +* :setting:`DOWNLOAD_DELAY`. + +Another way to change the download delay (per spider, instead of globally) is +by using the ``download_delay`` spider attribute, which takes more precedence +than this setting. + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT @@ -677,6 +686,27 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' +.. setting:: RANDOMIZE_DOWNLOAD_DELAY + +RANDOMIZE_DOWNLOAD_DELAY +------------------------ + +Default: ``True`` + +If enabled, Scrapy will wait a random amount of time (between 0.5 and 1.5 +* :setting:`DOWNLOAD_DELAY`) while fetching requests from the same +spider. + +This randomization decreases the chance of the crawler being detected (and +subsequently blocked) by sites which analyze requests looking for statistically +significant similarities in the time between their times. + +The randomization policy is the same used by `wget`_ ``--random-wait`` option. + +If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. + +.. _wget: http://www.gnu.org/software/wget/manual/wget.html + .. setting:: REDIRECT_MAX_TIMES REDIRECT_MAX_TIMES diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 8892e41b4..76feb0c6f 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -122,6 +122,8 @@ MYSQL_CONNECTION_SETTINGS = {} NEWSPIDER_MODULE = '' +RANDOMIZE_DOWNLOAD_DELAY = True + REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index b53db5811..aec17b05e 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -2,6 +2,7 @@ Download web pages using asynchronous IO """ +import random from time import time from twisted.internet import reactor, defer @@ -20,15 +21,21 @@ class SpiderInfo(object): def __init__(self, download_delay=None, max_concurrent_requests=None): if download_delay is None: - self.download_delay = settings.getfloat('DOWNLOAD_DELAY') + self._download_delay = settings.getfloat('DOWNLOAD_DELAY') else: - self.download_delay = download_delay - if self.download_delay: + self._download_delay = float(download_delay) + if self._download_delay: self.max_concurrent_requests = 1 elif max_concurrent_requests is None: self.max_concurrent_requests = settings.getint('CONCURRENT_REQUESTS_PER_SPIDER') else: self.max_concurrent_requests = max_concurrent_requests + if self._download_delay and settings.getbool('RANDOMIZE_DOWNLOAD_DELAY'): + # same policy as wget --random-wait + self.random_delay_interval = (0.5*self._download_delay, \ + 1.5*self._download_delay) + else: + self.random_delay_interval = None self.active = set() self.queue = [] @@ -44,6 +51,12 @@ class SpiderInfo(object): # use self.active to include requests in the downloader middleware return len(self.active) > 2 * self.max_concurrent_requests + def download_delay(self): + if self.random_delay_interval: + return random.uniform(*self.random_delay_interval) + else: + return self._download_delay + def cancel_request_calls(self): for call in self.next_request_calls: call.cancel() @@ -99,8 +112,9 @@ class Downloader(object): # Delay queue processing if a download_delay is configured now = time() - if site.download_delay: - penalty = site.download_delay - now + site.lastseen + delay = site.download_delay() + if delay: + penalty = delay - now + site.lastseen if penalty > 0: d = defer.Deferred() d.addCallback(self._process_queue) From cb99edd153e7b90fda1058c13178b96f9dfee179 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Fri, 19 Feb 2010 23:16:55 -0200 Subject: [PATCH 09/68] simplified and improved AUTHORS file --- AUTHORS | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/AUTHORS b/AUTHORS index a0fbe722f..1392aa71f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,28 +1,25 @@ Scrapy was brought to life by Shane Evans while hacking a scraping framework prototype for Mydeco (mydeco.com). It soon became maintained, extended and -improved by Insophia (insophia.com), with the sponsorship of By Design (the -company behind Mydeco). +improved by Insophia (insophia.com), with the initial sponsorship of Mydeco to +bootstrap the project. -Here is the list of the primary authors & contributors, along with their user -name (in Scrapy trac/subversion). Emails are intentionally left out to avoid -spam. +Here is the list of the primary authors & contributors: - * Pablo Hoffman (pablo) - * Daniel Graña (daniel) - * Martin Olveyra (olveyra) - * Gabriel García (elpolilla) - * Michael Cetrulo (samus_) - * Artem Bogomyagkov (artem) - * Damian Canabal (calarval) - * Andres Moreira (andres) - * Ismael Carnales (ismael) - * Matías Aguirre (omab) - * German Hoffman (german) - * Anibal Pacheco (anibal) + * Pablo Hoffman + * Daniel Graña + * Martin Olveyra + * Gabriel García + * Michael Cetrulo + * Artem Bogomyagkov + * Damian Canabal + * Andres Moreira + * Ismael Carnales + * Matías Aguirre + * German Hoffmann + * Anibal Pacheco * Bruno Deferrari * Shane Evans - -And here is the list of people who have helped to put the Scrapy homepage live: - - * Ezequiel Rivero (ezequiel) + * Ezequiel Rivero + * Patrick Mezard + * Rolando Espinoza From 7b1ad321e3beda87afd7fc8d66d3bdf812a8b763 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente <darkrho@gmail.com> Date: Fri, 19 Feb 2010 21:31:17 -0400 Subject: [PATCH 10/68] examples/experimental: added imdb top movies spider --- examples/experimental/imdb/imdb/__init__.py | 1 + examples/experimental/imdb/imdb/items.py | 12 ++ examples/experimental/imdb/imdb/pipelines.py | 8 + examples/experimental/imdb/imdb/settings.py | 20 +++ .../imdb/imdb/spiders/__init__.py | 8 + .../imdb/imdb/spiders/imdb_site.py | 140 ++++++++++++++++++ examples/experimental/imdb/scrapy-ctl.py | 7 + 7 files changed, 196 insertions(+) create mode 100644 examples/experimental/imdb/imdb/__init__.py create mode 100644 examples/experimental/imdb/imdb/items.py create mode 100644 examples/experimental/imdb/imdb/pipelines.py create mode 100644 examples/experimental/imdb/imdb/settings.py create mode 100644 examples/experimental/imdb/imdb/spiders/__init__.py create mode 100644 examples/experimental/imdb/imdb/spiders/imdb_site.py create mode 100644 examples/experimental/imdb/scrapy-ctl.py diff --git a/examples/experimental/imdb/imdb/__init__.py b/examples/experimental/imdb/imdb/__init__.py new file mode 100644 index 000000000..5bb534f79 --- /dev/null +++ b/examples/experimental/imdb/imdb/__init__.py @@ -0,0 +1 @@ +# package diff --git a/examples/experimental/imdb/imdb/items.py b/examples/experimental/imdb/imdb/items.py new file mode 100644 index 000000000..03bb5c2c3 --- /dev/null +++ b/examples/experimental/imdb/imdb/items.py @@ -0,0 +1,12 @@ +# Define here the models for your scraped items +# +# See documentation in: +# http://doc.scrapy.org/topics/items.html + +from scrapy.item import Item, Field + +class ImdbItem(Item): + # define the fields for your item here like: + # name = Field() + title = Field() + url = Field() diff --git a/examples/experimental/imdb/imdb/pipelines.py b/examples/experimental/imdb/imdb/pipelines.py new file mode 100644 index 000000000..e60714159 --- /dev/null +++ b/examples/experimental/imdb/imdb/pipelines.py @@ -0,0 +1,8 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: http://doc.scrapy.org/topics/item-pipeline.html + +class ImdbPipeline(object): + def process_item(self, spider, item): + return item diff --git a/examples/experimental/imdb/imdb/settings.py b/examples/experimental/imdb/imdb/settings.py new file mode 100644 index 000000000..de026dc14 --- /dev/null +++ b/examples/experimental/imdb/imdb/settings.py @@ -0,0 +1,20 @@ +# Scrapy settings for imdb project +# +# For simplicity, this file contains only the most important settings by +# default. All the other settings are documented here: +# +# http://doc.scrapy.org/topics/settings.html +# +# Or you can copy and paste them from where they're defined in Scrapy: +# +# scrapy/conf/default_settings.py +# + +BOT_NAME = 'imdb' +BOT_VERSION = '1.0' + +SPIDER_MODULES = ['imdb.spiders'] +NEWSPIDER_MODULE = 'imdb.spiders' +DEFAULT_ITEM_CLASS = 'imdb.items.ImdbItem' +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) + diff --git a/examples/experimental/imdb/imdb/spiders/__init__.py b/examples/experimental/imdb/imdb/spiders/__init__.py new file mode 100644 index 000000000..5065ccba5 --- /dev/null +++ b/examples/experimental/imdb/imdb/spiders/__init__.py @@ -0,0 +1,8 @@ +# This package will contain the spiders of your Scrapy project +# +# To create the first spider for your project use this command: +# +# scrapy-ctl.py genspider myspider myspider-domain.com +# +# For more info see: +# http://doc.scrapy.org/topics/spiders.html diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py new file mode 100644 index 000000000..5c3f3c06b --- /dev/null +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -0,0 +1,140 @@ +from scrapy.http import Request +from scrapy.selector import HtmlXPathSelector +from scrapy.contrib.loader import XPathItemLoader +from scrapy.contrib_exp.crawlspider import CrawlSpider, Rule +from scrapy.contrib_exp.crawlspider.reqext import SgmlRequestExtractor +from scrapy.contrib_exp.crawlspider.reqproc import Canonicalize, \ + FilterDupes, FilterUrl +from scrapy.utils.url import urljoin_rfc + +from imdb.items import ImdbItem, Field + +from itertools import chain, imap, izip + +class UsaOpeningWeekMovie(ImdbItem): + pass + +class UsaTopWeekMovie(ImdbItem): + pass + +class Top250Movie(ImdbItem): + rank = Field() + rating = Field() + year = Field() + votes = Field() + +class MovieItem(ImdbItem): + release_date = Field() + tagline = Field() + + +class ImdbSiteSpider(CrawlSpider): + domain_name = 'imdb.com' + start_urls = ['http://www.imdb.com/'] + + # extract requests using this classes from urls matching 'follow' flag + request_extractors = [ + SgmlRequestExtractor(tags=['a'], attrs=['href']), + ] + + # process requests using this classes from urls matching 'follow' flag + request_processors = [ + Canonicalize(), + FilterDupes(), + FilterUrl(deny=r'/tt\d+/$'), # deny movie url as we will dispatch + # manually the movie requests + ] + + # include domain bit for demo purposes + rules = ( + # these two rules expects requests from start url + Rule(r'imdb.com/nowplaying/$', 'parse_now_playing'), + Rule(r'imdb.com/chart/top$', 'parse_top_250'), + # this rule will parse requests manually dispatched + Rule(r'imdb.com/title/tt\d+/$', 'parse_movie_info'), + ) + + def parse_now_playing(self, response): + """Scrapes USA openings this week and top 10 in week""" + self.log("Parsing USA Top Week") + hxs = HtmlXPathSelector(response) + + _urljoin = lambda url: self._urljoin(response, url) + + # + # openings this week + # + openings = hxs.select('//table[@class="movies"]//a[@class="title"]') + boxoffice = hxs.select('//table[@class="boxoffice movies"]//a[@class="title"]') + + opening_titles = openings.select('text()').extract() + opening_urls = imap(_urljoin, openings.select('@href').extract()) + + box_titles = boxoffice.select('text()').extract() + box_urls = imap(_urljoin, boxoffice.select('@href').extract()) + + # items + opening_items = (UsaOpeningWeekMovie(title=title, url=url) + for (title, url) + in izip(opening_titles, opening_urls)) + + box_items = (UsaTopWeekMovie(title=title, url=url) + for (title, url) + in izip(box_titles, box_urls)) + + # movie requests + requests = imap(self.make_requests_from_url, + chain(opening_urls, box_urls)) + + return chain(opening_items, box_items, requests) + + def parse_top_250(self, response): + """Scrapes movies from top 250 list""" + self.log("Parsing Top 250") + hxs = HtmlXPathSelector(response) + + # scrap each row in the table + rows = hxs.select('//div[@id="main"]/table/tr//a/ancestor::tr') + for row in rows: + fields = row.select('td//text()').extract() + url, = row.select('td//a/@href').extract() + url = self._urljoin(response, url) + + item = Top250Movie() + item['title'] = fields[2] + item['url'] = url + item['rank'] = fields[0] + item['rating'] = fields[1] + item['year'] = fields[3] + item['votes'] = fields[4] + + # scrapped top250 item + yield item + # fetch movie + yield self.make_requests_from_url(url) + + def parse_movie_info(self, response): + """Scrapes movie information""" + self.log("Parsing Movie Info") + hxs = HtmlXPathSelector(response) + selector = hxs.select('//div[@class="maindetails"]') + + item = MovieItem() + # set url + item['url'] = response.url + + # use item loader for other attributes + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('title', './/h1/text()') + l.add_xpath('release_date', './/h5[text()="Release Date:"]' + '/following-sibling::div/text()') + l.add_xpath('tagline', './/h5[text()="Tagline:"]' + '/following-sibling::div/text()') + + yield l.load_item() + + def _urljoin(self, response, url): + """Helper to convert relative urls to absolute""" + return urljoin_rfc(response.url, url, response.encoding) + +SPIDER = ImdbSiteSpider() diff --git a/examples/experimental/imdb/scrapy-ctl.py b/examples/experimental/imdb/scrapy-ctl.py new file mode 100644 index 000000000..df57621b3 --- /dev/null +++ b/examples/experimental/imdb/scrapy-ctl.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +import os +os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'imdb.settings') + +from scrapy.command.cmdline import execute +execute() From 180c091fb2a281ab1f79f9897b32687a877582e6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Wed, 24 Feb 2010 14:01:29 -0200 Subject: [PATCH 11/68] Fixed encoding issue (reported in #135) when the encoding declared in the HTTP header is unknown. This is the patch proposed by Rolando, with an update to the Request/Response documentation. --- docs/topics/request-response.rst | 8 +++++--- scrapy/http/response/text.py | 8 +++++++- scrapy/tests/test_http_response.py | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3472b1937..9f4dcdc46 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -466,12 +466,14 @@ TextResponse objects .. attribute:: TextResponse.encoding - A string with the encoding of this response. The encoding is resolved in the - following order: + A string with the encoding of this response. The encoding is resolved by + trying the following mechanisms, in order: 1. the encoding passed in the constructor `encoding` argument - 2. the encoding declared in the Content-Type HTTP header + 2. the encoding declared in the Content-Type HTTP header. If this + encoding is not valid (ie. unknown), it is ignored and the next + resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class doesn't provide any special functionality for this. However, the diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1c13e729c..d42a89197 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +import codecs import re from scrapy.xlib.BeautifulSoup import UnicodeDammit @@ -64,7 +65,12 @@ class TextResponse(Response): if content_type: encoding = self._ENCODING_RE.search(content_type) if encoding: - return encoding.group(1) + enc = encoding.group(1) + try: + codecs.lookup(enc) # check if the encoding is valid + return enc + except LookupError: + pass @memoizemethod_noargs def body_as_unicode(self): diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 3b0a144d2..5851572ac 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -175,6 +175,8 @@ class TextResponseTest(BaseResponseTest): r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") + r5 = self.response_class("http://www.example.com", + headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") self.assertEqual(r1.headers_encoding(), "utf-8") self.assertEqual(r2.headers_encoding(), None) @@ -182,6 +184,8 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3.headers_encoding(), "iso-8859-1") self.assertEqual(r3.encoding, 'iso-8859-1') self.assertEqual(r4.headers_encoding(), None) + self.assertEqual(r5.headers_encoding(), None) + self.assertEqual(r5.encoding, "utf-8") assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3") From d12cd22d5eacf6e09853db59708dc219e0d46486 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Thu, 4 Mar 2010 10:15:58 -0200 Subject: [PATCH 12/68] switched default scheduler order to DFO, which consumes less memory by default --- docs/topics/settings.rst | 2 +- scrapy/conf/default_settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d7a7d7fcf..4058eec3e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -803,7 +803,7 @@ The scheduler to use for crawling. SCHEDULER_ORDER --------------- -Default: ``'BFO'`` +Default: ``'DFO'`` Scope: ``scrapy.core.scheduler`` diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 76feb0c6f..3ee377a44 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -152,7 +152,7 @@ SCHEDULER_MIDDLEWARES_BASE = { 'scrapy.contrib.schedulermiddleware.duplicatesfilter.DuplicatesFilterMiddleware': 500, } -SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO +SCHEDULER_ORDER = 'DFO' SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager' From 861f9691c7bde83d7352cca721f5069fb68abeb8 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Thu, 4 Mar 2010 10:40:41 -0200 Subject: [PATCH 13/68] removed partly-obsolete module scrapy.contrib.groupsettings --- scrapy/contrib/groupsettings.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 scrapy/contrib/groupsettings.py diff --git a/scrapy/contrib/groupsettings.py b/scrapy/contrib/groupsettings.py deleted file mode 100644 index 4bad4100b..000000000 --- a/scrapy/contrib/groupsettings.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Extensions to override scrapy settings with per-group settings according to the -group the spider belongs to. It only overrides the settings when running the -crawl command with *only one domain as argument*. -""" - -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured -from scrapy.command.cmdline import command_executed - -class GroupSettings(object): - - def __init__(self): - if not settings.getbool("GROUPSETTINGS_ENABLED"): - raise NotConfigured - - if command_executed and command_executed['name'] == 'crawl': - mod = __import__(settings['GROUPSETTINGS_MODULE'], {}, {}, ['']) - args = command_executed['args'] - if len(args) == 1 and not args[0].startswith('http://'): - domain = args[0] - settings.overrides.update(mod.default_settings) - for group, domains in mod.group_spiders.iteritems(): - if domain in domains: - settings.overrides.update(mod.group_settings.get(group, {})) - From 4c1ec0c97ec85a0c94b424b2f8543dcb017dfae1 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Thu, 4 Mar 2010 10:58:18 -0200 Subject: [PATCH 14/68] replaced hacky command_executed dict by standard signal --- scrapy/command/cmdline.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index 25f531cea..e51b7e8bf 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -7,20 +7,14 @@ import cProfile import scrapy from scrapy import log -from scrapy.spider import spiders from scrapy.xlib import lsprofcalltree from scrapy.conf import settings from scrapy.command.models import ScrapyCommand +from scrapy.utils.signal import send_catch_log -# This dict holds information about the executed command for later use -command_executed = {} - -def _save_command_executed(cmdname, cmd, args, opts): - """Save command executed info for later reference""" - command_executed['name'] = cmdname - command_executed['class'] = cmd - command_executed['args'] = args[:] - command_executed['opts'] = opts.__dict__.copy() +# Signal that carries information about the command which was executed +# args: cmdname, cmdobj, args, opts +command_executed = object() def _find_commands(dir): try: @@ -127,7 +121,8 @@ def execute(argv=None): sys.exit(2) del args[0] # remove command name from args - _save_command_executed(cmdname, cmd, args, opts) + send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \ + args=args, opts=opts) from scrapy.core.manager import scrapymanager scrapymanager.configure(control_reactor=True) ret = _run_command(cmd, args, opts) From a505a9d490143b8aa745930f1977798eeb4d9c84 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Thu, 4 Mar 2010 11:09:16 -0200 Subject: [PATCH 15/68] minor code refactoring on scrapy.command.cmdline module --- scrapy/command/cmdline.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index e51b7e8bf..1a190549d 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -131,23 +131,25 @@ def execute(argv=None): def _run_command(cmd, args, opts): if opts.profile or opts.lsprof: - if opts.profile: - log.msg("writing cProfile stats to %r" % opts.profile) - if opts.lsprof: - log.msg("writing lsprof stats to %r" % opts.lsprof) - loc = locals() - p = cProfile.Profile() - p.runctx('ret = cmd.run(args, opts)', globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) - k = lsprofcalltree.KCacheGrind(p) - if opts.lsprof: - with open(opts.lsprof, 'w') as f: - k.output(f) - ret = loc['ret'] + return _run_command_profiled(cmd, args, opts) else: - ret = cmd.run(args, opts) - return ret + return cmd.run(args, opts) + +def _run_command_profiled(cmd, args, opts): + if opts.profile: + log.msg("writing cProfile stats to %r" % opts.profile) + if opts.lsprof: + log.msg("writing lsprof stats to %r" % opts.lsprof) + loc = locals() + p = cProfile.Profile() + p.runctx('ret = cmd.run(args, opts)', globals(), loc) + if opts.profile: + p.dump_stats(opts.profile) + k = lsprofcalltree.KCacheGrind(p) + if opts.lsprof: + with open(opts.lsprof, 'w') as f: + k.output(f) + return loc['ret'] if __name__ == '__main__': execute() From 39e4df0cff0af45c9e7629fca611e67f9ceb8e48 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Wed, 10 Mar 2010 00:10:36 -0200 Subject: [PATCH 16/68] removed unmaintained (and untested) contrib_exp ShoveItemPipeline --- scrapy/contrib_exp/pipeline/shoveitem.py | 55 ------------------------ 1 file changed, 55 deletions(-) delete mode 100644 scrapy/contrib_exp/pipeline/shoveitem.py diff --git a/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py deleted file mode 100644 index 869c62c58..000000000 --- a/scrapy/contrib_exp/pipeline/shoveitem.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -A pipeline to persist objects using shove. - -Shove is a "new generation" shelve. For more information see: -http://pypi.python.org/pypi/shove -""" - -from string import Template - -from shove import Shove -from scrapy.xlib.pydispatch import dispatcher - -from scrapy import log -from scrapy.core import signals -from scrapy.conf import settings -from scrapy.core.exceptions import NotConfigured - -class ShoveItemPipeline(object): - - def __init__(self): - self.uritpl = settings['SHOVEITEM_STORE_URI'] - if not self.uritpl: - raise NotConfigured - self.opts = settings['SHOVEITEM_STORE_OPT'] or {} - self.stores = {} - - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def process_item(self, spider, item): - guid = str(item.guid) - - if guid in self.stores[spider]: - if self.stores[spider][guid] == item: - status = 'old' - else: - status = 'upd' - else: - status = 'new' - - if not status == 'old': - self.stores[spider][guid] = item - self.log(spider, item, status) - return item - - def spider_opened(self, spider): - uri = Template(self.uritpl).substitute(domain=spider.domain_name) - self.stores[spider] = Shove(uri, **self.opts) - - def spider_closed(self, spider): - self.stores[spider].sync() - - def log(self, spider, item, status): - log.msg("Shove (%s): Item guid=%s" % (status, item.guid), level=log.DEBUG, \ - spider=spider) From 38a296aa2c04adf9b136ba8000f4f0b7f2364e61 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Fri, 12 Mar 2010 09:52:39 -0200 Subject: [PATCH 17/68] Added tests to open_in_browser() function --- scrapy/tests/test_utils_response.py | 14 ++++++++++++-- scrapy/utils/response.py | 6 +++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 9281f4f40..97d443ef3 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -1,9 +1,9 @@ import unittest from scrapy.xlib.BeautifulSoup import BeautifulSoup -from scrapy.http import Response, TextResponse +from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh, \ - response_httprepr, get_cached_beautifulsoup + response_httprepr, get_cached_beautifulsoup, open_in_browser class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body='dummy_response') @@ -131,5 +131,15 @@ class ResponseUtilsTest(unittest.TestCase): assert soup1 is soup2 assert soup1 is not soup3 + def test_open_in_browser(self): + url = "http:///www.example.com/some/page.html" + body = "<html> <head> <title>test page test body " + response = HtmlResponse(url, body=body) + newbody = open_in_browser(response, debug=True) + assert '' % url in newbody + + self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \ + debug=True) + if __name__ == "__main__": unittest.main() diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 6ed6f43a3..66aacacc5 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -92,7 +92,7 @@ def response_httprepr(response): s += response.body return s -def open_in_browser(response): +def open_in_browser(response, debug=False): """Open the given response in a local web browser, populating the tag for external links to work """ @@ -106,4 +106,8 @@ def open_in_browser(response): fd, fname = tempfile.mkstemp('.html') os.write(fd, body) os.close(fd) + if debug: # for testing purposes only + body = open(fname).read() + os.remove(fname) + return body webbrowser.open("file://%s" % fname) From 54ae2c36d025cfcdfd35ffb19d3f582ea2c67f95 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 12 Mar 2010 10:19:50 -0200 Subject: [PATCH 18/68] better implementation of open_in_browser() tests --- scrapy/tests/test_utils_response.py | 10 +++++++--- scrapy/utils/response.py | 8 ++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 97d443ef3..de506b639 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -1,4 +1,5 @@ import unittest +import urlparse from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, TextResponse, HtmlResponse @@ -134,10 +135,13 @@ class ResponseUtilsTest(unittest.TestCase): def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" body = " test page test body " + def browser_open(burl): + bbody = open(urlparse.urlparse(burl).path).read() + assert '' % url in bbody, " tag not added" + return True response = HtmlResponse(url, body=body) - newbody = open_in_browser(response, debug=True) - assert '' % url in newbody - + assert open_in_browser(response, _openfunc=browser_open), \ + "Browser not called" self.assertRaises(TypeError, open_in_browser, Response(url, body=body), \ debug=True) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 66aacacc5..56c26c2f0 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -92,7 +92,7 @@ def response_httprepr(response): s += response.body return s -def open_in_browser(response, debug=False): +def open_in_browser(response, _openfunc=webbrowser.open): """Open the given response in a local web browser, populating the tag for external links to work """ @@ -106,8 +106,4 @@ def open_in_browser(response, debug=False): fd, fname = tempfile.mkstemp('.html') os.write(fd, body) os.close(fd) - if debug: # for testing purposes only - body = open(fname).read() - os.remove(fname) - return body - webbrowser.open("file://%s" % fname) + return _openfunc("file://%s" % fname) From 403a21ec743325eb289811c87b69ed453361c95b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 12 Mar 2010 17:28:33 -0200 Subject: [PATCH 19/68] removed obsolete scrapy.crawler module --- examples/scripts/count_and_follow_links.py | 51 ----------------- scrapy/crawler.py | 66 ---------------------- 2 files changed, 117 deletions(-) delete mode 100644 examples/scripts/count_and_follow_links.py delete mode 100644 scrapy/crawler.py diff --git a/examples/scripts/count_and_follow_links.py b/examples/scripts/count_and_follow_links.py deleted file mode 100644 index 4ead870fc..000000000 --- a/examples/scripts/count_and_follow_links.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -Simple script to follow links from a start url. The links are followed in no -particular order. - -Usage: -count_and_follow_links.py - -Example: -count_and_follow_links.py http://scrapy.org/ 20 - -For each page visisted, this script will print the page body size and the -number of links found. -""" - -import sys -from urlparse import urljoin - -from scrapy.crawler import Crawler -from scrapy.selector import HtmlXPathSelector -from scrapy.http import Request, HtmlResponse - -links_followed = 0 - -def parse(response): - global links_followed - links_followed += 1 - if links_followed >= links_to_follow: - crawler.stop() - - # ignore non-HTML responses - if not isinstance(response, HtmlResponse): - return - - links = HtmlXPathSelector(response).select('//a/@href').extract() - abslinks = [urljoin(response.url, l) for l in links] - - print "page %2d/%d: %s" % (links_followed, links_to_follow, response.url) - print " size : %d bytes" % len(response.body) - print " links: %d" % len(links) - print - - return [Request(l, callback=parse) for l in abslinks] - -if len(sys.argv) != 3: - print __doc__ - sys.exit(2) - -start_url, links_to_follow = sys.argv[1], int(sys.argv[2]) -request = Request(start_url, callback=parse) -crawler = Crawler() -crawler.crawl(request) diff --git a/scrapy/crawler.py b/scrapy/crawler.py deleted file mode 100644 index e793d9125..000000000 --- a/scrapy/crawler.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Crawler class - -The Crawler class can be used to crawl pages using the Scrapy crawler from -outside a Scrapy project, for example, from a standalone script. - -To use it, instantiate it and call the "crawl" method with one (or more) -requests. For example: - - >>> from scrapy.crawler import Crawler - >>> from scrapy.http import Request - >>> def parse_response(response): - ... print "Visited: %s" % response.url - ... - >>> request = Request('http://scrapy.org', callback=parse_response) - >>> crawler = Crawler() - >>> crawler.crawl(request) - Visited: http://scrapy.org - >>> - -Request callbacks follow the same API of spiders callback, which means that all -requests returned from the callbacks will be followed. - -See examples/scripts/count_and_follow_links.py for a more detailed example. - -WARNING: The Crawler class currently has a big limitation - it cannot be used -more than once in the same Python process. This is due to the fact that Twisted -reactors cannot be restarted. Hopefully, this limitation will be removed in the -future. -""" - -from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.conf import settings as scrapy_settings -from scrapy import log - -class Crawler(object): - - def __init__(self, enable_log=False, stop_on_error=False, silence_errors=False, \ - settings=None): - self.stop_on_error = stop_on_error - self.silence_errors = silence_errors - # disable offsite middleware (by default) because it prevents free crawling - if settings is not None: - settings.overrides.update(settings) - scrapy_settings.overrides['SPIDER_MIDDLEWARES'] = { - 'scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware': None} - scrapy_settings.overrides['LOG_ENABLED'] = enable_log - scrapymanager.configure() - dispatcher.connect(self._logmessage_received, signal=log.logmessage_received) - - def crawl(self, *args): - scrapymanager.runonce(*args) - - def stop(self): - scrapyengine.stop() - log.log_level = log.SILENT - scrapyengine.kill() - - def _logmessage_received(self, message, level): - if level <= log.ERROR: - if not self.silence_errors: - print "Crawler error: %s" % message - if self.stop_on_error: - self.stop() From 87e68e74381b6961c59c16efd212cbcb2d9e0386 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 22 Mar 2010 13:37:37 -0300 Subject: [PATCH 20/68] Made MailSender non IO-blocking, and improved MailSender documentation --- docs/topics/email.rst | 60 ++++++++++++++++++++++++++++--------------- scrapy/mail.py | 30 ++++++++-------------- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 7fc2f9d60..984132eaf 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -5,14 +5,17 @@ Sending email ============= .. module:: scrapy.mail - :synopsis: Helpers to easily send e-mail. + :synopsis: Email sending facility Although Python makes sending e-mail relatively easy via the `smtplib`_ -library, Scrapy provides its own class for sending emails which is very easy to -use and it's implemented using `Twisted non-blocking IO`_, to avoid affecting -the crawling performance. +library, Scrapy provides its own facility for sending emails which is very easy +to use and it's implemented using `Twisted non-blocking IO`_, to avoid +interfering with the non-blocking IO of the crawler. + +It's also very easy to configure by just configuring a few settings. .. _smtplib: http://docs.python.org/library/smtplib.html +.. _Twisted non-blocking IO: http://twistedmatrix.com/projects/core/documentation/howto/async.html It also has built-in support for sending attachments. @@ -34,30 +37,45 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. .. class:: MailSender(smtphost, mailfrom) - ``smtphost`` is a string with the SMTP host to use for sending the emails. - If omitted, :setting:`MAIL_HOST` will be used. + :param smtphost: the SMTP host to use for sending the emails. If omitted, + :setting:`MAIL_HOST` setting will be used. + :type smtphost: str - ``mailfrom`` is a string with the email address to use for sending messages - (in the ``From:`` header). If omitted, :setting:`MAIL_FROM` will be used. + :param mailfrom: the address used to send emails (in the ``From:`` header). + If omitted, :setting:`MAIL_FROM` setting will be used. + :type mailfrom: str -.. method:: MailSender.send(to, subject, body, cc=None, attachs=()) + .. method:: send(to, subject, body, cc=None, attachs=()) - Send mail to the given recipients + Send email to the given recipients - ``to`` is a list of email recipients + :param to: the email recipients + :type to: list - ``subject`` is a string with the subject of the message + :param subject: the subject of the email + :type subject: str - ``cc`` is a list of emails to CC + :param cc: the emails to CC + :type cc: list - ``body`` is a string with the body of the message + :param body: the email body + :type body: str - ``attachs`` is an iterable of tuples (attach_name, mimetype, file_object) - where: - - ``attach_name`` is a string with the name will appear on the emails attachment - ``mimetype`` is the mimetype of the attachment - ``file_object`` is a readable file object + :param attachs: an iterable of tuples ``(attach_name, mimetype, + file_object)`` where ``attach_name`` is a string with the name will + appear on the emails attachment, ``mimetype`` is the mimetype of the + attachment and ``file_object`` is a readable file object with the + contents of the attachment + :type attachs: iterable -.. _Twisted non-blocking IO: http://twistedmatrix.com/projects/core/documentation/howto/async.html +MailSender settings +=================== + +These settings define the default constructor values of the :class:`MailSender` +class, and can be used to configure email notifications in your project without +writing any code (for those extensions that use the :class:`MailSender` class): + +* :setting:`MAIL_FROM` +* :setting:`MAIL_HOST` + diff --git a/scrapy/mail.py b/scrapy/mail.py index 8275e75cb..87825ed9c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -47,34 +47,26 @@ class MailSender(object): part = MIMEBase(*mimetype.split('/')) part.set_payload(f.read()) Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="%s"' % attach_name) + part.add_header('Content-Disposition', 'attachment; filename="%s"' \ + % attach_name) msg.attach(part) else: msg.set_payload(body) - # FIXME --------------------------------------------------------------------- - # There seems to be a problem with sending emails using deferreds when - # the last thing left to do is sending the mail, cause the engine stops - # the reactor and the email don't get send. we need to fix this. until - # then, we'll revert to use Python standard (IO-blocking) smtplib. - - #dfd = self._sendmail(self.smtphost, self.mailfrom, rcpts, msg.as_string()) - #dfd.addCallbacks(self._sent_ok, self._sent_failed, - # callbackArgs=[to, cc, subject, len(attachs)], - # errbackArgs=[to, cc, subject, len(attachs)]) - import smtplib - smtp = smtplib.SMTP(self.smtphost) - smtp.sendmail(self.mailfrom, rcpts, msg.as_string()) - log.msg('Mail sent: To=%s Cc=%s Subject="%s"' % (to, cc, subject)) - smtp.close() - # --------------------------------------------------------------------------- + dfd = self._sendmail(self.smtphost, self.mailfrom, rcpts, msg.as_string()) + dfd.addCallbacks(self._sent_ok, self._sent_failed, + callbackArgs=[to, cc, subject, len(attachs)], + errbackArgs=[to, cc, subject, len(attachs)]) + reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) def _sent_ok(self, result, to, cc, subject, nattachs): - log.msg('Mail sent OK: To=%s Cc=%s Subject="%s" Attachs=%d' % (to, cc, subject, nattachs)) + log.msg('Mail sent OK: To=%s Cc=%s Subject="%s" Attachs=%d' % \ + (to, cc, subject, nattachs)) def _sent_failed(self, failure, to, cc, subject, nattachs): errstr = str(failure.value) - log.msg('Unable to send mail: To=%s Cc=%s Subject="%s" Attachs=%d - %s' % (to, cc, subject, nattachs, errstr), level=log.ERROR) + log.msg('Unable to send mail: To=%s Cc=%s Subject="%s" Attachs=%d - %s' % \ + (to, cc, subject, nattachs, errstr), level=log.ERROR) def _sendmail(self, smtphost, from_addr, to_addrs, msg, port=25): """ This is based on twisted.mail.smtp.sendmail except that it From 4fa833c849e98ff9d0fe1e063db575f23f1d929a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 24 Mar 2010 12:13:38 -0300 Subject: [PATCH 21/68] Added LOG_ENCODING setting --- docs/topics/logging.rst | 11 +++++++++++ docs/topics/settings.rst | 11 ++++++++++- scrapy/log.py | 10 ++++++---- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index f57128287..5e1a0e4bb 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -129,3 +129,14 @@ scrapy.log module Log level for debugging messages (recommended level for development) +Logging settings +================ + +These settings can be used to configure the logging: + +* :setting:`LOG_ENABLED` +* :setting:`LOG_ENCODING` +* :setting:`LOG_FILE` +* :setting:`LOG_LEVEL` +* :setting:`LOG_STDOUT` + diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4058eec3e..ffb2921e7 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -526,7 +526,16 @@ LOG_ENABLED Default: ``True`` -Enable logging. +Whether to enable logging. + +.. setting:: LOG_ENCODING + +LOG_ENCODING +------------ + +Default: ``'utf-8'`` + +The encoding to use for logging. .. setting:: LOG_FILE diff --git a/scrapy/log.py b/scrapy/log.py index 71897ffee..c4f9287e5 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -29,8 +29,9 @@ BOT_NAME = settings['BOT_NAME'] # args: message, level, spider logmessage_received = object() -# default logging level +# default values log_level = DEBUG +log_encoding = 'utf-8' started = False @@ -47,11 +48,12 @@ def _get_log_level(level_name_or_id=None): def start(logfile=None, loglevel=None, logstdout=None): """Initialize and start logging facility""" - global log_level, started + global log_level, log_encoding, started if started or not settings.getbool('LOG_ENABLED'): return log_level = _get_log_level(loglevel) + log_encoding = settings['LOG_ENCODING'] started = True # set log observer @@ -74,7 +76,7 @@ def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): dispatcher.send(signal=logmessage_received, message=message, level=level, \ spider=spider) system = domain or (spider.domain_name if spider else component) - msg_txt = unicode_to_str("%s: %s" % (level_names[level], message)) + msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding) log.msg(msg_txt, system=system) def exc(message, level=ERROR, component=BOT_NAME, domain=None, spider=None): @@ -93,5 +95,5 @@ def err(_stuff=None, _why=None, **kwargs): "use 'spider' argument instead", DeprecationWarning, stacklevel=2) kwargs['system'] = domain or (spider.domain_name if spider else component) if _why: - _why = unicode_to_str("ERROR: %s" % _why) + _why = unicode_to_str("ERROR: %s" % _why, log_encoding) log.err(_stuff, _why, **kwargs) From 45411926b52c678e079ae3d5bb2c80bcfd9a6974 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 24 Mar 2010 12:14:07 -0300 Subject: [PATCH 22/68] Improved encoding support by explicitly passing encoding to all str_to_unicode() and unicode_to_str() calls --- scrapy/__init__.py | 1 + scrapy/conf/default_settings.py | 1 + scrapy/contrib/linkextractors/image.py | 12 ++++----- scrapy/selector/__init__.py | 4 +-- scrapy/utils/markup.py | 35 +++++++++++++------------- scrapy/utils/python.py | 8 ++++-- scrapy/utils/url.py | 5 ++-- 7 files changed, 37 insertions(+), 29 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 6f8ca15e6..4a62ea9c3 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -18,6 +18,7 @@ from scrapy.xlib import twisted_250_monkeypatches from scrapy.utils.encoding import add_encoding_alias add_encoding_alias('gb2312', 'zh-cn') add_encoding_alias('cp1251', 'win-1251') +add_encoding_alias('cp1252', 'iso8859-1', overwrite=True) # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 3ee377a44..5550e75ba 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -101,6 +101,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' ITEM_PIPELINES = [] LOG_ENABLED = True +LOG_ENCODING = 'utf-8' LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline' LOG_STDOUT = False LOG_LEVEL = 'DEBUG' diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 711623b81..79048a496 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -3,7 +3,6 @@ This module implements the HtmlImageLinkExtractor for extracting image links only. """ -import urlparse from scrapy.link import Link from scrapy.utils.url import canonicalize_url, urljoin_rfc @@ -25,13 +24,13 @@ class HTMLImageLinkExtractor(object): self.unique = unique self.canonicalize = canonicalize - def extract_from_selector(self, selector, parent=None): + def extract_from_selector(self, selector, encoding, parent=None): ret = [] def _add_link(url_sel, alt_sel=None): url = flatten([url_sel.extract()]) alt = flatten([alt_sel.extract()]) if alt_sel else (u'', ) if url: - ret.append(Link(unicode_to_str(url[0]), alt[0])) + ret.append(Link(unicode_to_str(url[0], encoding), alt[0])) if selector.xmlNode.type == 'element': if selector.xmlNode.name == 'img': @@ -41,7 +40,7 @@ class HTMLImageLinkExtractor(object): children = selector.select('child::*') if len(children): for child in children: - ret.extend(self.extract_from_selector(child, parent=selector)) + ret.extend(self.extract_from_selector(child, encoding, parent=selector)) elif selector.xmlNode.name == 'a' and not parent: _add_link(selector.select('@href'), selector.select('@title')) else: @@ -52,7 +51,8 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) base_url = xs.select('//base/@href').extract() - base_url = unicode_to_str(base_url[0]) if base_url else unicode_to_str(response.url) + base_url = unicode_to_str(base_url[0], response.encoding) if base_url \ + else unicode_to_str(response.url, response.encoding) links = [] for location in self.locations: @@ -64,7 +64,7 @@ class HTMLImageLinkExtractor(object): continue for selector in selectors: - links.extend(self.extract_from_selector(selector)) + links.extend(self.extract_from_selector(selector, response.encoding)) seen, ret = set(), [] for link in links: diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 37f86d431..bf2dc52c7 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -29,8 +29,8 @@ class XPathSelector(object_ref): self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc elif text: - response = TextResponse(url='about:blank', body=unicode_to_str(text), \ - encoding='utf-8') + response = TextResponse(url='about:blank', \ + body=unicode_to_str(text, 'utf-8'), encoding='utf-8') self.doc = Libxml2Document(response, factory=self._get_libxml2_doc) self.xmlNode = self.doc.xmlDoc self.expr = expr diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index d422fbde5..5e2e0b1c1 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -60,10 +60,10 @@ def remove_entities(text, keep=(), remove_illegal=True, encoding='utf-8'): return _ent_re.sub(convert_entity, str_to_unicode(text, encoding)) -def has_entities(text): - return bool(_ent_re.search(str_to_unicode(text))) +def has_entities(text, encoding=None): + return bool(_ent_re.search(str_to_unicode(text, encoding))) -def replace_tags(text, token=''): +def replace_tags(text, token='', encoding=None): """Replace all markup tags found in the given text by the given token. By default token is a null string so it just remove all tags. @@ -71,43 +71,44 @@ def replace_tags(text, token=''): Always returns a unicode string. """ - return _tag_re.sub(token, str_to_unicode(text)) + return _tag_re.sub(token, str_to_unicode(text, encoding)) -def remove_comments(text): +def remove_comments(text, encoding=None): """ Remove HTML Comments. """ - return re.sub('', u'', str_to_unicode(text), re.DOTALL) + return re.sub('', u'', str_to_unicode(text, encoding), re.DOTALL) -def remove_tags(text, which_ones=()): +def remove_tags(text, which_ones=(), encoding=None): """ Remove HTML Tags only. which_ones -- is a tuple of which tags we want to remove. if is empty remove all tags. """ if which_ones: - tags = ['<%s>|<%s .*?>|' % (tag,tag,tag) for tag in which_ones] + tags = ['<%s>|<%s .*?>|' % (tag, tag, tag) for tag in which_ones] regex = '|'.join(tags) else: regex = '<.*?>' retags = re.compile(regex, re.DOTALL | re.IGNORECASE) - return retags.sub(u'', str_to_unicode(text)) + return retags.sub(u'', str_to_unicode(text, encoding)) -def remove_tags_with_content(text, which_ones=()): +def remove_tags_with_content(text, which_ones=(), encoding=None): """ Remove tags and its content. which_ones -- is a tuple of which tags with its content we want to remove. if is empty do nothing. """ - text = str_to_unicode(text) + text = str_to_unicode(text, encoding) if which_ones: - tags = '|'.join(['<%s.*?' % (tag,tag) for tag in which_ones]) + tags = '|'.join(['<%s.*?' % (tag, tag) for tag in which_ones]) retags = re.compile(tags, re.DOTALL | re.IGNORECASE) text = retags.sub(u'', text) return text -def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): +def replace_escape_chars(text, which_ones=('\n', '\t', '\r'), replace_by=u'', \ + encoding=None): """ Remove escape chars. Default : \\n, \\t, \\r which_ones -- is a tuple of which escape chars we want to remove. @@ -117,10 +118,10 @@ def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): It defaults to '', so the escape chars are removed. """ for ec in which_ones: - text = text.replace(ec, str_to_unicode(replace_by)) - return str_to_unicode(text) + text = text.replace(ec, str_to_unicode(replace_by, encoding)) + return str_to_unicode(text, encoding) -def unquote_markup(text, keep=(), remove_illegal=True): +def unquote_markup(text, keep=(), remove_illegal=True, encoding=None): """ This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following: - removes entities (except the ones in 'keep') from any part of it that it's not inside a CDATA @@ -138,7 +139,7 @@ def unquote_markup(text, keep=(), remove_illegal=True): offset = match_e yield txt[offset:] - text = str_to_unicode(text) + text = str_to_unicode(text, encoding) ret_text = u'' for fragment in _get_fragments(text, _cdata_re): if isinstance(fragment, basestring): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 319acdc05..99aefcd74 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -64,13 +64,15 @@ def unique(list_, key=lambda x: x): return result -def str_to_unicode(text, encoding='utf-8'): +def str_to_unicode(text, encoding=None): """Return the unicode representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a unicode object without the risk of double-decoding problems (which can happen if you don't use the default 'ascii' encoding) """ + if encoding is None: + encoding = 'utf-8' if isinstance(text, str): return text.decode(encoding) elif isinstance(text, unicode): @@ -78,13 +80,15 @@ def str_to_unicode(text, encoding='utf-8'): else: raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__) -def unicode_to_str(text, encoding='utf-8'): +def unicode_to_str(text, encoding=None): """Return the str representation of text in the given encoding. Unlike .encode(encoding) this function can be applied directly to a str object without the risk of double-decoding problems (which can happen if you don't use the default 'ascii' encoding) """ + if encoding is None: + encoding = 'utf-8' if isinstance(text, unicode): return text.encode(encoding) elif isinstance(text, str): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1c2fe18ae..7496328fc 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -130,7 +130,8 @@ def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False) name+'='+new_value) return next_url -def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): +def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ + encoding=None): """Canonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value @@ -147,7 +148,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): For examples see the tests in scrapy.tests.test_utils_url """ - url = unicode_to_str(url) + url = unicode_to_str(url, encoding) scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) keyvals = cgi.parse_qsl(query, keep_blank_values) keyvals.sort() From cb49567ca671647882408763f309cb3f37b9734e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 24 Mar 2010 12:15:18 -0300 Subject: [PATCH 23/68] Removed wrong line added in previous commit --- scrapy/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 4a62ea9c3..6f8ca15e6 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -18,7 +18,6 @@ from scrapy.xlib import twisted_250_monkeypatches from scrapy.utils.encoding import add_encoding_alias add_encoding_alias('gb2312', 'zh-cn') add_encoding_alias('cp1251', 'win-1251') -add_encoding_alias('cp1252', 'iso8859-1', overwrite=True) # optional_features is a set containing Scrapy optional features optional_features = set() From 9ddcd1095dd9f305ff24edf10fa47dfbc3d7a74a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 25 Mar 2010 11:45:06 -0300 Subject: [PATCH 24/68] sort setting alphabetically --- docs/topics/settings.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ffb2921e7..d2a409abb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -340,16 +340,6 @@ Default: ``True`` Whether to collect depth stats. -.. setting:: DOMAIN_SCHEDULER - -SPIDER_SCHEDULER ----------------- - -Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` - -The Spider Scheduler to use. The spider scheduler returns the next spider to -scrape. - .. setting:: DOWNLOADER_DEBUG DOWNLOADER_DEBUG @@ -897,6 +887,16 @@ Example:: SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev'] +.. setting:: SPIDER_SCHEDULER + +SPIDER_SCHEDULER +---------------- + +Default: ``'scrapy.contrib.spiderscheduler.FifoSpiderScheduler'`` + +The Spider Scheduler to use. The spider scheduler returns the next spider to +scrape. + .. setting:: STATS_CLASS STATS_CLASS From 173e94386b6afe715cefeb9724da0f1e8c152770 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 25 Mar 2010 12:38:37 -0300 Subject: [PATCH 25/68] Support relative url used in base tag. closes #148 --HG-- extra : rebase_source : 1bff87c127a7e9d8d12c772b3068feb11eb5d97f --- scrapy/contrib/linkextractors/htmlparser.py | 2 +- scrapy/contrib/linkextractors/image.py | 3 +- scrapy/contrib/linkextractors/lxmlparser.py | 2 +- scrapy/contrib/linkextractors/regex.py | 3 +- scrapy/contrib/linkextractors/sgml.py | 2 +- scrapy/contrib_exp/crawlspider/reqext.py | 2 +- .../test_contrib_exp_crawlspider_reqext.py | 37 ++++++++++++++----- scrapy/tests/test_contrib_linkextractors.py | 14 +++++++ 8 files changed, 49 insertions(+), 16 deletions(-) diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index 2714fb562..fb3fd661b 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -26,7 +26,7 @@ class HtmlParserLinkExtractor(HTMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 79048a496..88dd78577 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -51,8 +51,7 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) base_url = xs.select('//base/@href').extract() - base_url = unicode_to_str(base_url[0], response.encoding) if base_url \ - else unicode_to_str(response.url, response.encoding) + base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url links = [] for location in self.locations: diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py index 390e3a304..27cd0697a 100644 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ b/scrapy/contrib/linkextractors/lxmlparser.py @@ -29,7 +29,7 @@ class LxmlLinkExtractor(object): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 08e1e4526..1de044df3 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -16,8 +16,9 @@ def clean_link(link_text): class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" + def _extract_links(self, response_text, response_url, response_encoding): - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip() diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index d548626ab..9ec664bda 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -28,7 +28,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py index bb7318f79..e23e78082 100644 --- a/scrapy/contrib_exp/crawlspider/reqext.py +++ b/scrapy/contrib_exp/crawlspider/reqext.py @@ -30,7 +30,7 @@ class BaseSgmlRequestExtractor(FixedSGMLParser): self.feed(response_text) self.close() - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url self._make_absolute_urls(base_url, response_encoding) self._fix_link_text_encoding(response_encoding) diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py index d0d0d1b5f..0259b30fb 100644 --- a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py @@ -50,20 +50,39 @@ class RequestExtractorTest(AbstractRequestExtractorTest): ) def test_base_url(self): + reqx = BaseSgmlRequestExtractor() + html = """Page title<title> <base href="http://otherdomain.com/base/" /> <body><p><a href="item/12.html">Item 12</a></p> </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", - body=html) - reqx = BaseSgmlRequestExtractor() + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('http://otherdomain.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) - self.failUnless( - self._requests_equals(reqx.extract_requests(response), - [ Request('http://otherdomain.com/base/item/12.html', - meta={'link_text': 'Item 12'}) ] - ) - ) + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title> + <base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://example.org/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + # base url has no scheme + html = """<html><head><title>Page title<title> + <base href="//noscheme.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://noscheme.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) def test_extraction_encoding(self): #TODO: use own fixtures diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 4a60b8a2d..65ffad714 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -35,6 +35,20 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title><base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://example.org/item/12.html', text='Item 12')]) + + # base url has no scheme + html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + def test_extraction_encoding(self): body = get_testdata('link_extractor', 'linkextractor_noenc.html') response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) From 1330697c3dc9cc64189b0da9506767bbbe6d70f4 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman <pablo@pablohoffman.com> Date: Thu, 25 Mar 2010 15:47:10 -0300 Subject: [PATCH 26/68] Some improvements to Response encoding support: * added encoding aliases, configurable through a new ENCODING_ALIASES setting * Response.encoding now returns the real encoding detected for the body * simplified TextResponse API by removing body_encoding() and headers_encoding() methods * Response.encoding now tries to infer the encoding from the body always (it was done before only on HtmlResponse and TextResponse) * removed scrapy.utils.encoding.add_encoding_alias() function * updated implementation of scrapy.utils.response function to reflect these API changes * updated documentation to reflect API changes --- docs/topics/request-response.rst | 14 +---- docs/topics/settings.rst | 46 ++++++++++++++++ scrapy/__init__.py | 5 -- scrapy/conf/default_settings.py | 17 ++++++ .../contrib/downloadermiddleware/redirect.py | 10 ++-- scrapy/http/response/html.py | 3 - scrapy/http/response/text.py | 55 ++++++++++--------- scrapy/http/response/xml.py | 3 - .../test_downloadermiddleware_redirect.py | 8 +-- scrapy/tests/test_encoding_aliases.py | 21 ------- scrapy/tests/test_http_response.py | 34 +++++++----- scrapy/tests/test_utils_encoding.py | 25 +++++++++ scrapy/tests/test_utils_response.py | 23 +++----- scrapy/utils/encoding.py | 23 +++++--- scrapy/utils/response.py | 14 ++--- 15 files changed, 179 insertions(+), 122 deletions(-) delete mode 100644 scrapy/tests/test_encoding_aliases.py create mode 100644 scrapy/tests/test_utils_encoding.py diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9f4dcdc46..63d289d09 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -485,23 +485,11 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: - .. method:: TextResponse.headers_encoding() - - Returns a string with the encoding declared in the headers (ie. the - Content-Type HTTP header). - - .. method:: TextResponse.body_encoding() - - Returns a string with the encoding of the body, either declared or inferred - from its contents. The body encoding declaration is implemented in - :class:`TextResponse` subclasses such as: :class:`HtmlResponse` or - :class:`XmlResponse`. - .. method:: TextResponse.body_as_unicode() Returns the body of the response as unicode. This is equivalent to:: - response.body.encode(response.encoding) + response.body.decode(response.encoding) But **not** equivalent to:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d2a409abb..6e5949eb5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -438,6 +438,52 @@ The class used to detect and filter duplicate requests. The default (``RequestFingerprintDupeFilter``) filters based on request fingerprint (using ``scrapy.utils.request.request_fingerprint``) and grouping per domain. +.. setting:: ENCODING_ALIASES + +ENCODING_ALIASES +---------------- + +Default: ``{}`` + +A mapping of custom encoding aliases for your project, where the keys are the +aliases (and must be lower case) and the values are the encodings they map to. + +This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which +contains some default mappings. + +.. setting:: ENCODING_ALIASES_BASE + +ENCODING_ALIASES_BASE +--------------------- + +Default:: + + { + 'gb2312': 'zh-cn', + 'cp1251': 'win-1251', + 'macintosh' : 'mac-roman', + 'x-sjis': 'shift-jis', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'latin_1': 'cp1252', + 'l1': 'cp1252', + } + +The default encoding aliases defined in Scrapy. Don't override this setting in +your project, override :setting:`ENCODING_ALIASES` instead. + +The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is +due to a well known browser hack. For more information see: `Character +encodings in HTML`_. + +.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 +.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 +.. _Character encodings in HTML: http://www.gnu.org/software/wget/manual/wget.html + .. setting:: EXTENSIONS EXTENSIONS diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 6f8ca15e6..fffaed1dc 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -14,11 +14,6 @@ if sys.version_info < (2,5): # monkey patches to fix external library issues from scrapy.xlib import twisted_250_monkeypatches -# add some common encoding aliases not included by default in Python -from scrapy.utils.encoding import add_encoding_alias -add_encoding_alias('gb2312', 'zh-cn') -add_encoding_alias('cp1251', 'win-1251') - # optional_features is a set containing Scrapy optional features optional_features = set() diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 5550e75ba..5e7bea3cd 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -71,6 +71,23 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' +ENCODING_ALIASES = {} + +ENCODING_ALIASES_BASE = { + 'zh-cn': 'gb2312', + 'win-1251': 'cp1251', + 'macintosh' : 'mac-roman', + 'x-sjis': 'shift-jis', + 'iso-8859-1': 'cp1252', + 'iso8859-1': 'cp1252', + '8859': 'cp1252', + 'cp819': 'cp1252', + 'latin': 'cp1252', + 'latin1': 'cp1252', + 'latin_1': 'cp1252', + 'l1': 'cp1252', +} + EXTENSIONS = {} EXTENSIONS_BASE = { diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 249862824..1c6297b49 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,5 @@ from scrapy import log +from scrapy.http import HtmlResponse from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh from scrapy.core.exceptions import IgnoreRequest @@ -24,10 +25,11 @@ class RedirectMiddleware(object): redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) - interval, url = get_meta_refresh(response) - if url and interval < self.max_metarefresh_delay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + if isinstance(response, HtmlResponse): + interval, url = get_meta_refresh(response) + if url and interval < self.max_metarefresh_delay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index f1557e6f7..dc812ac0e 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -23,9 +23,6 @@ class HtmlResponse(TextResponse): METATAG_RE = re.compile(r'<meta\s+%s\s+%s' % (_httpequiv_re, _content_re), re.I) METATAG_RE2 = re.compile(r'<meta\s+%s\s+%s' % (_content_re, _httpequiv_re), re.I) - def body_encoding(self): - return self._body_declared_encoding() or super(HtmlResponse, self).body_encoding() - @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index d42a89197..23bb21e73 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,13 +5,13 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ -import codecs import re from scrapy.xlib.BeautifulSoup import UnicodeDammit from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs +from scrapy.utils.encoding import encoding_exists, resolve_encoding from scrapy.conf import settings class TextResponse(Response): @@ -19,12 +19,12 @@ class TextResponse(Response): _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING'] _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - __slots__ = ['_encoding', '_body_inferred_encoding'] + __slots__ = ['_encoding', '_cached_benc'] def __init__(self, url, status=200, headers=None, body=None, meta=None, \ flags=None, encoding=None): self._encoding = encoding - self._body_inferred_encoding = None + self._cached_benc = None super(TextResponse, self).__init__(url, status, headers, body, meta, flags) def _get_url(self): @@ -57,36 +57,39 @@ class TextResponse(Response): @property def encoding(self): - return self._encoding or self.headers_encoding() or self.body_encoding() + enc = self._declared_encoding() + if not (enc and encoding_exists(enc)): + enc = self._body_inferred_encoding() or self._DEFAULT_ENCODING + return resolve_encoding(enc) - @memoizemethod_noargs - def headers_encoding(self): - content_type = self.headers.get('Content-Type') - if content_type: - encoding = self._ENCODING_RE.search(content_type) - if encoding: - enc = encoding.group(1) - try: - codecs.lookup(enc) # check if the encoding is valid - return enc - except LookupError: - pass + def _declared_encoding(self): + return self._encoding or self._headers_encoding() \ + or self._body_declared_encoding() @memoizemethod_noargs def body_as_unicode(self): """Return body as unicode""" - possible_encodings = (self._encoding, self.headers_encoding(), \ - self._body_declared_encoding()) - dammit = UnicodeDammit(self.body, possible_encodings) - self._body_inferred_encoding = dammit.originalEncoding - if self._body_inferred_encoding in ('ascii', None): - self._body_inferred_encoding = self._DEFAULT_ENCODING - return dammit.unicode + denc = self._declared_encoding() + dencs = [resolve_encoding(denc)] if denc else [] + dammit = UnicodeDammit(self.body, dencs) + benc = dammit.originalEncoding + self._cached_benc = benc if benc != 'ascii' else None + return self.body.decode(benc) if benc == 'utf-16' else dammit.unicode - def body_encoding(self): - if self._body_inferred_encoding is None: + @memoizemethod_noargs + def _headers_encoding(self): + content_type = self.headers.get('Content-Type') + if content_type: + m = self._ENCODING_RE.search(content_type) + if m: + encoding = m.group(1) + if encoding_exists(encoding): + return encoding + + def _body_inferred_encoding(self): + if self._cached_benc is None: self.body_as_unicode() - return self._body_inferred_encoding + return self._cached_benc def _body_declared_encoding(self): # implemented in subclasses (XmlResponse, HtmlResponse) diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 95f57cd39..90a06f34c 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -18,9 +18,6 @@ class XmlResponse(TextResponse): _encoding_re = _template % ('encoding', r'(?P<charset>[\w-]+)') XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) - def body_encoding(self): - return self._body_declared_encoding() or super(XmlResponse, self).body_encoding() - @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index ba7daece8..f9a93b910 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -3,7 +3,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import BaseSpider from scrapy.core.exceptions import IgnoreRequest -from scrapy.http import Request, Response, Headers +from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): @@ -58,7 +58,7 @@ class RedirectMiddlewareTest(unittest.TestCase): <head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head> </html>""" req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -70,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase): <head><meta http-equiv="refresh" content="1000;url=http://example.org/newpage" /></head> </html>""" req = Request(url='http://example.org') - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) rsp2 = self.mw.process_response(req, rsp, self.spider) assert rsp is rsp2 @@ -81,7 +81,7 @@ class RedirectMiddlewareTest(unittest.TestCase): </html>""" req = Request(url='http://example.org', method='POST', body='test', headers={'Content-Type': 'text/plain', 'Content-length': '4'}) - rsp = Response(url='http://example.org', body=body) + rsp = HtmlResponse(url='http://example.org', body=body) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) diff --git a/scrapy/tests/test_encoding_aliases.py b/scrapy/tests/test_encoding_aliases.py deleted file mode 100644 index cc3a0089f..000000000 --- a/scrapy/tests/test_encoding_aliases.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest - -import scrapy # adds encoding aliases (if not added before) - -class EncodingAliasesTestCase(unittest.TestCase): - - def test_encoding_aliases(self): - """Test common encdoing aliases not included in Python""" - - uni = u'\u041c\u041e\u0421K\u0412\u0410' - str = uni.encode('windows-1251') - self.assertEqual(uni.encode('windows-1251'), uni.encode('win-1251')) - self.assertEqual(str.decode('windows-1251'), str.decode('win-1251')) - - text = u'\u8f6f\u4ef6\u540d\u79f0' - str = uni.encode('gb2312') - self.assertEqual(uni.encode('gb2312'), uni.encode('zh-cn')) - self.assertEqual(str.decode('gb2312'), str.decode('zh-cn')) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 5851572ac..081d4c864 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -2,6 +2,7 @@ import unittest import weakref from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers +from scrapy.utils.encoding import resolve_encoding from scrapy.conf import settings @@ -112,10 +113,13 @@ class BaseResponseTest(unittest.TestCase): body_str = body assert isinstance(response.body, str) - self.assertEqual(response.encoding, encoding) + self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_str) self.assertEqual(response.body_as_unicode(), body_unicode) + def _assert_response_encoding(self, response, encoding): + self.assertEqual(response.encoding, resolve_encoding(encoding)) + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): @@ -134,14 +138,14 @@ class TextResponseTest(BaseResponseTest): assert isinstance(r2, self.response_class) self.assertEqual(r2.url, "http://www.example.com/other") - self.assertEqual(r2.encoding, "cp852") + self._assert_response_encoding(r2, "cp852") self.assertEqual(r3.url, "http://www.example.com/other") - self.assertEqual(r3.encoding, "latin1") + self.assertEqual(r3._declared_encoding(), "latin1") def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) resp = self.response_class(u"http://www.example.com/") - self.assertEqual(resp.encoding, settings['DEFAULT_RESPONSE_ENCODING']) + self._assert_response_encoding(resp, settings['DEFAULT_RESPONSE_ENCODING']) # make sure urls are converted to str resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') @@ -175,18 +179,18 @@ class TextResponseTest(BaseResponseTest): r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3") r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") - r5 = self.response_class("http://www.example.com", - headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") + r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") - self.assertEqual(r1.headers_encoding(), "utf-8") - self.assertEqual(r2.headers_encoding(), None) - self.assertEqual(r2.encoding, 'utf-8') - self.assertEqual(r3.headers_encoding(), "iso-8859-1") - self.assertEqual(r3.encoding, 'iso-8859-1') - self.assertEqual(r4.headers_encoding(), None) - self.assertEqual(r5.headers_encoding(), None) - self.assertEqual(r5.encoding, "utf-8") - assert r4.body_encoding() is not None and r4.body_encoding() != 'ascii' + self.assertEqual(r1._headers_encoding(), "utf-8") + self.assertEqual(r2._headers_encoding(), None) + self.assertEqual(r2._declared_encoding(), 'utf-8') + self._assert_response_encoding(r2, 'utf-8') + self.assertEqual(r3._headers_encoding(), "iso-8859-1") + self.assertEqual(r3._declared_encoding(), "iso-8859-1") + self.assertEqual(r4._headers_encoding(), None) + self.assertEqual(r5._headers_encoding(), None) + self._assert_response_encoding(r5, "utf-8") + assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii' self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3") self._assert_response_values(r3, 'iso-8859-1', u"\xa3") diff --git a/scrapy/tests/test_utils_encoding.py b/scrapy/tests/test_utils_encoding.py new file mode 100644 index 000000000..b6800cd75 --- /dev/null +++ b/scrapy/tests/test_utils_encoding.py @@ -0,0 +1,25 @@ +import unittest + +from scrapy.utils.encoding import encoding_exists, resolve_encoding + +class UtilsEncodingTestCase(unittest.TestCase): + + _ENCODING_ALIASES = { + 'foo': 'cp1252', + 'bar': 'none', + } + + def test_resolve_encoding(self): + self.assertEqual(resolve_encoding('latin1', self._ENCODING_ALIASES), + 'latin1') + self.assertEqual(resolve_encoding('foo', self._ENCODING_ALIASES), + 'cp1252') + + def test_encoding_exists(self): + assert encoding_exists('latin1', self._ENCODING_ALIASES) + assert encoding_exists('foo', self._ENCODING_ALIASES) + assert not encoding_exists('bar', self._ENCODING_ALIASES) + assert not encoding_exists('none', self._ENCODING_ALIASES) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index de506b639..a317cd732 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -29,7 +29,7 @@ class ResponseUtilsTest(unittest.TestCase): self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode)) def test_get_base_url(self): - response = Response(url='http://example.org', body="""\ + response = HtmlResponse(url='http://example.org', body="""\ <html>\ <head><title>Dummy\ blahablsdfsal&\ @@ -42,17 +42,17 @@ class ResponseUtilsTest(unittest.TestCase): Dummy blahablsdfsal& """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # refresh without url should return (None, None) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (None, None)) body = """""" - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # meta refresh in multiple lines @@ -60,17 +60,17 @@ class ResponseUtilsTest(unittest.TestCase): """ - response = Response(url='http://example.org', body=body) + response = TextResponse(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage')) # entities in the redirect url body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other')) # relative redirects body = """""" - response = Response(url='http://example.com/page/this.html', body=body) + response = TextResponse(url='http://example.com/page/this.html', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html')) # non-standard encodings (utf-16) @@ -81,7 +81,7 @@ class ResponseUtilsTest(unittest.TestCase): # non-ascii chars in the url (default encoding - utf8) body = """""" - response = Response(url='http://example.com', body=body) + response = TextResponse(url='http://example.com', body=body) self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) # non-ascii chars in the url (custom encoding - latin1) @@ -89,13 +89,8 @@ class ResponseUtilsTest(unittest.TestCase): response = TextResponse(url='http://example.com', body=body, encoding='latin1') self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) - # wrong encodings (possibly caused by truncated chunks) - body = """""" - response = Response(url='http://example.com', body=body) - self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT')) - # responses without refresh tag should return None None - response = Response(url='http://example.org') + response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) diff --git a/scrapy/utils/encoding.py b/scrapy/utils/encoding.py index 9eb06d942..c7d645041 100644 --- a/scrapy/utils/encoding.py +++ b/scrapy/utils/encoding.py @@ -1,11 +1,20 @@ import codecs -def add_encoding_alias(encoding, alias, overwrite=False): +from scrapy.conf import settings + +_ENCODING_ALIASES = dict(settings['ENCODING_ALIASES_BASE']) +_ENCODING_ALIASES.update(settings['ENCODING_ALIASES']) + +def encoding_exists(encoding, _aliases=_ENCODING_ALIASES): + """Returns ``True`` if encoding is valid, otherwise returns ``False``""" try: - codecs.lookup(alias) - alias_exists = True + codecs.lookup(resolve_encoding(encoding, _aliases)) except LookupError: - alias_exists = False - if overwrite or not alias_exists: - codec = codecs.lookup(encoding) - codecs.register(lambda x: codec if x == alias else None) + return False + return True + +def resolve_encoding(alias, _aliases=_ENCODING_ALIASES): + """Return the encoding the given alias maps to, or the alias as passed if + no mapping is found. + """ + return _aliases.get(alias.lower(), alias) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 56c26c2f0..15b2afbab 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -18,7 +18,8 @@ from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse def body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ + assert isinstance(obj, (Response, basestring)), \ + "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): return obj.body_as_unicode() if unicode else obj.body elif isinstance(obj, str): @@ -26,16 +27,17 @@ def body_or_str(obj, unicode=True): else: return obj if unicode else obj.encode('utf-8') -BASEURL_RE = re.compile(r']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', re.DOTALL | re.IGNORECASE) +META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', \ + re.DOTALL | re.IGNORECASE) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv parameter of the HTML meta element from the given @@ -46,9 +48,7 @@ def get_meta_refresh(response): If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - encoding = getattr(response, 'encoding', None) or 'utf-8' - body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \ - errors='ignore')) + body_chunk = remove_entities(response.body_as_unicode()[0:4096]) match = META_REFRESH_RE.search(body_chunk) if match: interval = int(match.group('int')) From 996a1b3574cde64823408a42ae27e5f27186ac26 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 25 Mar 2010 15:50:34 -0300 Subject: [PATCH 27/68] fix handling of relative base urls in get_base_url util --HG-- extra : rebase_source : eb552219e6bf40bc0d2e35968c367105233b6ecc --- scrapy/tests/test_utils_response.py | 18 +++++++++++++++++- scrapy/utils/response.py | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index a317cd732..71c14569e 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -29,13 +29,29 @@ class ResponseUtilsTest(unittest.TestCase): self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode)) def test_get_base_url(self): - response = HtmlResponse(url='http://example.org', body="""\ + response = HtmlResponse(url='https://example.org', body="""\ \ Dummy\ blahablsdfsal&\ """) self.assertEqual(get_base_url(response), 'http://example.org/something') + # relative url with absolute path + response = HtmlResponse(url='https://example.org', body="""\ + \ + Dummy\ + blahablsdfsal&\ + """) + self.assertEqual(get_base_url(response), 'https://example.org/absolutepath') + + # no scheme url + response = HtmlResponse(url='https://example.org', body="""\ + \ + Dummy\ + blahablsdfsal&\ + """) + self.assertEqual(get_base_url(response), 'https://noscheme.com/path') + def test_get_meta_refresh(self): body = """ diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 15b2afbab..ca258ef44 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -33,7 +33,7 @@ def get_base_url(response): """ Return the base url of the given response used to resolve relative links. """ if response not in _baseurl_cache: match = BASEURL_RE.search(response.body_as_unicode()[0:4096]) - _baseurl_cache[response] = match.group(1) if match else response.url + _baseurl_cache[response] = urljoin_rfc(response.url, match.group(1)) if match else response.url return _baseurl_cache[response] META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', \ From 2299deda6657d398a11bd394998f52791603b597 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 26 Mar 2010 14:02:33 -0300 Subject: [PATCH 28/68] updated wrong link in doc --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6e5949eb5..eacc2107c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -482,7 +482,7 @@ encodings in HTML`_. .. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 .. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 -.. _Character encodings in HTML: http://www.gnu.org/software/wget/manual/wget.html +.. _Character encodings in HTML: http://en.wikipedia.org/wiki/Character_encodings_in_HTML .. setting:: EXTENSIONS From de896fa62decef1010db863390895ad372017035 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 31 Mar 2010 16:29:53 -0300 Subject: [PATCH 29/68] Refactored implementation of Request.replace() and Response.replace() --- scrapy/http/request/__init__.py | 20 ++++++-------------- scrapy/http/response/__init__.py | 17 +++++------------ 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 187238b1c..4dce92880 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -96,20 +96,12 @@ class Request(object_ref): """Return a copy of this Request""" return self.replace() - def replace(self, url=None, callback=None, method=None, headers=None, body=None, \ - cookies=None, meta=None, encoding=None, priority=None, \ - dont_filter=None, errback=None): + def replace(self, *args, **kwargs): """Create a new Request with the same attributes except for those given new values. """ - return self.__class__(url=self.url if url is None else url, - callback=callback, - method=self.method if method is None else method, - headers=copy.deepcopy(self.headers) if headers is None else headers, - body=self.body if body is None else body, - cookies=self.cookies if cookies is None else cookies, - meta=self.meta if meta is None else meta, - encoding=self.encoding if encoding is None else encoding, - priority=self.priority if priority is None else priority, - dont_filter=self.dont_filter if dont_filter is None else dont_filter, - errback=errback) + for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', \ + 'encoding', 'priority', 'dont_filter']: + kwargs.setdefault(x, getattr(self, x)) + cls = kwargs.pop('cls', self.__class__) + return cls(*args, **kwargs) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f632ea117..251e77ad7 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -71,18 +71,11 @@ class Response(object_ref): """Return a copy of this Response""" return self.replace() - def replace(self, url=None, status=None, headers=None, body=None, meta=None, \ - flags=None, cls=None, **kwargs): + def replace(self, *args, **kwargs): """Create a new Response with the same attributes except for those given new values. """ - if cls is None: - cls = self.__class__ - new = cls(url=self.url if url is None else url, - status=self.status if status is None else status, - headers=copy.deepcopy(self.headers) if headers is None else headers, - body=self.body if body is None else body, - meta=self.meta if meta is None else meta, - flags=self.flags if flags is None else flags, - **kwargs) - return new + for x in ['url', 'status', 'headers', 'body', 'meta', 'flags']: + kwargs.setdefault(x, getattr(self, x)) + cls = kwargs.pop('cls', self.__class__) + return cls(*args, **kwargs) From 83d5eff0b7bffda97571c763920abc8f6f3a9798 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 31 Mar 2010 18:21:41 -0300 Subject: [PATCH 30/68] More refactoring to encoding handling in TextResponse and subclasses --- scrapy/http/response/text.py | 32 ++++++++++++++++++++---------- scrapy/tests/test_http_request.py | 7 ------- scrapy/tests/test_http_response.py | 31 ++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 23bb21e73..c294ade63 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -19,12 +19,13 @@ class TextResponse(Response): _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING'] _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - __slots__ = ['_encoding', '_cached_benc'] + __slots__ = ['_encoding', '_cached_benc', '_cached_ubody'] def __init__(self, url, status=200, headers=None, body=None, meta=None, \ flags=None, encoding=None): self._encoding = encoding self._cached_benc = None + self._cached_ubody = None super(TextResponse, self).__init__(url, status, headers, body, meta, flags) def _get_url(self): @@ -57,24 +58,27 @@ class TextResponse(Response): @property def encoding(self): + return self._get_encoding(infer=True) + + def _get_encoding(self, infer=False): enc = self._declared_encoding() - if not (enc and encoding_exists(enc)): - enc = self._body_inferred_encoding() or self._DEFAULT_ENCODING + if enc and not encoding_exists(enc): + enc = None + if not enc and infer: + enc = self._body_inferred_encoding() + if not enc: + enc = self._DEFAULT_ENCODING return resolve_encoding(enc) def _declared_encoding(self): return self._encoding or self._headers_encoding() \ or self._body_declared_encoding() - @memoizemethod_noargs def body_as_unicode(self): """Return body as unicode""" - denc = self._declared_encoding() - dencs = [resolve_encoding(denc)] if denc else [] - dammit = UnicodeDammit(self.body, dencs) - benc = dammit.originalEncoding - self._cached_benc = benc if benc != 'ascii' else None - return self.body.decode(benc) if benc == 'utf-16' else dammit.unicode + if self._cached_ubody is None: + self._cached_ubody = self.body.decode(self.encoding, 'replace') + return self._cached_ubody @memoizemethod_noargs def _headers_encoding(self): @@ -88,7 +92,13 @@ class TextResponse(Response): def _body_inferred_encoding(self): if self._cached_benc is None: - self.body_as_unicode() + enc = self._get_encoding() + dammit = UnicodeDammit(self.body, [enc]) + benc = dammit.originalEncoding + self._cached_benc = benc + # UnicodeDammit is buggy decoding utf-16 + if self._cached_ubody is None and benc != 'utf-16': + self._cached_ubody = dammit.unicode return self._cached_benc def _body_declared_encoding(self): diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index c0ae0cab5..b37eb33e7 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -171,13 +171,6 @@ class RequestTest(unittest.TestCase): self.assertEqual(r4.meta, {}) assert r4.dont_filter is False - # __init__ and replace() signatures must be equal unles *args,**kwargs is used - i_args, i_varargs, i_varkwargs, _ = getargspec(self.request_class.__init__) - self.assertFalse(bool(i_varargs) ^ bool(i_varkwargs)) - if not i_varargs: - r_args, _, _, _ = getargspec(self.request_class.replace) - self.assertEqual(i_args, r_args) - def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" x = self.request_class('http://www.example.com') diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 081d4c864..842fc3790 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -3,7 +3,6 @@ import weakref from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers from scrapy.utils.encoding import resolve_encoding -from scrapy.conf import settings class BaseResponseTest(unittest.TestCase): @@ -145,7 +144,7 @@ class TextResponseTest(BaseResponseTest): def test_unicode_url(self): # instantiate with unicode url without encoding (should set default encoding) resp = self.response_class(u"http://www.example.com/") - self._assert_response_encoding(resp, settings['DEFAULT_RESPONSE_ENCODING']) + self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING) # make sure urls are converted to str resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') @@ -198,6 +197,32 @@ class TextResponseTest(BaseResponseTest): # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") + def test_declared_encoding_invalid(self): + """Check that unknown declared encodings are ignored""" + r = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=UKNOWN"]}, body="\xc2\xa3") + self.assertEqual(r._declared_encoding(), None) + self._assert_response_values(r, 'utf-8', u"\xa3") + + def test_utf16(self): + """Test utf-16 because UnicodeDammit is known to have problems with""" + r = self.response_class("http://www.example.com", body='\xff\xfeh\x00i\x00', encoding='utf-16') + self._assert_response_values(r, 'utf-16', u"hi") + + def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): + r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xef\xbb\xbfWORD\xe3\xab") + self.assertEqual(r6.encoding, 'utf-8') + self.assertEqual(r6.body_as_unicode(), u'\ufeffWORD\ufffd') + + def test_replace_wrong_encoding(self): + """Test invalid chars are replaced properly""" + # XXX: Policy for replacing invalid chars may change without prior notice + r = self.response_class("http://www.example.com", encoding='utf-8', body='PREFIX\xe3\xabSUFFIX') + assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse + #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') + #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse @@ -239,7 +264,7 @@ class XmlResponseTest(TextResponseTest): body = "" r1 = self.response_class("http://www.example.com", body=body) - self._assert_response_values(r1, settings['DEFAULT_RESPONSE_ENCODING'], body) + self._assert_response_values(r1, self.response_class._DEFAULT_ENCODING, body) body = """""" r2 = self.response_class("http://www.example.com", body=body) From 4dc886e3192095ccdbd6ec11c81dc676757308f6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 31 Mar 2010 18:26:35 -0300 Subject: [PATCH 31/68] Improved comment --- scrapy/tests/test_http_response.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 842fc3790..d17d5d272 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -215,9 +215,11 @@ class TextResponseTest(BaseResponseTest): def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" - # XXX: Policy for replacing invalid chars may change without prior notice r = self.response_class("http://www.example.com", encoding='utf-8', body='PREFIX\xe3\xabSUFFIX') + # XXX: Policy for replacing invalid chars may suffer minor variations + # but it should always contain the unicode replacement char (u'\ufffd') assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) From 32f9c5fe68800376e730ad1cecd5ea4537d573c8 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 1 Apr 2010 04:05:53 -0300 Subject: [PATCH 32/68] removed old untested (and probably broken) code --- extras/sql/scraping.sql | 72 --------------------------------- scrapy/stats/collector/mysql.py | 31 -------------- 2 files changed, 103 deletions(-) delete mode 100644 extras/sql/scraping.sql delete mode 100644 scrapy/stats/collector/mysql.py diff --git a/extras/sql/scraping.sql b/extras/sql/scraping.sql deleted file mode 100644 index c3314f5a1..000000000 --- a/extras/sql/scraping.sql +++ /dev/null @@ -1,72 +0,0 @@ -DROP TABLE IF EXISTS `url_history`; -DROP TABLE IF EXISTS `version`; -DROP TABLE IF EXISTS `url_status`; -DROP TABLE IF EXISTS `ticket`; -DROP TABLE IF EXISTS `domain_stats`; -DROP TABLE IF EXISTS `domain_stats_history`; -DROP TABLE IF EXISTS `domain_data_history`; - -CREATE TABLE `ticket` ( - `guid` char(40) NOT NULL, - `domain` varchar(255) default NULL, - `url` varchar(2048) default NULL, - `url_hash` char(40) default NULL, -- so we can join to url_status - PRIMARY KEY (`guid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `version` ( - `id` bigint(20) NOT NULL auto_increment, - `guid` char(40) NOT NULL, - `version` char(40) NOT NULL, - `seen` datetime NOT NULL, - PRIMARY KEY (`id`), - FOREIGN KEY (`guid`) REFERENCES ticket(guid) ON UPDATE CASCADE ON DELETE CASCADE, - UNIQUE KEY (`version`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_status` ( - -- see http://support.microsoft.com/kb/q208427/ for explanation of 2048 - `url_hash` char(40) NOT NULL, -- for faster searches - `url` varchar(2048) NOT NULL, - `parent_hash` char(40) default NULL, -- the url that was followed to this one - for reporting - `last_version` char(40) default NULL, -- can be null if it generated an error the last time is was checked - `last_checked` datetime NOT NULL, - PRIMARY KEY (`url_hash`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `url_history` ( - `url_hash` char(40) NOT NULL, - `version` char(40) NOT NULL, - `postdata_hash` char(40) default NULL, - `created` datetime NOT NULL, - PRIMARY KEY (`version`), - FOREIGN KEY (`url_hash`) REFERENCES url_status(url_hash) ON UPDATE CASCADE ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats` ( - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` text, - PRIMARY KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_stats_history` ( - `id` bigint(20) NOT NULL auto_increment, - `key1` varchar(128) NOT NULL, - `key2` varchar(128) NOT NULL, - `value` varchar(2048) NOT NULL, - `stored` datetime NOT NULL, - PRIMARY KEY (`id`), - KEY `key1_key2` (`key1`, `key2`), - KEY `key1` (`key1`), - KEY `stored` (`stored`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -CREATE TABLE `domain_data_history` ( - `domain` varchar(255) NOT NULL, - `stored` datetime NOT NULL, - `data` text, - KEY `domain_stored` (`domain`, `stored`), - KEY `domain` (`domain`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/scrapy/stats/collector/mysql.py b/scrapy/stats/collector/mysql.py deleted file mode 100644 index 2cf145347..000000000 --- a/scrapy/stats/collector/mysql.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -A Stats collector for persisting stats (pickled) to a MySQL db -""" - -import cPickle as pickle -from datetime import datetime - -from scrapy.stats.collector import StatsCollector -from scrapy.utils.mysql import mysql_connect -from scrapy.conf import settings - -class MysqlStatsCollector(StatsCollector): - - def __init__(self): - super(MysqlStatsCollector, self).__init__() - mysqluri = settings['STATS_MYSQL_URI'] - self._mysql_conn = mysql_connect(mysqluri, use_unicode=False) if mysqluri else None - - def _persist_stats(self, stats, spider=None): - if spider is None: # only store spider-specific stats - return - if self._mysql_conn is None: - return - stored = datetime.utcnow() - datas = pickle.dumps(stats) - table = 'domain_data_history' - - c = self._mysql_conn.cursor() - c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % table, \ - (spider.domain_name, stored, datas)) - self._mysql_conn.commit() From 8db67b17a372fee109a6b2872b35742bf6f70740 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Thu, 1 Apr 2010 17:16:38 -0300 Subject: [PATCH 33/68] scrapy manager refactor * ExecutionManager * deprecated runonce(*args) * changed start() to start(keep_alive=Bool) * changed crawl(*args) to crawl(requests, spider=None) * if no spider given, tries to resolve spider for each request * added crawl_url(url, spider=None) * added crawl_request(request, spider=None) * added crawl_domain(domain) * added crawl_spider(spider) * updated commands: crawl, runspider, start * updated webconsole * updated crawler * updated tests.test_engine * updated utils.fetch --- scrapy/command/commands/crawl.py | 11 +++- scrapy/command/commands/runspider.py | 6 +- scrapy/command/commands/start.py | 2 +- scrapy/contrib/webconsole/spiderctl.py | 4 +- scrapy/core/manager.py | 83 +++++++++++--------------- scrapy/shell.py | 2 +- scrapy/tests/test_engine.py | 3 +- scrapy/utils/fetch.py | 9 ++- 8 files changed, 61 insertions(+), 59 deletions(-) diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index 3a04423fc..f53de139d 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -1,6 +1,7 @@ from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.conf import settings +from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -24,4 +25,12 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - scrapymanager.runonce(*args) + for arg in args: + # schedule arg as url or domain + if is_url(arg): + scrapymanager.crawl_url(arg) + else: + scrapymanager.crawl_domain(arg) + + # crawl just scheduled arguments without keeping idle + scrapymanager.start() diff --git a/scrapy/command/commands/runspider.py b/scrapy/command/commands/runspider.py index e18f13cdd..2bfcd87f9 100644 --- a/scrapy/command/commands/runspider.py +++ b/scrapy/command/commands/runspider.py @@ -52,6 +52,10 @@ class Command(ScrapyCommand): dispatcher.connect(exporter.export_item, signal=signals.item_passed) exporter.start_exporting() module = _import_file(args[0]) - scrapymanager.runonce(module.SPIDER) + + # schedule spider and start engine + scrapymanager.crawl_spider(module.SPIDER) + scrapymanager.start() + if opts.output: exporter.finish_exporting() diff --git a/scrapy/command/commands/start.py b/scrapy/command/commands/start.py index 032f12bcc..2c7304787 100644 --- a/scrapy/command/commands/start.py +++ b/scrapy/command/commands/start.py @@ -9,4 +9,4 @@ class Command(ScrapyCommand): return "Start the Scrapy manager but don't run any spider (idle mode)" def run(self, args, opts): - scrapymanager.start(*args) + scrapymanager.start(keep_alive=True) diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 9719d12f0..933801854 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -135,14 +135,14 @@ class Spiderctl(object): if "add_pending_domains" in args: for domain in args["add_pending_domains"]: if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) + scrapymanager.crawl_domain(domain) s += "

" s += "Scheduled spiders:

  • %s
" % "
  • ".join(args["add_pending_domains"]) s += "

    " if "rerun_finished_domains" in args: for domain in args["rerun_finished_domains"]: if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl(domain) + scrapymanager.crawl_domain(domain) self.finished.remove(domain) s += "

    " s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index 2078aaa95..05586a117 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -1,5 +1,4 @@ import signal -from collections import defaultdict from twisted.internet import reactor @@ -12,40 +11,6 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import is_url from scrapy.utils.ossignal import install_shutdown_handlers, signal_names -def _get_spider_requests(*args): - """Collect requests and spiders from the given arguments. Returns a dict of - spider -> list of requests - """ - spider_requests = defaultdict(list) - for arg in args: - if isinstance(arg, tuple): - request, spider = arg - spider_requests[spider] = request - elif isinstance(arg, Request): - spider = spiders.fromurl(arg.url) or BaseSpider('default') - if spider: - spider_requests[spider] += [arg] - else: - log.msg('Could not find spider for request: %s' % arg, log.ERROR) - elif isinstance(arg, BaseSpider): - spider_requests[arg] += arg.start_requests() - elif is_url(arg): - spider = spiders.fromurl(arg) or BaseSpider('default') - if spider: - for req in arg_to_iter(spider.make_requests_from_url(arg)): - spider_requests[spider] += [req] - else: - log.msg('Could not find spider for url: %s' % arg, log.ERROR) - elif isinstance(arg, basestring): - spider = spiders.fromdomain(arg) - if spider: - spider_requests[spider] += spider.start_requests() - else: - log.msg('Could not find spider for domain: %s' % arg, log.ERROR) - else: - raise TypeError("Unsupported argument: %r" % arg) - return spider_requests - class ExecutionManager(object): """Process a list of sites or urls. @@ -78,24 +43,44 @@ class ExecutionManager(object): scrapyengine.configure() self.configured = True - def crawl(self, *args): - """Schedule the given args for crawling. args is a list of urls or domains""" + def crawl_url(self, url, spider=None): + """Schedule given url for crawling.""" + spider = spider or spiders.fromurl(url) + if spider: + requests = arg_to_iter(spider.make_requests_from_url(url)) + self._crawl_requests(requests, spider) + else: + log.msg('Could not find spider for url: %s' % url, log.ERROR) + + def crawl_request(self, request, spider=None): + """Schedule request for crawling.""" assert self.configured, "Scrapy Manager not yet configured" - spider_requests = _get_spider_requests(*args) - for spider, requests in spider_requests.iteritems(): - for request in requests: - scrapyengine.crawl(request, spider) + spider = spider or spiders.fromurl(request.url) + if spider: + scrapyengine.crawl(request, spider) + else: + log.msg('Could not find spider for request: %s' % url, log.ERROR) - def runonce(self, *args): - """Run the engine until it finishes scraping all domains and then exit""" - self.crawl(*args) - scrapyengine.start() - if self.control_reactor: - reactor.run(installSignalHandlers=False) + def crawl_domain(self, domain): + """Schedule given domain for crawling.""" + spider = spiders.fromdomain(domain) + if spider: + self.crawl_spider(spider) + else: + log.msg('Could not find spider for domain: %s' % domain, log.ERROR) - def start(self): + def crawl_spider(self, spider): + """Schedule spider for crawling.""" + requests = spider.start_requests() + self._crawl_requests(requests, spider) + + def _crawl_requests(self, requests, spider): + for req in requests: + self.crawl_request(req, spider) + + def start(self, keep_alive=False): """Start the scrapy server, without scheduling any domains""" - scrapyengine.keep_alive = True + scrapyengine.keep_alive = keep_alive scrapyengine.start() if self.control_reactor: reactor.run(installSignalHandlers=False) diff --git a/scrapy/shell.py b/scrapy/shell.py index 8af6af006..82baf63eb 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -104,7 +104,7 @@ class Shell(object): signal.signal(signal.SIGINT, signal.SIG_IGN) reactor.callInThread(self._console_thread, url) - scrapymanager.start() + scrapymanager.start(keep_alive=True) def inspect_response(self, response): print diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index f205ff264..873e0eb30 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -97,7 +97,8 @@ class CrawlingSession(object): dispatcher.connect(self.response_downloaded, signals.response_downloaded) scrapymanager.configure() - scrapymanager.runonce(self.spider) + scrapymanager.crawl_spider(self.spider) + scrapymanager.start() self.port.stopListening() self.wasrun = True diff --git a/scrapy/utils/fetch.py b/scrapy/utils/fetch.py index b489c75bf..a1eed0fd8 100644 --- a/scrapy/utils/fetch.py +++ b/scrapy/utils/fetch.py @@ -10,8 +10,11 @@ def fetch(urls): commands or standalone scripts. """ responses = [] - requests = [Request(url, callback=responses.append, dont_filter=True) \ - for url in urls] - scrapymanager.runonce(*requests) + for url in urls: + req = Request(url, callback=responses.append, dont_filter=True) + # @@@ request will require a suitable spider. + # If not will not be schedule + scrapymanager.crawl_request(req) + scrapymanager.start() return responses From dd477914db1c4713dd1e73f4f5a8e9fed524361c Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Thu, 1 Apr 2010 17:16:38 -0300 Subject: [PATCH 34/68] spidermanager refactoring * Implements find/create method in Spider Manager API, removed fromdomain and fromurl This method is now in charge of spider resolution, it must return spider object from its argument or raise KeyError if no spider is found. This method obsoletes from_domain and from_url methods. The default implementation of resolve only searches against spider.name, it won't use spider.allowed_domains like the old fromdomain. This is the reason of why you must supply a spider if you want to crawl an url. Find methods returns only available spider names. Not spider instances. If no spider found returns empty list. Affected modules: * command.models (force_domain) * removed spiders.force_domain * each command pass spider to crawl_* commands * command.commands.* * crawl * set spider from opts.spider if arg is url * group urls by spider to instance spider just once * genspider * use spiders.create() to check spider id * parse * log error if more than one spider found * core.manager * on crawl_* log message if multiple spiders found for url or request * shell * prints "Multiple found" if more than one spider found for url or request * populate_vars(): added spider keyword parameter * contrib.spidermanager: * removed fromdomain() & fromurl() * new create(spider_id) -> Spider. Raises KeyError if spider not found * new find_by_request(request) -> list(spiders) --- scrapy/command/commands/crawl.py | 47 ++++++++++++++++++++++++++-- scrapy/command/commands/genspider.py | 16 +++++++--- scrapy/command/commands/parse.py | 13 ++++++-- scrapy/command/models.py | 4 --- scrapy/contrib/spidermanager.py | 35 ++++++++++----------- scrapy/core/manager.py | 42 ++++++++++++++----------- scrapy/shell.py | 12 ++++--- 7 files changed, 114 insertions(+), 55 deletions(-) diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index f53de139d..a1115ffbd 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -1,8 +1,12 @@ +from scrapy import log from scrapy.command import ScrapyCommand from scrapy.core.manager import scrapymanager from scrapy.conf import settings +from scrapy.http import Request +from scrapy.spider import spiders from scrapy.utils.url import is_url +from collections import defaultdict class Command(ScrapyCommand): @@ -25,12 +29,49 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): + if opts.spider: + spider = spiders.create(opts.spider) + else: + spider = None + + # aggregate urls and domains + urls = [] + domains = [] for arg in args: - # schedule arg as url or domain if is_url(arg): - scrapymanager.crawl_url(arg) + urls.append(arg) else: - scrapymanager.crawl_domain(arg) + domains.append(arg) + + # schedule first domains + for dom in domains: + scrapymanager.crawl_domain(dom) + + # if forced spider schedule urls directly + if spider: + for url in urls: + scrapymanager.crawl_url(url, spider) + else: + # group urls by spider + spider_urls = defaultdict(list) + find_by_url = lambda url: spiders.find_by_request(Request(url)) + for url in urls: + spider_names = find_by_url(url) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + + # schedule grouped urls with same spider + for name, urls in spider_urls.iteritems(): + # instance spider for each url-list + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) # crawl just scheduled arguments without keeping idle scrapymanager.start() diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index f39d52d14..b2e0a5fb6 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -59,11 +59,17 @@ class Command(ScrapyCommand): module = sanitize_module_name(args[0]) domain = args[1] - spider = spiders.fromdomain(domain) - if spider and not opts.force: - print "Spider '%s' already exists in module:" % domain - print " %s" % spider.__module__ - sys.exit(1) + + # if spider already exists and not force option then halt + try: + spider = spiders.create(domain) + except KeyError: + pass + else: + if not opts.force: + print "Spider '%s' already exists in module:" % domain + print " %s" % spider.__module__ + sys.exit(1) template_file = self._find_template(opts.template) if template_file: diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 5d5143f08..23bdd2ed5 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -37,10 +37,17 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for url: %s' % response.url, level=log.ERROR) + spider_names = spiders.find_by_request(response.request) + if not spider_names: + log.msg('Cannot find spider for url: %s' % response.url, + level=log.ERROR) return (), () + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % response.url, + level=log.ERROR) + return (), () + else: + spider = spiders.create(spider_names[0]) if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index b019c3e52..12eeef930 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -99,10 +99,6 @@ class ScrapyCommand(object): if opts.nolog: settings.overrides['LOG_ENABLED'] = False - if opts.spider: - from scrapy.spider import spiders - spiders.force_domain = opts.spider - if opts.pidfile: with open(opts.pidfile, "w") as f: f.write(str(os.getpid())) diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 229301c2c..e37e861a5 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -19,35 +19,34 @@ class TwistedPluginSpiderManager(object): def __init__(self): self.loaded = False - self.force_domain = None - self._invaliddict = {} self._spiders = {} - def fromdomain(self, domain): - return self._spiders.get(domain) + def create(self, spider_id): + """ + Returns Spider instance by given identifier. + If not exists raises KeyError. + """ + #@@@ currently spider_id = domain + # if lookup fails let dict's KeyError exception propagate + return self._spiders[spider_id] - def fromurl(self, url): - if self.force_domain: - return self._spiders.get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self._spiders: # try first locating by domain - return self._spiders[domain] - else: # else search spider by spider - plist = self._spiders.values() - for p in plist: - if url_is_from_spider(url, p): - return p + def find_by_request(self, request): + """ + Returns list of spiders ids that match given Request. + """ + # just find by request.url + return [domain for domain, spider in self._spiders.iteritems() + if url_is_from_spider(request.url, spider)] def list(self): + """Returns list of spiders available.""" return self._spiders.keys() def load(self, spider_modules=None): + """Load spiders from module directory.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._invaliddict = {} self._spiders = {} modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index 05586a117..dc6bba782 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -6,20 +6,13 @@ from scrapy.extension import extensions from scrapy import log from scrapy.http import Request from scrapy.core.engine import scrapyengine -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import spiders from scrapy.utils.misc import arg_to_iter -from scrapy.utils.url import is_url from scrapy.utils.ossignal import install_shutdown_handlers, signal_names class ExecutionManager(object): - """Process a list of sites or urls. - This class should be used in a main for process a list of sites/urls. - - It extracts products and could be used to store results in a database or - just for testing spiders. - """ def __init__(self): self.interrupted = False self.configured = False @@ -45,29 +38,30 @@ class ExecutionManager(object): def crawl_url(self, url, spider=None): """Schedule given url for crawling.""" - spider = spider or spiders.fromurl(url) + if spider is None: + spider = self._create_spider_for_request(Request(url), log_none=True, \ + log_multiple=True) if spider: requests = arg_to_iter(spider.make_requests_from_url(url)) self._crawl_requests(requests, spider) - else: - log.msg('Could not find spider for url: %s' % url, log.ERROR) def crawl_request(self, request, spider=None): """Schedule request for crawling.""" assert self.configured, "Scrapy Manager not yet configured" - spider = spider or spiders.fromurl(request.url) + if spider is None: + spider = self._create_spider_for_request(request, log_none=True, \ + log_multiple=True) if spider: scrapyengine.crawl(request, spider) - else: - log.msg('Could not find spider for request: %s' % url, log.ERROR) def crawl_domain(self, domain): """Schedule given domain for crawling.""" - spider = spiders.fromdomain(domain) - if spider: - self.crawl_spider(spider) - else: + try: + spider = spiders.create(domain) + except KeyError: log.msg('Could not find spider for domain: %s' % domain, log.ERROR) + else: + self.crawl_spider(spider) def crawl_spider(self, spider): """Schedule spider for crawling.""" @@ -75,6 +69,7 @@ class ExecutionManager(object): self._crawl_requests(requests, spider) def _crawl_requests(self, requests, spider): + """Shortcut to schedule a list of requests""" for req in requests: self.crawl_request(req, spider) @@ -90,6 +85,17 @@ class ExecutionManager(object): self.interrupted = True scrapyengine.stop() + def _create_spider_for_request(self, request, default=None, log_none=False, \ + log_multiple=False): + spider_names = spiders.find_by_request(request) + if len(spider_names) == 1: + return spiders.create(spider_names[0]) + if len(spider_names) > 1 and log_multiple: + log.msg('More than one spider found for: %s' % request, log.ERROR) + if len(spider_names) == 0 and log_none: + log.msg('Could not find spider for: %s' % request, log.ERROR) + return default + def _signal_shutdown(self, signum, _): signame = signal_names[signum] log.msg("Received %s, shutting down gracefully. Send again to force " \ diff --git a/scrapy/shell.py b/scrapy/shell.py index 82baf63eb..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -35,6 +35,7 @@ def parse_url(url): u = urlparse.urlparse(url) return url + class Shell(object): requires_project = False @@ -52,18 +53,21 @@ class Shell(object): else: url = parse_url(request_or_url) request = Request(url) - spider = spiders.fromurl(url) or BaseSpider('default') + + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default'), log_multiple=True) + print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ request, spider) if response: - self.populate_vars(url, response, request) + self.populate_vars(url, response, request, spider) if print_help: self.print_help() else: print "Done - use shelp() to see available objects" - def populate_vars(self, url=None, response=None, request=None): + def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class() self.vars['item'] = item if url: @@ -73,7 +77,7 @@ class Shell(object): self.vars['url'] = url self.vars['response'] = response self.vars['request'] = request - self.vars['spider'] = spiders.fromurl(url) + self.vars['spider'] = spider if not self.nofetch: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser From 35a7059636182d40de9f8b0080e19aa00fd5a19f Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Thu, 1 Apr 2010 17:16:38 -0300 Subject: [PATCH 35/68] cleanup and refactor of parse & fetch commands * removed scrapy.utils.fetch * each command schedule requests and start scrapy engine * fetch command instance BaseSpider if given url does not match any spider or match more than one * parse command schedule url if one spider matches * parse and fetch doesn't support multiple urls as parameter * force spider behavior --spider moved from BaseCommand to only commands: fetch, parse, crawl --- scrapy/command/commands/crawl.py | 75 +++++++++++----------- scrapy/command/commands/fetch.py | 30 +++++++-- scrapy/command/commands/parse.py | 106 +++++++++++++++++++------------ scrapy/command/models.py | 2 - scrapy/utils/fetch.py | 20 ------ scrapy/utils/spider.py | 1 - 6 files changed, 128 insertions(+), 106 deletions(-) delete mode 100644 scrapy/utils/fetch.py diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index a1115ffbd..00f375df2 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -20,6 +20,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider when arguments are urls") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \ help="don't follow links (for use with URLs only)") @@ -29,12 +31,41 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - if opts.spider: - spider = spiders.create(opts.spider) - else: - spider = None - # aggregate urls and domains + urls, domains = self._split_urls_and_domains(args) + for dom in domains: + scrapymanager.crawl_domain(dom) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + for url in urls: + scrapymanager.crawl_url(url, spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + else: + for name, urls in self._group_urls_by_spider(urls): + spider = spiders.create(name) + for url in urls: + scrapymanager.crawl_url(url, spider) + + scrapymanager.start() + + def _group_urls_by_spider(self, urls): + spider_urls = defaultdict(list) + for url in urls: + spider_names = spiders.find_by_request(Request(url)) + if not spider_names: + log.msg('Could not find spider for url: %s' % url, + log.ERROR) + elif len(spider_names) > 1: + log.msg('More than one spider found for url: %s' % url, + log.ERROR) + else: + spider_urls[spider_names[0]].append(url) + return spider_urls.items() + + def _split_urls_and_domains(self, args): urls = [] domains = [] for arg in args: @@ -42,36 +73,4 @@ class Command(ScrapyCommand): urls.append(arg) else: domains.append(arg) - - # schedule first domains - for dom in domains: - scrapymanager.crawl_domain(dom) - - # if forced spider schedule urls directly - if spider: - for url in urls: - scrapymanager.crawl_url(url, spider) - else: - # group urls by spider - spider_urls = defaultdict(list) - find_by_url = lambda url: spiders.find_by_request(Request(url)) - for url in urls: - spider_names = find_by_url(url) - if not spider_names: - log.msg('Could not find spider for url: %s' % url, - log.ERROR) - elif len(spider_names) > 1: - log.msg('More than one spider found for url: %s' % url, - log.ERROR) - else: - spider_urls[spider_names[0]].append(url) - - # schedule grouped urls with same spider - for name, urls in spider_urls.iteritems(): - # instance spider for each url-list - spider = spiders.create(name) - for url in urls: - scrapymanager.crawl_url(url, spider) - - # crawl just scheduled arguments without keeping idle - scrapymanager.start() + return urls, domains diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index bbc2efa9e..bb2df1400 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -1,7 +1,11 @@ import pprint +from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager +from scrapy.http import Request +from scrapy.spider import BaseSpider, spiders +from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -19,17 +23,33 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", + help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") def run(self, args, opts): - if len(args) != 1: - print "One URL is required" - return + if len(args) != 1 or not is_url(args[0]): + return False + responses = [] # to collect downloaded responses + request = Request(args[0], callback=responses.append, dont_filter=True) - responses = fetch(args) + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg("Could not find spider: %s" % opts.spider, log.ERROR) + else: + spider = scrapymanager._create_spider_for_request(request, \ + BaseSpider('default')) + + scrapymanager.crawl_request(request, spider) + scrapymanager.start() + + # display response if responses: if opts.headers: pprint.pprint(responses[0].headers) else: print responses[0].body + diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index 23bdd2ed5..d67a26d79 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -1,11 +1,15 @@ from scrapy.command import ScrapyCommand -from scrapy.utils.fetch import fetch +from scrapy.core.manager import scrapymanager from scrapy.http import Request from scrapy.item import BaseItem from scrapy.spider import spiders from scrapy.utils import display +from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.url import is_url from scrapy import log +from collections import defaultdict + class Command(ScrapyCommand): requires_project = True @@ -18,6 +22,8 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) + parser.add_option("--spider", dest="spider", default=None, \ + help="always use this spider") parser.add_option("--nolinks", dest="nolinks", action="store_true", \ help="don't show extracted links") parser.add_option("--noitems", dest="noitems", action="store_true", \ @@ -37,25 +43,13 @@ class Command(ScrapyCommand): return item def run_callback(self, spider, response, callback, args, opts): - spider_names = spiders.find_by_request(response.request) - if not spider_names: - log.msg('Cannot find spider for url: %s' % response.url, - level=log.ERROR) - return (), () - elif len(spider_names) > 1: - log.msg('More than one spider found for url: %s' % response.url, - level=log.ERROR) - return (), () - else: - spider = spiders.create(spider_names[0]) - if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) if not callback_fcn: log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name)) return (), () - result = callback_fcn(response) + result = iterate_spider_output(callback_fcn(response)) links = [i for i in result if isinstance(i, Request)] items = [self.pipeline_process(i, spider, opts) for i in result if \ isinstance(i, BaseItem)] @@ -78,36 +72,68 @@ class Command(ScrapyCommand): display.pprint(list(links)) def run(self, args, opts): - if not args: - print "An URL is required" + if not len(args) == 1 or not is_url(args[0]): + return False + + request = Request(args[0]) + + if opts.spider: + try: + spider = spiders.create(opts.spider) + except KeyError: + log.msg('Could not find spider: %s' % opts.spider, log.ERROR) + return + else: + spider = scrapymanager._create_spider_for_request(request, \ + log_none=True, log_multiple=True) + + if not spider: return - for response in fetch(args): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Cannot find spider for "%s"' % response.url) - continue + responses = [] # to collect downloaded responses + request = request.replace(callback=responses.append) - if self.callbacks: - for callback in self.callbacks: - items, links = self.run_callback(spider, response, callback, args, opts) - self.print_results(items, links, callback, opts) + scrapymanager.crawl_request(request, spider) + scrapymanager.start() - elif opts.rules: - rules = getattr(spider, 'rules', None) - if rules: - items, links = [], [] - for rule in rules: - if rule.callback and rule.link_extractor.matches(response.url): - items, links = self.run_callback(spider, response, rule.callback, args, opts) - self.print_results(items, links, rule.callback, opts) - break - else: - log.msg('No rules found for spider "%s", please specify a callback for parsing' \ - % spider.domain_name) - continue + if not responses: + log.msg('No response returned', log.ERROR, spider=spider) + return + # now process response + # - if callbacks defined then call each one print results + # - if --rules option given search for matching spider's rule + # - default print result using default 'parse' spider's callback + response = responses[0] + + if self.callbacks: + # apply each callback + for callback in self.callbacks: + items, links = self.run_callback(spider, response, + callback, args, opts) + self.print_results(items, links, callback, opts) + elif opts.rules: + # search for matching spider's rule + if hasattr(spider, 'rules') and spider.rules: + items, links = [], [] + for rule in spider.rules: + if rule.link_extractor.matches(response.url) \ + and rule.callback: + + items, links = self.run_callback(spider, + response, rule.callback, + args, opts) + self.print_results(items, links, + rule.callback, opts) + # first-match rule breaks rules loop + break else: - items, links = self.run_callback(spider, response, 'parse', args, opts) - self.print_results(items, links, 'parse', opts) + log.msg('No rules found for spider "%s", ' \ + 'please specify a callback for parsing' \ + % spider.domain_name, log.ERROR) + else: + # default callback 'parse' + items, links = self.run_callback(spider, response, + 'parse', args, opts) + self.print_results(items, links, 'parse', opts) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index 12eeef930..049365ebb 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -57,8 +57,6 @@ class ScrapyCommand(object): help="log level (default: %s)" % settings['LOGLEVEL']) group.add_option("--nolog", action="store_true", dest="nolog", \ help="disable logging completely") - group.add_option("--spider", dest="spider", default=None, \ - help="always use this spider when arguments are urls") group.add_option("--profile", dest="profile", metavar="FILE", default=None, \ help="write python cProfile stats to FILE") group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \ diff --git a/scrapy/utils/fetch.py b/scrapy/utils/fetch.py deleted file mode 100644 index a1eed0fd8..000000000 --- a/scrapy/utils/fetch.py +++ /dev/null @@ -1,20 +0,0 @@ -from scrapy.http import Request -from scrapy.core.manager import scrapymanager - -def fetch(urls): - """Fetch a list of urls and return a list of the downloaded Scrapy - Responses. - - This is a blocking function not suitable for calling from spiders. Instead, - it is indended to be called from outside the framework such as Scrapy - commands or standalone scripts. - """ - responses = [] - for url in urls: - req = Request(url, callback=responses.append, dont_filter=True) - # @@@ request will require a suitable spider. - # If not will not be schedule - scrapymanager.crawl_request(req) - scrapymanager.start() - return responses - diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 9307fc234..29432f0e9 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,4 +4,3 @@ from scrapy.utils.misc import arg_to_iter def iterate_spider_output(result): return [result] if isinstance(result, BaseItem) else arg_to_iter(result) - From db5c3df67994350f98259824cb01702cf3892e60 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Thu, 1 Apr 2010 18:27:22 -0300 Subject: [PATCH 36/68] SEP12 implementation * Rename BaseSpider.domain_name to BaseSpider.name This patch implements the domain_name to name change in BaseSpider class and change all spider instantiations to use the new attribute. * Add allowed_domains to spider This patch implements the merging of spider.domain_name and spider.extra_domain_names in spider.allowed_domains for offsite checking purposes. Note that spider.domain_name is not touched by this patch, only not used. * Remove spider.domain_name references from scrapy.stats * Rename domain_stats to spider_stats in MemoryStatsCollector * Use ``spider`` instead of ``domain`` in SimpledbStatsCollector * Rename domain_stats_history table to spider_data_history and rename domain field to spider in MysqlStatsCollector * Refactor genspider command The new signature for genspider is: genspider [options] . Genspider uses domain_name for spider name and for the module name. * Remove spider.domain_name references * Update crawl command signature * docs: updated references to domain_name * examples/experimental: use spider.name * genspider: require * spidermanager: renamed crawl_domain to crawl_spider_name * spiderctl: updated references of *domain* to spider * added backward compatiblity with legacy spider's attributes 'domain_name' and 'extra_domain_names' --- docs/intro/overview.rst | 3 +- docs/intro/tutorial.rst | 13 +-- docs/topics/downloader-middleware.rst | 2 +- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 4 +- docs/topics/firebug.rst | 3 +- docs/topics/request-response.rst | 2 +- docs/topics/shell.rst | 2 +- docs/topics/spider-middleware.rst | 7 +- docs/topics/spiders.rst | 41 ++++++---- docs/topics/stats.rst | 14 ++-- .../googledir/spiders/google_directory.py | 3 +- .../imdb/imdb/spiders/imdb_site.py | 3 +- .../googledir/spiders/google_directory.py | 3 +- scrapy/command/commands/crawl.py | 19 ++--- scrapy/command/commands/genspider.py | 28 ++++--- scrapy/command/commands/parse.py | 4 +- scrapy/contrib/aws.py | 2 +- .../contrib/downloadermiddleware/httpcache.py | 2 +- scrapy/contrib/itemsampler.py | 20 ++--- scrapy/contrib/pipeline/images.py | 4 +- scrapy/contrib/spidermanager.py | 10 +-- scrapy/contrib/spidermiddleware/offsite.py | 3 +- scrapy/contrib/statsmailer.py | 4 +- scrapy/contrib/webconsole/livestats.py | 2 +- scrapy/contrib/webconsole/spiderctl.py | 82 +++++++++---------- scrapy/contrib/webconsole/stats.py | 2 +- scrapy/core/manager.py | 8 +- scrapy/log.py | 4 +- scrapy/spider/models.py | 45 ++++++---- scrapy/stats/collector/__init__.py | 4 +- scrapy/stats/collector/simpledb.py | 4 +- scrapy/templates/spiders/basic.tmpl | 5 +- scrapy/templates/spiders/crawl.tmpl | 7 +- scrapy/templates/spiders/csvfeed.tmpl | 5 +- scrapy/templates/spiders/xmlfeed.tmpl | 5 +- scrapy/tests/test_commands.py | 14 +++- scrapy/tests/test_engine.py | 8 +- scrapy/tests/test_spider.py | 56 +++++++++++++ scrapy/tests/test_spidermiddleware_offsite.py | 4 +- scrapy/utils/url.py | 4 +- 41 files changed, 273 insertions(+), 184 deletions(-) create mode 100644 scrapy/tests/test_spider.py diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7eec036cf..5dd61100e 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -128,7 +128,8 @@ Finally, here's the spider code:: class MininovaSpider(CrawlSpider): - domain_name = 'mininova.org' + name = 'mininova.org' + allowed_domains = ['mininova.org'] start_urls = ['http://www.mininova.org/today'] rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 4358a7ae0..8fb6c7db5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -102,8 +102,8 @@ to parse the contents of those pages to extract :ref:`items `. To create a Spider, you must subclass :class:`scrapy.spider.BaseSpider`, and define the three main, mandatory, attributes: -* :attr:`~scrapy.spider.BaseSpider.domain_name`: identifies the Spider. It must - be unique, that is, you can't set the same domain name for different Spiders. +* :attr:`~scrapy.spider.BaseSpider.name`: identifies the Spider. It must be + unique, that is, you can't set the same name for different Spiders. * :attr:`~scrapy.spider.BaseSpider.start_urls`: is a list of URLs where the Spider will begin to crawl from. So, the first pages downloaded will be those @@ -128,7 +128,8 @@ This is the code for our first Spider, save it in a file named from scrapy.spider import BaseSpider class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -354,7 +355,8 @@ Let's add this code to our spider:: from scrapy.selector import HtmlXPathSelector class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -398,7 +400,8 @@ scraped so far, the code for our Spider should be like this:: from dmoz.items import DmozItem class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15ead63be..d8629849f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -199,7 +199,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' - domain_name = 'intranet.example.com' + name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 711df19ce..a4312cb90 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -52,7 +52,7 @@ Exporter to export scraped items to different files, one per spider:: self.files = {} def spider_opened(self, spider): - file = open('%s_products.xml' % spider.domain_name, 'w+b') + file = open('%s_products.xml' % spider.name, 'w+b') self.files[spider] = file self.exporter = XmlItemExporter(file) self.exporter.start_exporting() diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 1bde87bb7..1ff1f068f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -105,10 +105,10 @@ everytime a domain/spider is opened and closed:: dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def spider_opened(self, spider): - log.msg("opened spider %s" % spider.domain_name) + log.msg("opened spider %s" % spider.name) def spider_closed(self, spider): - log.msg("closed spider %s" % spider.domain_name) + log.msg("closed spider %s" % spider.name) .. _topics-extensions-ref-manager: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 9f198a410..f42e6a77e 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -79,7 +79,8 @@ This is how the spider would look so far:: from scrapy.contrib.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allowed_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 63d289d09..48ec81ccf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -321,7 +321,7 @@ user name and password. You can use the :meth:`FormRequest.from_response` method for this job. Here's an example spider which uses it:: class LoginSpider(BaseSpider): - domain_name = 'example.com' + name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] def parse(self, response): diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 6e3de9bc1..45d9cf0c3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -163,7 +163,7 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function. Here's an example of how you would call it from your spider:: class MySpider(BaseSpider): - domain_name = 'example.com' + ... def parse(self, response): if response.url == 'http://www.example.com/products.php': diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c0e57ed06..0c5dcc675 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -210,11 +210,8 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. - This middleware filters out every request whose host names don't match - :attr:`~scrapy.spider.BaseSpider.domain_name`, or the spider - :attr:`~scrapy.spider.BaseSpider.domain_name` prefixed by "www.". - Spider can add more domains to exclude using - :attr:`~scrapy.spider.BaseSpider.extra_domain_names` attribute. + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e60549098..b69f9a445 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -70,20 +70,22 @@ BaseSpider requests the given ``start_urls``/``start_requests``, and calls the spider's method ``parse`` for each of the resulting responses. - .. attribute:: domain_name + .. attribute:: name - A string which defines the domain name for this spider, which will also be - the unique identifier for this spider (which means you can't have two - spider with the same ``domain_name``). This is the most important spider - attribute and it's required, and it's the name by which Scrapy will known - the spider. + A string which defines the name for this spider. The spider name is how + the spider is located (and instantiated) by Scrapy, so it must be + unique. However, nothing prevents you from instantiating more than one + instance of the same spider. This is the most important spider attribute + and it's required. - .. attribute:: extra_domain_names + Is recommended to name your spiders after the domain that their crawl. - An optional list of strings containing additional domains that this - spider is allowed to crawl. Requests for URLs not belonging to the - domain name specified in :attr:`domain_name` or this list won't be - followed. + .. attribute:: allowed_domains + + An optional list of strings containing domains that this spider is + allowed to crawl. Requests for URLs not belonging to the domain names + specified in this list won't be followed if + :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls @@ -144,7 +146,7 @@ BaseSpider .. method:: log(message, [level, component]) Log a message using the :func:`scrapy.log.msg` function, automatically - populating the domain argument with the :attr:`domain_name` of this + populating the spider argument with the :attr:`name` of this spider. For more information see :ref:`topics-logging`. @@ -157,7 +159,8 @@ Let's see an example:: from scrapy.spider import BaseSpider class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -177,7 +180,8 @@ Another example returning multiples Requests and Items from a single callback:: from myproject.items import MyItem class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -254,7 +258,8 @@ Let's now take a look at an example CrawlSpider with rules:: from scrapy.item import Item class MySpider(CrawlSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com'] rules = ( @@ -378,7 +383,8 @@ These spiders are pretty easy to use, let's have at one example:: from myproject.items import TestItem class MySpider(XMLFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.xml'] iterator = 'iternodes' # This is actually unnecesary, since it's the default value itertag = 'item' @@ -435,7 +441,8 @@ Let's see an example similar to the previous one, but using a from myproject.items import TestItem class MySpider(CSVFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.csv'] delimiter = ';' headers = ['id', 'name', 'description'] diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 971a33364..34a3c8b7c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -204,15 +204,15 @@ MemoryStatsCollector A simple stats collector that keeps the stats of the last scraping run (for each spider) in memory, after they're closed. The stats can be accessed - through the :attr:`domain_stats` attribute, which is a dict keyed by spider + through the :attr:`spider_stats` attribute, which is a dict keyed by spider domain name. This is the default Stats Collector used in Scrapy. - .. attribute:: domain_stats + .. attribute:: spider_stats - A dict of dicts (keyed by spider domain name) containing the stats of - the last scraping run for each domain. + A dict of dicts (keyed by spider name) containing the stats of the last + scraping run for each spider. DummyStatsCollector ------------------- @@ -240,11 +240,11 @@ SimpledbStatsCollector In addition to the existing stats keys the following keys are added at persitance time: - * ``domain``: the spider domain (so you can use it later for querying stats - for that domain) + * ``spider``: the spider name (so you can use it later for querying stats + for that spider) * ``timestamp``: the timestamp when the stats were persisited - Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB + Both the ``spider`` and ``timestamp`` are used for generating the SimpleDB item name in order to avoid overwriting stats of previous scraping runs. As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py index fdbf3f24a..2ed7c52a6 100644 --- a/examples/experimental/googledir/googledir/spiders/google_directory.py +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -6,7 +6,8 @@ from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'google_directory' + allowed_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py index 5c3f3c06b..8c2ebcd01 100644 --- a/examples/experimental/imdb/imdb/spiders/imdb_site.py +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -29,7 +29,8 @@ class MovieItem(ImdbItem): class ImdbSiteSpider(CrawlSpider): - domain_name = 'imdb.com' + name = 'imdb.com' + allowed_domains = ['imdb.com'] start_urls = ['http://www.imdb.com/'] # extract requests using this classes from urls matching 'follow' flag diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index 054cef022..2af52bf1f 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -6,7 +6,8 @@ from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allow_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index 00f375df2..9c02b08e3 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -13,10 +13,10 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] ..." + return "[options] ..." def short_desc(self): - return "Start crawling a domain or URL" + return "Start crawling from a spider or URL" def add_options(self, parser): ScrapyCommand.add_options(self, parser) @@ -31,10 +31,9 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - - urls, domains = self._split_urls_and_domains(args) - for dom in domains: - scrapymanager.crawl_domain(dom) + urls, names = self._split_urls_and_names(args) + for name in names: + scrapymanager.crawl_spider_name(name) if opts.spider: try: @@ -65,12 +64,12 @@ class Command(ScrapyCommand): spider_urls[spider_names[0]].append(url) return spider_urls.items() - def _split_urls_and_domains(self, args): + def _split_urls_and_names(self, args): urls = [] - domains = [] + names = [] for arg in args: if is_url(arg): urls.append(arg) else: - domains.append(arg) - return urls, domains + names.append(arg) + return urls, names diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index b2e0a5fb6..2289fee1a 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -15,10 +15,11 @@ SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders') def sanitize_module_name(module_name): - """Sanitize the given module name, by replacing dashes with underscores and - prefixing it with a letter if it doesn't start with one + """Sanitize the given module name, by replacing dashes and points + with underscores and prefixing it with a letter if it doesn't start + with one """ - module_name = module_name.replace('-', '_') + module_name = module_name.replace('-', '_').replace('.', '_') if module_name[0] not in string.ascii_letters: module_name = "a" + module_name return module_name @@ -28,7 +29,7 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] " + return "[options] " def short_desc(self): return "Generate new spider based on template passed with -t or --template" @@ -54,34 +55,37 @@ class Command(ScrapyCommand): print template.read() return - if len(args) < 2: + if len(args) != 2: return False - module = sanitize_module_name(args[0]) + name = args[0] domain = args[1] + module = sanitize_module_name(name) + # if spider already exists and not force option then halt try: - spider = spiders.create(domain) + spider = spiders.create(name) except KeyError: pass else: if not opts.force: - print "Spider '%s' already exists in module:" % domain + print "Spider '%s' already exists in module:" % name print " %s" % spider.__module__ sys.exit(1) template_file = self._find_template(opts.template) if template_file: - self._genspider(module, domain, opts.template, template_file) + self._genspider(module, name, domain, opts.template, template_file) - def _genspider(self, module, domain, template_name, template_file): + def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" tvars = { 'project_name': settings.get('BOT_NAME'), 'ProjectName': string_camelcase(settings.get('BOT_NAME')), 'module': module, - 'site': domain, + 'name': name, + 'domain': domain, 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } @@ -92,7 +96,7 @@ class Command(ScrapyCommand): shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print "Created spider %r using template %r in module:" % (domain, \ + print "Created spider %r using template %r in module:" % (name, \ template_name) print " %s.%s" % (spiders_module.__name__, module) diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index d67a26d79..9aabe94ff 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -46,7 +46,7 @@ class Command(ScrapyCommand): if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) if not callback_fcn: - log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name)) + log.msg('Cannot find callback %s in %s spider' % (callback, spider.name)) return (), () result = iterate_spider_output(callback_fcn(response)) @@ -130,7 +130,7 @@ class Command(ScrapyCommand): else: log.msg('No rules found for spider "%s", ' \ 'please specify a callback for parsing' \ - % spider.domain_name, log.ERROR) + % spider.name, log.ERROR) else: # default callback 'parse' items, links = self.run_callback(spider, response, diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 1f54b7634..91b134a0e 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -20,7 +20,7 @@ class AWSMiddleware(object): def process_request(self, request, spider): hostname = urlparse_cached(request).hostname - if spider.domain_name == 's3.amazonaws.com' \ + if spider.name == 's3.amazonaws.com' \ or (hostname and hostname.endswith('s3.amazonaws.com')): request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \ time.gmtime()) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 239c05bc7..341e9b5a1 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -108,7 +108,7 @@ class FilesystemCacheStorage(object): def _get_request_path(self, spider, request): key = request_fingerprint(request) - return join(self.cachedir, spider.domain_name, key[0:2], key) + return join(self.cachedir, spider.name, key[0:2], key) def _read_meta(self, spider, request): rpath = self._get_request_path(spider, request) diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index 6021bbf5a..9f8666467 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -1,6 +1,6 @@ """ This module provides a mechanism for collecting one (or more) sample items per -domain. +spider. The items are collected in a dict of guid->item and persisted by pickling that dict into a file. @@ -8,7 +8,7 @@ dict into a file. This can be useful for testing changes made to the framework or other common code that affects several spiders. -It uses the scrapy stats service to keep track of which domains are already +It uses the scrapy stats service to keep track of which spiders are already sampled. Settings that affect this module: @@ -48,7 +48,7 @@ class ItemSamplerPipeline(object): raise NotConfigured self.items = {} self.spiders_count = 0 - self.empty_domains = set() + self.empty_spiders = set() dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -66,21 +66,21 @@ class ItemSamplerPipeline(object): def engine_stopped(self): with open(self.filename, 'w') as f: pickle.dump(self.items, f) - if self.empty_domains: - log.msg("No products sampled for: %s" % " ".join(self.empty_domains), \ + if self.empty_spiders: + log.msg("No products sampled for: %s" % " ".join(self.empty_spiders), \ level=log.WARNING) def spider_closed(self, spider, reason): if reason == 'finished' and not stats.get_value("items_sampled", spider=spider): - self.empty_domains.add(spider.domain_name) + self.empty_spiders.add(spider.name) self.spiders_count += 1 - log.msg("Sampled %d domains so far (%d empty)" % (self.spiders_count, \ - len(self.empty_domains)), level=log.INFO) + log.msg("Sampled %d spiders so far (%d empty)" % (self.spiders_count, \ + len(self.empty_spiders)), level=log.INFO) class ItemSamplerMiddleware(object): - """This middleware drops items and requests (when domain sampling has been - completed) to accelerate the processing of remaining domains""" + """This middleware drops items and requests (when spider sampling has been + completed) to accelerate the processing of remaining spiders""" def __init__(self): if not settings['ITEMSAMPLER_FILE']: diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 3a93553dd..a56b3da0b 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -47,7 +47,7 @@ class FSImagesStore(object): dispatcher.connect(self.spider_closed, signals.spider_closed) def spider_closed(self, spider): - self.created_directories.pop(spider.domain_name, None) + self.created_directories.pop(spider.name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) @@ -92,7 +92,7 @@ class _S3AmazonAWSSpider(BaseSpider): It means that a spider that uses download_delay or alike is not going to be delayed even more because it is uploading images to s3. """ - domain_name = "s3.amazonaws.com" + name = "s3.amazonaws.com" start_urls = ['http://s3.amazonaws.com/'] max_concurrent_requests = 100 diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index e37e861a5..331f55aa5 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -53,7 +53,7 @@ class TwistedPluginSpiderManager(object): for module in modules: for spider in self._getspiders(ISpider, module): ISpider.validateInvariants(spider) - self._spiders[spider.domain_name] = spider + self._spiders[spider.name] = spider self.loaded = True def _getspiders(self, interface, package): @@ -76,14 +76,14 @@ class TwistedPluginSpiderManager(object): """Reload spider module to release any resources held on to by the spider """ - domain = spider.domain_name - if domain not in self._spiders: + name = spider.name + if name not in self._spiders: return - spider = self._spiders[domain] + spider = self._spiders[name] module_name = spider.__module__ module = sys.modules[module_name] if hasattr(module, 'SPIDER'): log.msg("Reloading module %s" % module_name, spider=spider, \ level=log.DEBUG) new_module = rebuild(module, doLog=0) - self._spiders[domain] = new_module.SPIDER + self._spiders[name] = new_module.SPIDER diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 5e40d9915..f28b0d53d 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -47,8 +47,7 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - domains = [spider.domain_name] + spider.extra_domain_names - self.host_regexes[spider] = self.get_host_regex(domains) + self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains) self.domains_seen[spider] = set() def spider_closed(self, spider): diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index c2185e0a7..fc76ccfbf 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -23,6 +23,6 @@ class StatsMailer(object): mail = MailSender() body = "Global stats\n\n" body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.domain_name + body += "\n\n%s stats\n\n" % spider.name body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - mail.send(self.recipients, "Scrapy stats for: %s" % spider.domain_name, body) + mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 22afc20c1..2e61e8bc3 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -60,7 +60,7 @@ class LiveStats(object): runtime = datetime.now() - stats.started s += '%s%d%d%d%d%d%d%s%s\n' % \ - (spider.domain_name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) + (spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) totdomains += 1 totscraped += stats.scraped diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 933801854..ce3fdffd6 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -25,18 +25,18 @@ class Spiderctl(object): dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) def spider_opened(self, spider): - self.running[spider.domain_name] = spider + self.running[spider.name] = spider def spider_closed(self, spider): - del self.running[spider.domain_name] - self.finished.add(spider.domain_name) + del self.running[spider.name] + self.finished.add(spider.name) def webconsole_render(self, wc_request): if wc_request.args: changes = self.webconsole_control(wc_request) - self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders] - self.idle = [d for d in self.enabled_domains if d not in self.scheduled + self.scheduled = [s.name for s in scrapyengine.spider_scheduler._pending_spiders] + self.idle = [d for d in self.enabled_spiders if d not in self.scheduled and d not in self.running and d not in self.finished] @@ -53,9 +53,9 @@ class Spiderctl(object): # idle s += "\n" s += '
    \n' - s += '\n' + for name in sorted(self.idle): + s += "\n" % name s += '
    \n' s += '
    ' s += '\n' @@ -65,9 +65,9 @@ class Spiderctl(object): # scheduled s += "\n" s += '\n' - s += '\n' + for name in self.scheduled: + s += "\n" % name s += '
    \n' s += '
    ' s += '\n' @@ -78,9 +78,9 @@ class Spiderctl(object): # running s += "\n" s += '\n' - s += '\n' + for name in sorted(self.running): + s += "\n" % name s += '
    \n' s += '
    ' s += '\n' @@ -90,9 +90,9 @@ class Spiderctl(object): # finished s += "\n" s += '\n' - s += '\n' + for name in sorted(self.finished): + s += "\n" % name s += '
    \n' s += '
    ' s += '\n' @@ -114,42 +114,42 @@ class Spiderctl(object): args = wc_request.args s = "
    \n" - if "stop_running_domains" in args: + if "stop_running_spiders" in args: s += "

    " - stopped_domains = [] - for domain in args["stop_running_domains"]: - if domain in self.running: - scrapyengine.close_spider(self.running[domain]) - stopped_domains.append(domain) - s += "Stopped spiders:

    • %s
    " % "
  • ".join(stopped_domains) + stopped_spiders = [] + for name in args["stop_running_spiders"]: + if name in self.running: + scrapyengine.close_spider(self.running[name]) + stopped_spiders.append(name) + s += "Stopped spiders:
    • %s
    " % "
  • ".join(stopped_spiders) s += "

    " - if "remove_pending_domains" in args: + if "remove_pending_spiders" in args: removed = [] - for domain in args["remove_pending_domains"]: - if scrapyengine.spider_scheduler.remove_pending_domain(domain): - removed.append(domain) + for name in args["remove_pending_spiders"]: + if scrapyengine.spider_scheduler.remove_pending_spider(name): + removed.append(name) if removed: s += "

    " - s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_domains"]) + s += "Removed scheduled spiders:
    • %s
    " % "
  • ".join(args["remove_pending_spiders"]) s += "

    " - if "add_pending_domains" in args: - for domain in args["add_pending_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl_domain(domain) + if "add_pending_spiders" in args: + for name in args["add_pending_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) s += "

    " - s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_domains"]) + s += "Scheduled spiders:
    • %s
    " % "
  • ".join(args["add_pending_spiders"]) s += "

    " - if "rerun_finished_domains" in args: - for domain in args["rerun_finished_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl_domain(domain) - self.finished.remove(domain) + if "rerun_finished_spiders" in args: + for name in args["rerun_finished_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) + self.finished.remove(name) s += "

    " - s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) + s += "Re-scheduled finished spiders:
    • %s
    " % "
  • ".join(args["rerun_finished_spiders"]) s += "

    " return s def webconsole_discover_module(self): - self.enabled_domains = spiders.list() + self.enabled_spiders = spiders.list() return self diff --git a/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py index 837cbcf81..e8c707f64 100644 --- a/scrapy/contrib/webconsole/stats.py +++ b/scrapy/contrib/webconsole/stats.py @@ -23,7 +23,7 @@ class StatsDump(object): s += "

    Global stats

    \n" s += stats_html_table(stats.get_stats()) for spider, spider_stats in stats.iter_spider_stats(): - s += "

    %s

    \n" % spider.domain_name + s += "

    %s

    \n" % spider.name s += stats_html_table(spider_stats) s += "\n" s += "\n" diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index dc6bba782..b1dc3754f 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -54,12 +54,12 @@ class ExecutionManager(object): if spider: scrapyengine.crawl(request, spider) - def crawl_domain(self, domain): - """Schedule given domain for crawling.""" + def crawl_spider_name(self, name): + """Schedule given spider by name for crawling.""" try: - spider = spiders.create(domain) + spider = spiders.create(name) except KeyError: - log.msg('Could not find spider for domain: %s' % domain, log.ERROR) + log.msg('Could not find spider: %s' % name, log.ERROR) else: self.crawl_spider(spider) diff --git a/scrapy/log.py b/scrapy/log.py index c4f9287e5..c8e969928 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -75,7 +75,7 @@ def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): "use 'spider' argument instead", DeprecationWarning, stacklevel=2) dispatcher.send(signal=logmessage_received, message=message, level=level, \ spider=spider) - system = domain or (spider.domain_name if spider else component) + system = domain or (spider.name if spider else component) msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding) log.msg(msg_txt, system=system) @@ -93,7 +93,7 @@ def err(_stuff=None, _why=None, **kwargs): import warnings warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \ "use 'spider' argument instead", DeprecationWarning, stacklevel=2) - kwargs['system'] = domain or (spider.domain_name if spider else component) + kwargs['system'] = domain or (spider.name if spider else component) if _why: _why = unicode_to_str("ERROR: %s" % _why, log_encoding) log.err(_stuff, _why, **kwargs) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index a9b64995e..757aaff55 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -3,6 +3,9 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + +import warnings + from zope.interface import Interface, Attribute, invariant, implements from twisted.plugin import IPlugin @@ -11,17 +14,9 @@ from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref -def _valid_domain_name(obj): - """Check the domain name specified is valid""" - if not obj.domain_name: - raise ValueError("Spider 'domain_name' attribute is required") - class ISpider(Interface, IPlugin) : - """Interface to be implemented by site-specific web spiders""" - - domain_name = Attribute("The domain name of the site to be scraped.") - - invariant(_valid_domain_name) + """Interface used by TwistedPluginSpiderManager to discover spiders""" + pass class BaseSpider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -31,19 +26,33 @@ class BaseSpider(object_ref): implements(ISpider) # XXX: class attributes kept for backwards compatibility - domain_name = None + name = None start_urls = [] - extra_domain_names = [] + allowed_domains = [] - def __init__(self, domain_name=None): - if domain_name is not None: - self.domain_name = domain_name + def __init__(self, name=None): + # XXX: SEP-12 backward compatibility (remove for 0.10) + if hasattr(self, 'domain_name'): + warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead", \ + DeprecationWarning, stacklevel=4) + self.name = self.domain_name + if hasattr(self, 'extra_domain_names'): + warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \ + DeprecationWarning, stacklevel=4) + self.allowed_domains = [self.name] + list(self.extra_domain_names) + + if name is not None: + self.name = name # XXX: create instance attributes (class attributes were kept for # backwards compatibility) if not self.start_urls: self.start_urls = [] - if not self.extra_domain_names: - self.extra_domain_names = [] + if not self.allowed_domains: + self.allowed_domains = [] + if not getattr(self, 'domain_name', None): + self.domain_name = self.name + if not getattr(self, 'extra_domain_names', None): + self.extra_domain_names = self.allowed_domains def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this @@ -67,6 +76,6 @@ class BaseSpider(object_ref): pass def __str__(self): - return "<%s %r>" % (type(self).__name__, self.domain_name) + return "<%s %r>" % (type(self).__name__, self.name) __repr__ = __str__ diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index f41916818..0acf36073 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -76,11 +76,11 @@ class MemoryStatsCollector(StatsCollector): def __init__(self): super(MemoryStatsCollector, self).__init__() - self.domain_stats = {} + self.spider_stats = {} def _persist_stats(self, stats, spider=None): if spider is not None: - self.domain_stats[spider.domain_name] = stats + self.spider_stats[spider.name] = stats class DummyStatsCollector(StatsCollector): diff --git a/scrapy/stats/collector/simpledb.py b/scrapy/stats/collector/simpledb.py index 850558442..d521c5e40 100644 --- a/scrapy/stats/collector/simpledb.py +++ b/scrapy/stats/collector/simpledb.py @@ -36,9 +36,9 @@ class SimpledbStatsCollector(StatsCollector): def _persist_to_sdb(self, spider, stats): ts = self._get_timestamp(spider).isoformat() - sdb_item_id = "%s_%s" % (spider.domain_name, ts) + sdb_item_id = "%s_%s" % (spider.name, ts) sdb_item = dict((k, self._to_sdb_value(v, k)) for k, v in stats.iteritems()) - sdb_item['domain'] = spider.domain_name + sdb_item['spider'] = spider.name sdb_item['timestamp'] = self._to_sdb_value(ts) connect_sdb().put_attributes(self._sdbdomain, sdb_item_id, sdb_item) diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 2e3baf992..246015466 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,9 +1,10 @@ from scrapy.spider import BaseSpider class $classname(BaseSpider): - domain_name = "$site" + name = "$name" + allowed_domains = ["$domain"] start_urls = ( - 'http://www.$site/', + 'http://www.$domain/', ) def parse(self, response): diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 18fd0ec42..578779c06 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -6,8 +6,9 @@ from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item class $classname(CrawlSpider): - domain_name = '$site' - start_urls = ['http://www.$site/'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/'] rules = ( Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), @@ -16,7 +17,7 @@ class $classname(CrawlSpider): def parse_item(self, response): hxs = HtmlXPathSelector(response) i = ${ProjectName}Item() - #i['site_id'] = hxs.select('//input[@id="sid"]/@value').extract() + #i['domain_id'] = hxs.select('//input[@id="sid"]/@value').extract() #i['name'] = hxs.select('//div[@id="name"]').extract() #i['description'] = hxs.select('//div[@id="description"]').extract() return i diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 794288570..c9a723000 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item class $classname(CSVFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.csv'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.csv'] # headers = ['id', 'name', 'description', 'image_link'] # delimiter = '\t' diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index f249b4537..f5ecbd707 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item class $classname(XMLFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.xml'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.xml'] def parse_item(self, response, selector): i = ${ProjectName}Item() diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index cb4e5ced1..6eed6d22a 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -59,10 +59,18 @@ class CommandTest(ProjectTest): class GenspiderCommandTest(CommandTest): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + self.assertEqual(0, self.call('genspider', 'test_name')) + assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + # pass two arguments . spider script should be created + self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) + assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + def test_template_default(self, *args): - self.assertEqual(0, self.call('genspider', 'testspider', 'test.com', *args)) - assert exists(join(self.proj_mod_path, 'spiders', 'testspider.py')) - self.assertEqual(1, self.call('genspider', 'otherspider', 'test.com')) + self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args)) + assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com')) def test_template_basic(self): self.test_template_default('--template=basic') diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 873e0eb30..2d7b5950c 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -22,8 +22,8 @@ class TestItem(Item): price = Field() class TestSpider(BaseSpider): - domain_name = "scrapytest.org" - extra_domain_names = ["localhost"] + name = "scrapytest.org" + allowed_domains = ["scrapytest.org", "localhost"] start_urls = ['http://localhost'] itemurl_re = re.compile("item\d+.html") @@ -68,7 +68,7 @@ def start_test_site(): class CrawlingSession(object): def __init__(self): - self.domain = 'scrapytest.org' + self.name = 'scrapytest.org' self.spider = None self.respplug = [] self.reqplug = [] @@ -139,7 +139,7 @@ class EngineTest(unittest.TestCase): Check the spider is loaded and located properly via the SpiderLocator """ assert session.spider is not None - self.assertEqual(session.spider.domain_name, session.domain) + self.assertEqual(session.spider.name, session.name) def test_visited_urls(self): """ diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py new file mode 100644 index 000000000..507de4abb --- /dev/null +++ b/scrapy/tests/test_spider.py @@ -0,0 +1,56 @@ +from __future__ import with_statement + +import sys +import warnings + +from twisted.trial import unittest + +from scrapy.spider import BaseSpider +from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter + +class OldSpider(BaseSpider): + + domain_name = 'example.com' + extra_domain_names = ('example.org', 'example.net') + +class NewSpider(BaseSpider): + + name = 'example.com' + allowed_domains = ('example.org', 'example.net') + + +class SpiderTest(unittest.TestCase): + + def setUp(self): + warnings.simplefilter("always") + + def tearDown(self): + warnings.resetwarnings() + + def test_sep12_deprecation_warnings(self): + if sys.version_info[:2] < (2, 6): + # warnings.catch_warnings() was added in Python 2.6 + raise unittest.SkipTest("This test requires Python 2.6+") + with warnings.catch_warnings(record=True) as w: + spider = OldSpider() + self.assertEqual(len(w), 2) # one for domain_name & one for extra_domain_names + self.assert_(issubclass(w[-1].category, DeprecationWarning)) + + def test_sep12_backwards_compatibility(self): + spider = OldSpider() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains) + + spider = NewSpider() + self.assertEqual(spider.domain_name, 'example.com') + self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names) + self.assert_('example.net' in spider.extra_domain_names, spider.extra_domain_names) + + def test_base_spider(self): + spider = BaseSpider("example.com") + self.assertEqual(spider.name, 'example.com') + self.assertEqual(spider.start_urls, []) + self.assertEqual(spider.allowed_domains, []) + diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 6c71213aa..cc40c8b20 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -9,8 +9,8 @@ class TestOffsiteMiddleware(TestCase): def setUp(self): self.spider = BaseSpider() - self.spider.domain_name = 'scrapytest.org' - self.spider.extra_domain_names = ['scrapy.org'] + self.spider.name = 'scrapytest.org' + self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] self.mw = OffsiteMiddleware() self.mw.spider_opened(self.spider) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 7496328fc..7ce27636d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,9 +22,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - domains = [spider.domain_name] - domains.extend(spider.extra_domain_names) - return url_is_from_any_domain(url, domains) + return url_is_from_any_domain(url, spider.allowed_domains) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref From c99e1af766bcf463c1bd1b6dcfd5bb3a5e8e9d39 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 5 Apr 2010 11:27:19 -0300 Subject: [PATCH 37/68] Added support for passing generic arguments to spider constructors (refs #152), extended Spider tests, added unittests for TwistedPluginSpiderManager --- scrapy/contrib/spidermanager.py | 22 +++---- scrapy/contrib/spiders/crawl.py | 4 +- scrapy/contrib/spiders/init.py | 4 +- scrapy/spider/models.py | 5 +- .../test_contrib_spidermanager/__init__.py | 39 +++++++++++++ .../test_contrib_spidermanager/spider1.py | 7 +++ .../test_contrib_spidermanager/spider2.py | 7 +++ scrapy/tests/test_spider.py | 58 +++++++++++++++---- 8 files changed, 117 insertions(+), 29 deletions(-) create mode 100644 scrapy/tests/test_contrib_spidermanager/__init__.py create mode 100644 scrapy/tests/test_contrib_spidermanager/spider1.py create mode 100644 scrapy/tests/test_contrib_spidermanager/spider2.py diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 331f55aa5..1c41b625d 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -4,7 +4,6 @@ spiders """ import sys -import urlparse from twisted.plugin import getCache from twisted.python.rebuild import rebuild @@ -21,21 +20,18 @@ class TwistedPluginSpiderManager(object): self.loaded = False self._spiders = {} - def create(self, spider_id): + def create(self, spider_name, **spider_kwargs): + """Returns a Spider instance for the given spider name, using the given + spider arguments. If the sipder name is not found, it raises a + KeyError. """ - Returns Spider instance by given identifier. - If not exists raises KeyError. - """ - #@@@ currently spider_id = domain - # if lookup fails let dict's KeyError exception propagate - return self._spiders[spider_id] + spider = self._spiders[spider_name] + spider.__dict__.update(spider_kwargs) + return spider def find_by_request(self, request): - """ - Returns list of spiders ids that match given Request. - """ - # just find by request.url - return [domain for domain, spider in self._spiders.iteritems() + """Returns list of spiders names that match the given Request""" + return [name for name, spider in self._spiders.iteritems() if url_is_from_spider(request.url, spider)] def list(self): diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 648765b06..2e74dab4e 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -59,9 +59,9 @@ class CrawlSpider(InitSpider): """ rules = () - def __init__(self): + def __init__(self, *a, **kw): """Constructor takes care of compiling rules""" - super(CrawlSpider, self).__init__() + super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() def parse(self, response): diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py index b37591ca5..f759fd81f 100644 --- a/scrapy/contrib/spiders/init.py +++ b/scrapy/contrib/spiders/init.py @@ -3,8 +3,8 @@ from scrapy.spider import BaseSpider class InitSpider(BaseSpider): """Base Spider with initialization facilities""" - def __init__(self): - super(InitSpider, self).__init__() + def __init__(self, *a, **kw): + super(InitSpider, self).__init__(*a, **kw) self._postinit_reqs = [] self._init_complete = False self._init_started = False diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index 757aaff55..3adf7a470 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -30,7 +30,8 @@ class BaseSpider(object_ref): start_urls = [] allowed_domains = [] - def __init__(self, name=None): + def __init__(self, name=None, **kwargs): + self.__dict__.update(kwargs) # XXX: SEP-12 backward compatibility (remove for 0.10) if hasattr(self, 'domain_name'): warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead", \ @@ -53,6 +54,8 @@ class BaseSpider(object_ref): self.domain_name = self.name if not getattr(self, 'extra_domain_names', None): self.extra_domain_names = self.allowed_domains + if not self.name: + raise ValueError("%s must have a name" % type(self).__name__) def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py new file mode 100644 index 000000000..d27c32263 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -0,0 +1,39 @@ +import unittest + +# just a hack to avoid cyclic imports of scrapy.spider when running this test +# alone +import scrapy.spider +from scrapy.contrib.spidermanager import TwistedPluginSpiderManager +from scrapy.http import Request + +class TwistedPluginSpiderManagerTest(unittest.TestCase): + + def setUp(self): + self.spiderman = TwistedPluginSpiderManager() + assert not self.spiderman.loaded + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager']) + assert self.spiderman.loaded + + def test_list(self): + self.assertEqual(set(self.spiderman.list()), + set(['spider1', 'spider2'])) + + def test_create(self): + spider1 = self.spiderman.create("spider1") + self.assertEqual(spider1.__class__.__name__, 'Spider1') + spider2 = self.spiderman.create("spider2", foo="bar") + self.assertEqual(spider2.__class__.__name__, 'Spider2') + self.assertEqual(spider2.foo, 'bar') + + def test_find_by_request(self): + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), + ['spider1']) + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy2.org/test')), + ['spider2']) + self.assertEqual(set(self.spiderman.find_by_request(Request('http://scrapy3.org/test'))), + set(['spider1', 'spider2'])) + self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), + []) + +if __name__ == '__main__': + unittest.main() diff --git a/scrapy/tests/test_contrib_spidermanager/spider1.py b/scrapy/tests/test_contrib_spidermanager/spider1.py new file mode 100644 index 000000000..0a9b60989 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/spider1.py @@ -0,0 +1,7 @@ +from scrapy.spider import BaseSpider + +class Spider1(BaseSpider): + name = "spider1" + allowed_domains = ["scrapy1.org", "scrapy3.org"] + +SPIDER = Spider1() diff --git a/scrapy/tests/test_contrib_spidermanager/spider2.py b/scrapy/tests/test_contrib_spidermanager/spider2.py new file mode 100644 index 000000000..52023277f --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/spider2.py @@ -0,0 +1,7 @@ +from scrapy.spider import BaseSpider + +class Spider2(BaseSpider): + name = "spider2" + allowed_domains = ["scrapy2.org", "scrapy3.org"] + +SPIDER = Spider2() diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 507de4abb..c488dad3b 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -6,20 +6,25 @@ import warnings from twisted.trial import unittest from scrapy.spider import BaseSpider +from scrapy.contrib.spiders.init import InitSpider +from scrapy.contrib.spiders.crawl import CrawlSpider +from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter -class OldSpider(BaseSpider): +class BaseSpiderTest(unittest.TestCase): - domain_name = 'example.com' - extra_domain_names = ('example.org', 'example.net') + spider_class = BaseSpider -class NewSpider(BaseSpider): + class OldSpider(spider_class): - name = 'example.com' - allowed_domains = ('example.org', 'example.net') + domain_name = 'example.com' + extra_domain_names = ('example.org', 'example.net') + class NewSpider(spider_class): + + name = 'example.com' + allowed_domains = ('example.org', 'example.net') -class SpiderTest(unittest.TestCase): def setUp(self): warnings.simplefilter("always") @@ -32,25 +37,56 @@ class SpiderTest(unittest.TestCase): # warnings.catch_warnings() was added in Python 2.6 raise unittest.SkipTest("This test requires Python 2.6+") with warnings.catch_warnings(record=True) as w: - spider = OldSpider() + spider = self.OldSpider() self.assertEqual(len(w), 2) # one for domain_name & one for extra_domain_names self.assert_(issubclass(w[-1].category, DeprecationWarning)) def test_sep12_backwards_compatibility(self): - spider = OldSpider() + spider = self.OldSpider() self.assertEqual(spider.name, 'example.com') self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains) self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains) - spider = NewSpider() + spider = self.NewSpider() self.assertEqual(spider.domain_name, 'example.com') self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names) self.assert_('example.net' in spider.extra_domain_names, spider.extra_domain_names) def test_base_spider(self): - spider = BaseSpider("example.com") + spider = self.spider_class("example.com") self.assertEqual(spider.name, 'example.com') self.assertEqual(spider.start_urls, []) self.assertEqual(spider.allowed_domains, []) + def test_spider_args(self): + """Constructor arguments are assigned to spider attributes""" + spider = self.spider_class('example.com', foo='bar') + self.assertEqual(spider.foo, 'bar') + + def test_spider_without_name(self): + """Constructor arguments are assigned to spider attributes""" + spider = self.spider_class('example.com') + self.assertRaises(ValueError, self.spider_class) + self.assertRaises(ValueError, self.spider_class, somearg='foo') + + +class InitSpiderTest(BaseSpiderTest): + + spider_class = InitSpider + +class XMLFeedSpiderTest(BaseSpiderTest): + + spider_class = XMLFeedSpider + +class CSVFeedSpiderTest(BaseSpiderTest): + + spider_class = CSVFeedSpider + +class CrawlSpiderTest(BaseSpiderTest): + + spider_class = CrawlSpider + + +if __name__ == '__main__': + unittest.main() From 77a4d9aba9dcd137fc772be89bcb3ecab05e95ad Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 5 Apr 2010 11:53:22 -0300 Subject: [PATCH 38/68] use a default name for spiders constructed without names --- scrapy/command/commands/fetch.py | 2 +- scrapy/shell.py | 2 +- scrapy/spider/models.py | 2 +- scrapy/tests/test_spider.py | 7 ++++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index bb2df1400..b394b5e76 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -41,7 +41,7 @@ class Command(ScrapyCommand): log.msg("Could not find spider: %s" % opts.spider, log.ERROR) else: spider = scrapymanager._create_spider_for_request(request, \ - BaseSpider('default')) + BaseSpider()) scrapymanager.crawl_request(request, spider) scrapymanager.start() diff --git a/scrapy/shell.py b/scrapy/shell.py index 96dc51270..88d2b1710 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -55,7 +55,7 @@ class Shell(object): request = Request(url) spider = scrapymanager._create_spider_for_request(request, \ - BaseSpider('default'), log_multiple=True) + BaseSpider(), log_multiple=True) print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index 3adf7a470..094cd5077 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -55,7 +55,7 @@ class BaseSpider(object_ref): if not getattr(self, 'extra_domain_names', None): self.extra_domain_names = self.allowed_domains if not self.name: - raise ValueError("%s must have a name" % type(self).__name__) + self.name = 'default' def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index c488dad3b..7d93db5b5 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -66,9 +66,10 @@ class BaseSpiderTest(unittest.TestCase): def test_spider_without_name(self): """Constructor arguments are assigned to spider attributes""" - spider = self.spider_class('example.com') - self.assertRaises(ValueError, self.spider_class) - self.assertRaises(ValueError, self.spider_class, somearg='foo') + spider = self.spider_class() + self.assertEqual(spider.name, 'default') + spider = self.spider_class(foo='bar') + self.assertEqual(spider.foo, 'bar') class InitSpiderTest(BaseSpiderTest): From 70ac6642d57a4400a62d2373dc2de36099cfe1d1 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 5 Apr 2010 12:09:43 -0300 Subject: [PATCH 39/68] SEP-012: bugfix backward compatibility of Spider.domain_name and Spider.extra_domain_names --HG-- extra : rebase_source : 66f779cddc6854092951078d443dbf9113f7576a --- scrapy/spider/models.py | 19 ++++++++++--------- scrapy/tests/test_spider.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index 094cd5077..615e920a6 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -34,13 +34,14 @@ class BaseSpider(object_ref): self.__dict__.update(kwargs) # XXX: SEP-12 backward compatibility (remove for 0.10) if hasattr(self, 'domain_name'): - warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead", \ + warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead and Spider.allowed_domains", \ DeprecationWarning, stacklevel=4) self.name = self.domain_name - if hasattr(self, 'extra_domain_names'): - warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \ - DeprecationWarning, stacklevel=4) - self.allowed_domains = [self.name] + list(self.extra_domain_names) + self.allowed_domains = [self.name] + if hasattr(self, 'extra_domain_names'): + warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \ + DeprecationWarning, stacklevel=4) + self.allowed_domains += list(self.extra_domain_names) if name is not None: self.name = name @@ -50,13 +51,13 @@ class BaseSpider(object_ref): self.start_urls = [] if not self.allowed_domains: self.allowed_domains = [] - if not getattr(self, 'domain_name', None): - self.domain_name = self.name - if not getattr(self, 'extra_domain_names', None): - self.extra_domain_names = self.allowed_domains if not self.name: self.name = 'default' + # XXX: SEP-12 forward compatibility (remove for 0.10) + self.domain_name = self.name + self.extra_domain_names = self.allowed_domains + def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this method to send log messages from your spider diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 7d93db5b5..9a618e720 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -11,6 +11,7 @@ from scrapy.contrib.spiders.crawl import CrawlSpider from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter + class BaseSpiderTest(unittest.TestCase): spider_class = BaseSpider @@ -20,6 +21,12 @@ class BaseSpiderTest(unittest.TestCase): domain_name = 'example.com' extra_domain_names = ('example.org', 'example.net') + + class OldSpiderWithoutExtradomains(spider_class): + + domain_name = 'example.com' + + class NewSpider(spider_class): name = 'example.com' @@ -48,6 +55,10 @@ class BaseSpiderTest(unittest.TestCase): self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains) self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains) + spider = self.OldSpiderWithoutExtradomains() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + spider = self.NewSpider() self.assertEqual(spider.domain_name, 'example.com') self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names) From 0dfec04439334b4c2da36b154b9df4703913faaa Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 5 Apr 2010 12:34:29 -0300 Subject: [PATCH 40/68] made Spider name required again (do not default) --- scrapy/command/commands/fetch.py | 2 +- scrapy/contrib_exp/crawlspider/spider.py | 4 ++-- scrapy/shell.py | 2 +- scrapy/spider/models.py | 2 +- .../test_contrib_exp_crawlspider_rules.py | 12 +++++----- .../test_contrib_exp_crawlspider_spider.py | 2 +- scrapy/tests/test_downloader_handlers.py | 24 +++++++++---------- .../test_downloadermiddleware_cookies.py | 2 +- ...test_downloadermiddleware_decompression.py | 2 +- ...est_downloadermiddleware_defaultheaders.py | 2 +- .../test_downloadermiddleware_httpauth.py | 2 +- ...st_downloadermiddleware_httpcompression.py | 2 +- .../test_downloadermiddleware_httpproxy.py | 2 +- .../test_downloadermiddleware_redirect.py | 2 +- .../tests/test_downloadermiddleware_retry.py | 2 +- .../test_downloadermiddleware_useragent.py | 2 +- scrapy/tests/test_dupefilter.py | 4 ++-- ...st_schedulermiddleware_duplicatesfilter.py | 2 +- scrapy/tests/test_spider.py | 7 ++---- .../tests/test_spidermiddleware_httperror.py | 2 +- scrapy/tests/test_spidermiddleware_offsite.py | 2 +- scrapy/tests/test_spidermiddleware_referer.py | 2 +- .../tests/test_spidermiddleware_urlfilter.py | 2 +- .../tests/test_spidermiddleware_urllength.py | 2 +- scrapy/tests/test_stats.py | 2 +- 25 files changed, 44 insertions(+), 47 deletions(-) diff --git a/scrapy/command/commands/fetch.py b/scrapy/command/commands/fetch.py index b394b5e76..bb2df1400 100644 --- a/scrapy/command/commands/fetch.py +++ b/scrapy/command/commands/fetch.py @@ -41,7 +41,7 @@ class Command(ScrapyCommand): log.msg("Could not find spider: %s" % opts.spider, log.ERROR) else: spider = scrapymanager._create_spider_for_request(request, \ - BaseSpider()) + BaseSpider('default')) scrapymanager.crawl_request(request, spider) scrapymanager.start() diff --git a/scrapy/contrib_exp/crawlspider/spider.py b/scrapy/contrib_exp/crawlspider/spider.py index 730ad0e8d..e2bf1a34b 100644 --- a/scrapy/contrib_exp/crawlspider/spider.py +++ b/scrapy/contrib_exp/crawlspider/spider.py @@ -15,9 +15,9 @@ class CrawlSpider(BaseSpider): request_processors = None rules = [] - def __init__(self): + def __init__(self, *a, **kw): """Initialize dispatcher""" - super(CrawlSpider, self).__init__() + super(CrawlSpider, self).__init__(*a, **kw) # auto follow start urls if self.start_urls: diff --git a/scrapy/shell.py b/scrapy/shell.py index 88d2b1710..96dc51270 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -55,7 +55,7 @@ class Shell(object): request = Request(url) spider = scrapymanager._create_spider_for_request(request, \ - BaseSpider(), log_multiple=True) + BaseSpider('default'), log_multiple=True) print "Fetching %s..." % request response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index 615e920a6..c1ac2c1d4 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -52,7 +52,7 @@ class BaseSpider(object_ref): if not self.allowed_domains: self.allowed_domains = [] if not self.name: - self.name = 'default' + raise ValueError("%s must have a name" % type(self).__name__) # XXX: SEP-12 forward compatibility (remove for 0.10) self.domain_name = self.name diff --git a/scrapy/tests/test_contrib_exp_crawlspider_rules.py b/scrapy/tests/test_contrib_exp_crawlspider_rules.py index 0fbe52415..e04eb7b06 100644 --- a/scrapy/tests/test_contrib_exp_crawlspider_rules.py +++ b/scrapy/tests/test_contrib_exp_crawlspider_rules.py @@ -109,7 +109,7 @@ class CompiledRuleInitializationTest(unittest.TestCase): class RulesTest(unittest.TestCase): def test_rules_manager_basic(self): - spider = BaseSpider() + spider = BaseSpider('foo') response1 = HtmlResponse('http://example.org') response2 = HtmlResponse('http://othersite.org') rulesman = RulesManager([], spider) @@ -134,7 +134,7 @@ class RulesTest(unittest.TestCase): self.failUnlessEqual(rule1.follow, True) def test_rules_manager_empty_rule(self): - spider = BaseSpider() + spider = BaseSpider('foo') response = HtmlResponse('http://example.org') rulesman = RulesManager([Rule(follow=True)], spider) @@ -144,7 +144,7 @@ class RulesTest(unittest.TestCase): self.failUnless(isinstance(rule.matcher, BaseMatcher)) def test_rules_manager_default_matcher(self): - spider = BaseSpider() + spider = BaseSpider('foo') response = HtmlResponse('http://example.org') callback = lambda x: None @@ -156,7 +156,7 @@ class RulesTest(unittest.TestCase): self.failUnless(isinstance(rule.matcher, UrlMatcher)) def test_rules_manager_matchers(self): - spider = BaseSpider() + spider = BaseSpider('foo') response1 = HtmlResponse('http://example.org') response2 = HtmlResponse('http://othersite.org') @@ -200,7 +200,7 @@ class RulesTest(unittest.TestCase): def test_rules_manager_callbacks(self): mycallback = lambda: True - spider = BaseSpider() + spider = BaseSpider('foo') spider.parse_item = lambda: True response1 = HtmlResponse('http://example.org') @@ -229,7 +229,7 @@ class RulesTest(unittest.TestCase): def test_rules_manager_callback_with_arguments(self): - spider = BaseSpider() + spider = BaseSpider('foo') response = HtmlResponse('http://example.org') kwargs = {'a': 1} diff --git a/scrapy/tests/test_contrib_exp_crawlspider_spider.py b/scrapy/tests/test_contrib_exp_crawlspider_spider.py index 5e067508a..21f7f71e4 100644 --- a/scrapy/tests/test_contrib_exp_crawlspider_spider.py +++ b/scrapy/tests/test_contrib_exp_crawlspider_spider.py @@ -60,7 +60,7 @@ class CrawlSpiderTest(unittest.TestCase): Spider.request_extractors = extractors Spider.request_processors = processors - return Spider() + return Spider('foo') def test_start_url_auto_rule(self): spider = self.spider_factory() diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index a23d6c6a4..ae92e16da 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -32,11 +32,11 @@ class FileTestCase(unittest.TestCase): request = Request('file://%s' % self.tmpname + '^') assert request.url.upper().endswith('%5E') - return download_file(request, BaseSpider()).addCallback(_test) + return download_file(request, BaseSpider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = download_file(request, BaseSpider()) + d = download_file(request, BaseSpider('foo')) return self.assertFailure(d, IOError) @@ -66,20 +66,20 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_timeout_download_from_spider(self): - spider = BaseSpider() + spider = BaseSpider('foo') spider.download_timeout = 0.000001 request = Request(self.getURL('wait')) d = download_http(request, spider) @@ -91,7 +91,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -99,9 +99,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -109,14 +109,14 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d def test_broken_download(self): request = Request(self.getURL('broken')) - d = download_http(request, BaseSpider()) + d = download_http(request, BaseSpider('foo')) return self.assertFailure(d, PartialDownloadError) @@ -152,7 +152,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -161,4 +161,4 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return download_http(request, BaseSpider()).addCallback(_test) + return download_http(request, BaseSpider('foo')).addCallback(_test) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index d1b4275e8..a04efd79d 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -10,7 +10,7 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = CookiesMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index 1de53e85c..09dfdbaf8 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -21,7 +21,7 @@ class DecompressionMiddlewareTest(TestCase): def setUp(self): self.mw = DecompressionMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_known_compression_formats(self): for fmt in self.test_formats: diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index cd1624eb4..805289787 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -9,7 +9,7 @@ from scrapy.spider import BaseSpider class TestDefaultHeadersMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() self.default_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) diff --git a/scrapy/tests/test_downloadermiddleware_httpauth.py b/scrapy/tests/test_downloadermiddleware_httpauth.py index 79c815184..747ea9de1 100644 --- a/scrapy/tests/test_downloadermiddleware_httpauth.py +++ b/scrapy/tests/test_downloadermiddleware_httpauth.py @@ -18,7 +18,7 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): self.mw.default_useragent = 'default_useragent' - spider = TestSpider() + spider = TestSpider('foo') req = Request('http://scrapytest.org/') assert self.mw.process_request(req, spider) is None self.assertEquals(req.headers['Authorization'], 'Basic Zm9vOmJhcg==') diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 9490fb40f..c8fb38d22 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -20,7 +20,7 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpCompressionMiddleware() def _getresponse(self, coding): diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py index dd9602661..7f6f7ee4e 100644 --- a/scrapy/tests/test_downloadermiddleware_httpproxy.py +++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py @@ -8,7 +8,7 @@ from scrapy.http import Response, Request from scrapy.spider import BaseSpider from scrapy.conf import settings -spider = BaseSpider() +spider = BaseSpider('foo') class TestDefaultHeadersMiddleware(TestCase): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index f9a93b910..8409fec6d 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -8,7 +8,7 @@ from scrapy.http import Request, Response, HtmlResponse, Headers class RedirectMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RedirectMiddleware() def test_priority_adjust(self): diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index 101bb01c0..96e7cd297 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -10,7 +10,7 @@ from scrapy.http import Request, Response class RetryTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RetryMiddleware() self.mw.max_retry_times = 2 diff --git a/scrapy/tests/test_downloadermiddleware_useragent.py b/scrapy/tests/test_downloadermiddleware_useragent.py index d12f2bc63..338a63a23 100644 --- a/scrapy/tests/test_downloadermiddleware_useragent.py +++ b/scrapy/tests/test_downloadermiddleware_useragent.py @@ -9,7 +9,7 @@ from scrapy.conf import settings class UserAgentMiddlewareTest(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UserAgentMiddleware() def tearDown(self): diff --git a/scrapy/tests/test_dupefilter.py b/scrapy/tests/test_dupefilter.py index d11595a9d..fa9eef332 100644 --- a/scrapy/tests/test_dupefilter.py +++ b/scrapy/tests/test_dupefilter.py @@ -8,7 +8,7 @@ from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilt class RequestFingerprintDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = RequestFingerprintDupeFilter() filter.open_spider(spider) @@ -28,7 +28,7 @@ class RequestFingerprintDupeFilterTest(unittest.TestCase): class NullDupeFilterTest(unittest.TestCase): def test_filter(self): - spider = BaseSpider() + spider = BaseSpider('foo') filter = NullDupeFilter() filter.open_spider(spider) diff --git a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index db59de2d5..1384c2237 100644 --- a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -10,7 +10,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): def setUp(self): self.mw = DuplicatesFilterMiddleware() - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw.open_spider(self.spider) def tearDown(self): diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 9a618e720..14aa78168 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -9,7 +9,6 @@ from scrapy.spider import BaseSpider from scrapy.contrib.spiders.init import InitSpider from scrapy.contrib.spiders.crawl import CrawlSpider from scrapy.contrib.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter class BaseSpiderTest(unittest.TestCase): @@ -77,10 +76,8 @@ class BaseSpiderTest(unittest.TestCase): def test_spider_without_name(self): """Constructor arguments are assigned to spider attributes""" - spider = self.spider_class() - self.assertEqual(spider.name, 'default') - spider = self.spider_class(foo='bar') - self.assertEqual(spider.foo, 'bar') + self.assertRaises(ValueError, self.spider_class) + self.assertRaises(ValueError, self.spider_class, somearg='foo') class InitSpiderTest(BaseSpiderTest): diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index a0dfb492e..505b1a091 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware class TestHttpErrorMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = HttpErrorMiddleware() self.req = Request('http://scrapytest.org') diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index cc40c8b20..861595524 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware class TestOffsiteMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.spider.name = 'scrapytest.org' self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] diff --git a/scrapy/tests/test_spidermiddleware_referer.py b/scrapy/tests/test_spidermiddleware_referer.py index ec11b0489..467c301ba 100644 --- a/scrapy/tests/test_spidermiddleware_referer.py +++ b/scrapy/tests/test_spidermiddleware_referer.py @@ -8,7 +8,7 @@ from scrapy.contrib.spidermiddleware.referer import RefererMiddleware class TestRefererMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = RefererMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urlfilter.py b/scrapy/tests/test_spidermiddleware_urlfilter.py index 21f776f17..45621599a 100644 --- a/scrapy/tests/test_spidermiddleware_urlfilter.py +++ b/scrapy/tests/test_spidermiddleware_urlfilter.py @@ -9,7 +9,7 @@ from scrapy.utils.url import canonicalize_url class TestUrlFilterMiddleware(TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlFilterMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/scrapy/tests/test_spidermiddleware_urllength.py index de0e2eae0..cbd80d488 100644 --- a/scrapy/tests/test_spidermiddleware_urllength.py +++ b/scrapy/tests/test_spidermiddleware_urllength.py @@ -12,7 +12,7 @@ class TestUrlLengthMiddleware(TestCase): settings.disabled = False settings.overrides['URLLENGTH_LIMIT'] = 25 - self.spider = BaseSpider() + self.spider = BaseSpider('foo') self.mw = UrlLengthMiddleware() def test_process_spider_output(self): diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index 612ee0eaa..786021898 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -9,7 +9,7 @@ from scrapy.stats.signals import stats_spider_opened, stats_spider_closing, \ class StatsCollectorTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider() + self.spider = BaseSpider('foo') def test_collector(self): stats = StatsCollector() From 0b07742adb77e34982ebd9a5c9f32b8e8aad737d Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 5 Apr 2010 15:07:43 -0300 Subject: [PATCH 41/68] gb2312 and gbk encodings was superseded by gb18030 --- scrapy/conf/default_settings.py | 27 ++++++++++++++++++++++----- scrapy/tests/test_http_response.py | 4 ++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 5e7bea3cd..b80e89385 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -74,18 +74,35 @@ DUPEFILTER_CLASS = 'scrapy.contrib.dupefilter.RequestFingerprintDupeFilter' ENCODING_ALIASES = {} ENCODING_ALIASES_BASE = { - 'zh-cn': 'gb2312', - 'win-1251': 'cp1251', - 'macintosh' : 'mac-roman', - 'x-sjis': 'shift-jis', + # gb2312 is superseded by gb18030 + 'gb2312': 'gb18030', + 'chinese': 'gb18030', + 'csiso58gb231280': 'gb18030', + 'euc- cn': 'gb18030', + 'euccn': 'gb18030', + 'eucgb2312-cn': 'gb18030', + 'gb2312-1980': 'gb18030', + 'gb2312-80': 'gb18030', + 'iso- ir-58': 'gb18030', + # gbk is superseded by gb18030 + 'gbk': 'gb18030', + '936': 'gb18030', + 'cp936': 'gb18030', + 'ms936': 'gb18030', + # latin_1 is a subset of cp1252 + 'latin_1': 'cp1252', 'iso-8859-1': 'cp1252', 'iso8859-1': 'cp1252', '8859': 'cp1252', 'cp819': 'cp1252', 'latin': 'cp1252', 'latin1': 'cp1252', - 'latin_1': 'cp1252', 'l1': 'cp1252', + # others + 'zh-cn': 'gb18030', + 'win-1251': 'cp1251', + 'macintosh' : 'mac_roman', + 'x-sjis': 'shift_jis', } EXTENSIONS = {} diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index d17d5d272..87138219b 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -179,6 +179,8 @@ class TextResponseTest(BaseResponseTest): r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3") r4 = self.response_class("http://www.example.com", body="\xa2\xa3") r5 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=None"]}, body="\xc2\xa3") + r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gb2312"]}, body="\xa8D") + r7 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=gbk"]}, body="\xa8D") self.assertEqual(r1._headers_encoding(), "utf-8") self.assertEqual(r2._headers_encoding(), None) @@ -193,6 +195,8 @@ class TextResponseTest(BaseResponseTest): self._assert_response_values(r1, 'utf-8', u"\xa3") self._assert_response_values(r2, 'utf-8', u"\xa3") self._assert_response_values(r3, 'iso-8859-1', u"\xa3") + self._assert_response_values(r6, 'gb18030', u"\u2015") + self._assert_response_values(r7, 'gb18030', u"\u2015") # TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3") From 2cd591e8a7f99970abb25bef373fd2c1a2ff5775 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 6 Apr 2010 07:22:50 +0100 Subject: [PATCH 42/68] add missing dropin.cache file required by default spidermanager tests --- .../test_contrib_spidermanager/dropin.cache | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 scrapy/tests/test_contrib_spidermanager/dropin.cache diff --git a/scrapy/tests/test_contrib_spidermanager/dropin.cache b/scrapy/tests/test_contrib_spidermanager/dropin.cache new file mode 100644 index 000000000..81239e0b4 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/dropin.cache @@ -0,0 +1,71 @@ +(dp1 +S'spider2' +p2 +ccopy_reg +_reconstructor +p3 +(ctwisted.plugin +CachedDropin +p4 +c__builtin__ +object +p5 +NtRp6 +(dp7 +S'moduleName' +p8 +S'scrapy.tests.test_contrib_spidermanager.spider2' +p9 +sS'description' +p10 +NsS'plugins' +p11 +(lp12 +g3 +(ctwisted.plugin +CachedPlugin +p13 +g5 +NtRp14 +(dp15 +S'provided' +p16 +(lp17 +cscrapy.spider.models +ISpider +p18 +asS'dropin' +p19 +g6 +sS'name' +p20 +S'SPIDER' +p21 +sg10 +NsbasbsS'spider1' +p22 +g3 +(g4 +g5 +NtRp23 +(dp24 +g8 +S'scrapy.tests.test_contrib_spidermanager.spider1' +p25 +sg10 +Nsg11 +(lp26 +g3 +(g13 +g5 +NtRp27 +(dp28 +g16 +(lp29 +g18 +asg19 +g23 +sg20 +g21 +sg10 +Nsbasbs. \ No newline at end of file From 3fcd69c3476e88e0eed2285327ce89f5f5c9a412 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 6 Apr 2010 10:55:21 -0300 Subject: [PATCH 43/68] added a couple additional TwistedPluginSpiderManager tests --- .../test_contrib_spidermanager/__init__.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py index d27c32263..5b0d9e954 100644 --- a/scrapy/tests/test_contrib_spidermanager/__init__.py +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -1,3 +1,4 @@ +import weakref import unittest # just a hack to avoid cyclic imports of scrapy.spider when running this test @@ -25,6 +26,14 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.assertEqual(spider2.__class__.__name__, 'Spider2') self.assertEqual(spider2.foo, 'bar') + def test_create_uses_cache(self): + # TwistedPluginSpiderManager uses an internal cache which is + # invalidated in close_spider() but this isn't necessarily the best + # thing to do in all cases. + spider1 = self.spiderman.create("spider1") + spider2 = self.spiderman.create("spider1") + assert spider1 is spider2 + def test_find_by_request(self): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) @@ -35,5 +44,19 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), []) + def test_close_spider_remove_refs(self): + spider = self.spiderman.create("spider1") + wref = weakref.ref(spider) + assert wref() + self.spiderman.close_spider(spider) + del spider + assert not wref() + + def test_close_spider_invalidates_cache(self): + spider1 = self.spiderman.create("spider1") + self.spiderman.close_spider(spider1) + spider2 = self.spiderman.create("spider1") + assert spider1 is not spider2 + if __name__ == '__main__': unittest.main() From 8b86e1d008b84eee6fe4d2b73a39bf904b79b68a Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 7 Apr 2010 00:29:53 -0300 Subject: [PATCH 44/68] Minimize effect of http://bugs.python.org/issue8271 on TextResponses by changing str.decode errors policy by custom `replace` alike error handler --- scrapy/http/response/text.py | 11 ++++++++--- scrapy/tests/test_http_response.py | 9 ++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index c294ade63..59f44fb56 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -6,14 +6,19 @@ See documentation in docs/topics/request-response.rst """ import re - +import codecs from scrapy.xlib.BeautifulSoup import UnicodeDammit - from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs from scrapy.utils.encoding import encoding_exists, resolve_encoding from scrapy.conf import settings + +# Python decoder doesn't follow unicode standard when handling +# bad utf-8 encoded strings. see http://bugs.python.org/issue8271 +codecs.register_error('scrapy_replace', lambda exc: (u'\ufffd', exc.start+1)) + + class TextResponse(Response): _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING'] @@ -77,7 +82,7 @@ class TextResponse(Response): def body_as_unicode(self): """Return body as unicode""" if self._cached_ubody is None: - self._cached_ubody = self.body.decode(self.encoding, 'replace') + self._cached_ubody = self.body.decode(self.encoding, 'scrapy_replace') return self._cached_ubody @memoizemethod_noargs diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 87138219b..437d75b98 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -215,7 +215,7 @@ class TextResponseTest(BaseResponseTest): def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): r6 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.body_as_unicode(), u'\ufeffWORD\ufffd') + self.assertEqual(r6.body_as_unicode(), u'\ufeffWORD\ufffd\ufffd') def test_replace_wrong_encoding(self): """Test invalid chars are replaced properly""" @@ -223,6 +223,13 @@ class TextResponseTest(BaseResponseTest): # XXX: Policy for replacing invalid chars may suffer minor variations # but it should always contain the unicode replacement char (u'\ufffd') assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode()) + + # Do not destroy html tags due to encoding bugs + r = self.response_class("http://example.com", encoding='utf-8', \ + body='\xf0value') + assert u'value' in r.body_as_unicode(), repr(r.body_as_unicode()) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') From 68a875edb029b289d1166cac1898951ad7044722 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Wed, 7 Apr 2010 10:54:54 -0300 Subject: [PATCH 45/68] update ENCODING_ALIASES setting default value in settings documentation topic --- docs/topics/settings.rst | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index eacc2107c..e64ee329b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -459,18 +459,35 @@ ENCODING_ALIASES_BASE Default:: { - 'gb2312': 'zh-cn', - 'cp1251': 'win-1251', - 'macintosh' : 'mac-roman', - 'x-sjis': 'shift-jis', + # gb2312 is superseded by gb18030 + 'gb2312': 'gb18030', + 'chinese': 'gb18030', + 'csiso58gb231280': 'gb18030', + 'euc- cn': 'gb18030', + 'euccn': 'gb18030', + 'eucgb2312-cn': 'gb18030', + 'gb2312-1980': 'gb18030', + 'gb2312-80': 'gb18030', + 'iso- ir-58': 'gb18030', + # gbk is superseded by gb18030 + 'gbk': 'gb18030', + '936': 'gb18030', + 'cp936': 'gb18030', + 'ms936': 'gb18030', + # latin_1 is a subset of cp1252 + 'latin_1': 'cp1252', 'iso-8859-1': 'cp1252', 'iso8859-1': 'cp1252', '8859': 'cp1252', 'cp819': 'cp1252', 'latin': 'cp1252', 'latin1': 'cp1252', - 'latin_1': 'cp1252', 'l1': 'cp1252', + # others + 'zh-cn': 'gb18030', + 'win-1251': 'cp1251', + 'macintosh' : 'mac_roman', + 'x-sjis': 'shift_jis', } The default encoding aliases defined in Scrapy. Don't override this setting in From 0dbb5d44ae8f841d60a34ce507ca54ca8cf49a23 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 9 Apr 2010 14:16:00 -0300 Subject: [PATCH 46/68] images: avoid signing images based on spider name or request hostname, use request.meta instead --- scrapy/contrib/aws.py | 13 +++++-------- scrapy/contrib/pipeline/images.py | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 91b134a0e..b62c9e6b3 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -5,13 +5,13 @@ because Amazon Web Service use timestamps for authentication. """ import os -import time - -from scrapy.utils.httpobj import urlparse_cached +from time import strftime, gmtime from scrapy.utils.aws import sign_request from scrapy.conf import settings + class AWSMiddleware(object): + def __init__(self): self.access_key = settings['AWS_ACCESS_KEY_ID'] or \ os.environ.get('AWS_ACCESS_KEY_ID') @@ -19,9 +19,6 @@ class AWSMiddleware(object): os.environ.get('AWS_SECRET_ACCESS_KEY') def process_request(self, request, spider): - hostname = urlparse_cached(request).hostname - if spider.name == 's3.amazonaws.com' \ - or (hostname and hostname.endswith('s3.amazonaws.com')): - request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \ - time.gmtime()) + if request.meta.get('sign_s3_request'): + request.headers['Date'] = strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime()) sign_request(request, self.access_key, self.secret_key) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index a56b3da0b..640cc7179 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -143,7 +143,7 @@ class S3ImagesStore(object): def _build_request(self, key, method, body=None, headers=None): url = 'http://%s.s3.amazonaws.com/%s%s' % (self.bucket, self.prefix, key) return Request(url, method=method, body=body, headers=headers, \ - priority=self.request_priority) + meta={'sign_s3_request': True}, priority=self.request_priority) def _download_request(self, request, info): """This method is used for HEAD and PUT requests sent to amazon S3 From be45acd4576f75dbc063ec5058f071f1384dd00a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 11 Apr 2010 03:37:08 -0300 Subject: [PATCH 47/68] added scrapy.service and scrapy.tac for running from twistd --- bin/scrapy.tac | 5 ++++ scrapy/service.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ scrapy/utils/py26.py | 37 +++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 bin/scrapy.tac create mode 100644 scrapy/service.py create mode 100644 scrapy/utils/py26.py diff --git a/bin/scrapy.tac b/bin/scrapy.tac new file mode 100644 index 000000000..35e25c56f --- /dev/null +++ b/bin/scrapy.tac @@ -0,0 +1,5 @@ +from twisted.application.service import Application +from scrapy.service import ScrapyService + +application = Application("Scrapy") +ScrapyService().setServiceParent(application) diff --git a/scrapy/service.py b/scrapy/service.py new file mode 100644 index 000000000..7247fd0ca --- /dev/null +++ b/scrapy/service.py @@ -0,0 +1,60 @@ +import sys, os + +from twisted.python import log +from twisted.internet import reactor, protocol, error +from twisted.application.service import Service + +from scrapy.utils.py26 import cpu_count +from scrapy.conf import settings + + +class ScrapyService(Service): + + def startService(self): + reactor.callWhenRunning(self.start_processes) + + def start_processes(self): + for i in range(cpu_count()): + self.start_process(i+1) + + def start_process(self, id): + args = [sys.executable, '-m', 'scrapy.service'] + env = os.environ.copy() + self.set_log_file(env, id) + pp = ScrapyProcessProtocol(self, id, env.get('SCRAPY_LOG_FILE')) + reactor.spawnProcess(pp, sys.executable, args=args, env=env) + + def set_log_file(self, env, suffix): + logfile = settings['LOG_FILE'] + if logfile: + file, ext = os.path.splitext(logfile) + env['SCRAPY_LOG_FILE'] = "%s-%s%s" % (file, suffix, ext) + + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + + def __init__(self, service, id, logfile): + self.service = service + self.id = id + self.logfile = logfile + self.pid = None + + def connectionMade(self): + self.pid = self.transport.pid + log.msg("Process %r started: pid=%r logfile=%r" % (self.id, self.pid, \ + self.logfile)) + + def processEnded(self, status): + if isinstance(status.value, error.ProcessDone): + log.msg("Process %r finished: pid=%r logfile=%r" % (self.id, \ + self.pid, self.logfile)) + else: + log.msg("Process %r died: exitstatus=%r pid=%r logfile=%r" % \ + (self.id, status.value.exitCode, self.pid, self.logfile)) + reactor.callLater(5, self.service.start_process, self.id) + + +if __name__ == '__main__': + from scrapy.core.manager import scrapymanager + scrapymanager.configure() + scrapymanager.start(keep_alive=True) diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py new file mode 100644 index 000000000..44a416a55 --- /dev/null +++ b/scrapy/utils/py26.py @@ -0,0 +1,37 @@ +""" +This module provides functions added in Python 2.6, which weren't yet available +in Python 2.5. The Python 2.6 function is used when available. +""" + +import sys, os + +try: + import multiprocessing + cpu_count = multiprocessing.cpu_count +except ImportError: + def cpu_count(): + ''' + Returns the number of CPUs in the system + ''' + if sys.platform == 'win32': + try: + num = int(os.environ['NUMBER_OF_PROCESSORS']) + except (ValueError, KeyError): + num = 0 + elif 'bsd' in sys.platform or sys.platform == 'darwin': + try: + num = int(os.popen('sysctl -n hw.ncpu').read()) + except ValueError: + num = 0 + else: + try: + num = os.sysconf('SC_NPROCESSORS_ONLN') + except (ValueError, OSError, AttributeError): + num = 0 + + if num >= 1: + return num + else: + raise NotImplementedError('cannot determine number of cpus') + + From 650d1c4fbe68440e38d9387f0563f02c27f81b65 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 11 Apr 2010 03:47:48 -0300 Subject: [PATCH 48/68] moved copytree() function from utils.python to utils.py26 --- scrapy/utils/py26.py | 62 ++++++++++++++++++++++++++++++++++++++- scrapy/utils/python.py | 66 ------------------------------------------ 2 files changed, 61 insertions(+), 67 deletions(-) diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py index 44a416a55..64e211e61 100644 --- a/scrapy/utils/py26.py +++ b/scrapy/utils/py26.py @@ -3,7 +3,12 @@ This module provides functions added in Python 2.6, which weren't yet available in Python 2.5. The Python 2.6 function is used when available. """ -import sys, os +import sys +import os +import fnmatch +from shutil import copytree, ignore_patterns, copy2, copystat + +__all__ = ['cpu_count', 'copytree', 'ignore_patterns'] try: import multiprocessing @@ -34,4 +39,59 @@ except ImportError: else: raise NotImplementedError('cannot determine number of cpus') +if sys.version_info < (2, 6): + try: + WindowsError + except NameError: + WindowsError = None + class Error(EnvironmentError): + pass + + def ignore_patterns(*patterns): + def _ignore_patterns(path, names): + ignored_names = [] + for pattern in patterns: + ignored_names.extend(fnmatch.filter(names, pattern)) + return set(ignored_names) + return _ignore_patterns + + def copytree(src, dst, symlinks=False, ignore=None): + names = os.listdir(src) + if ignore is not None: + ignored_names = ignore(src, names) + else: + ignored_names = set() + + os.makedirs(dst) + errors = [] + for name in names: + if name in ignored_names: + continue + srcname = os.path.join(src, name) + dstname = os.path.join(dst, name) + try: + if symlinks and os.path.islink(srcname): + linkto = os.readlink(srcname) + os.symlink(linkto, dstname) + elif os.path.isdir(srcname): + copytree(srcname, dstname, symlinks, ignore) + else: + copy2(srcname, dstname) + # XXX What about devices, sockets etc.? + except (IOError, os.error), why: + errors.append((srcname, dstname, str(why))) + # catch the Error from the recursive copytree so that we can + # continue with other files + except Error, err: + errors.extend(err.args[0]) + try: + copystat(src, dst) + except OSError, why: + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) + if errors: + raise Error, errors diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 99aefcd74..abf3625e3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,12 +6,9 @@ higher than 2.5 which is the lowest version supported by Scrapy. """ import re -import os -import fnmatch import inspect import weakref from functools import wraps -from shutil import copy2, copystat from sgmllib import SGMLParser class FixedSGMLParser(SGMLParser): @@ -146,68 +143,6 @@ def isbinarytext(text): assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ return any(c in _BINARYCHARS for c in text) - -# ----- shutil.copytree function from Python 2.6 adds ignore argument ---- # - -try: - WindowsError -except NameError: - WindowsError = None - -class Error(EnvironmentError): - pass - -def ignore_patterns(*patterns): - def _ignore_patterns(path, names): - ignored_names = [] - for pattern in patterns: - ignored_names.extend(fnmatch.filter(names, pattern)) - return set(ignored_names) - return _ignore_patterns - -def copytree(src, dst, symlinks=False, ignore=None): - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - os.makedirs(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore) - else: - copy2(srcname, dstname) - # XXX What about devices, sockets etc.? - except (IOError, os.error), why: - errors.append((srcname, dstname, str(why))) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error, err: - errors.extend(err.args[0]) - try: - copystat(src, dst) - except OSError, why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.extend((src, dst, str(why))) - if errors: - raise Error, errors - -# ----- end of shutil.copytree function from Python 2.6 ---- # - - def get_func_args(func): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -221,7 +156,6 @@ def get_func_args(func): raise TypeError('%s is not callable' % type(func)) return func_args - def equal_attributes(obj1, obj2, attributes): """Compare two objects attributes""" # not attributes given return False by default From 025b34e122b7ca615c304266cee7afcebf4bce86 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 11 Apr 2010 07:07:38 -0300 Subject: [PATCH 49/68] bugfix for python < 2.6 --- scrapy/utils/py26.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py index 64e211e61..f7e6e6768 100644 --- a/scrapy/utils/py26.py +++ b/scrapy/utils/py26.py @@ -6,7 +6,7 @@ in Python 2.5. The Python 2.6 function is used when available. import sys import os import fnmatch -from shutil import copytree, ignore_patterns, copy2, copystat +from shutil import copy2, copystat __all__ = ['cpu_count', 'copytree', 'ignore_patterns'] @@ -39,7 +39,9 @@ except ImportError: else: raise NotImplementedError('cannot determine number of cpus') -if sys.version_info < (2, 6): +if sys.version_info >= (2, 6): + from shutil import copytree, ignore_patterns +else: try: WindowsError except NameError: From cd6aa72d7f13a7169a4ce0204559f194fdf229f3 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 12 Apr 2010 10:42:07 -0300 Subject: [PATCH 50/68] fixed import --- scrapy/command/commands/startproject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/command/commands/startproject.py b/scrapy/command/commands/startproject.py index eed8ae260..ee44026ce 100644 --- a/scrapy/command/commands/startproject.py +++ b/scrapy/command/commands/startproject.py @@ -7,7 +7,7 @@ from os.path import join, exists import scrapy from scrapy.command import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase -from scrapy.utils.python import ignore_patterns, copytree +from scrapy.utils.py26 import ignore_patterns, copytree TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project') From b94abf36a30df5482585b56c3e415856733ed9b3 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 12 Apr 2010 10:44:07 -0300 Subject: [PATCH 51/68] Added scrapy.utils.py26.json to use python2.6 json module when available, otherwise failback to simplejson module or scrapy.xlib.simplejson. This way we can always assume json and avoid conditional code. --- scrapy/contrib/exporter/jsonlines.py | 6 +- scrapy/tests/__init__.py | 3 - scrapy/tests/test_contrib_exporter.py | 18 +- scrapy/utils/py26.py | 8 + scrapy/xlib/simplejson/__init__.py | 437 +++++++++++++++++++++ scrapy/xlib/simplejson/decoder.py | 421 +++++++++++++++++++++ scrapy/xlib/simplejson/encoder.py | 501 +++++++++++++++++++++++++ scrapy/xlib/simplejson/ordered_dict.py | 119 ++++++ scrapy/xlib/simplejson/scanner.py | 77 ++++ scrapy/xlib/simplejson/tool.py | 39 ++ 10 files changed, 1606 insertions(+), 23 deletions(-) create mode 100644 scrapy/xlib/simplejson/__init__.py create mode 100644 scrapy/xlib/simplejson/decoder.py create mode 100644 scrapy/xlib/simplejson/encoder.py create mode 100644 scrapy/xlib/simplejson/ordered_dict.py create mode 100644 scrapy/xlib/simplejson/scanner.py create mode 100644 scrapy/xlib/simplejson/tool.py diff --git a/scrapy/contrib/exporter/jsonlines.py b/scrapy/contrib/exporter/jsonlines.py index 4bd1b46b3..d26e9607b 100644 --- a/scrapy/contrib/exporter/jsonlines.py +++ b/scrapy/contrib/exporter/jsonlines.py @@ -1,9 +1,5 @@ from scrapy.contrib.exporter import BaseItemExporter - -try: - import json -except ImportError: - import simplejson as json +from scrapy.utils.py26 import json class JsonLinesItemExporter(BaseItemExporter): diff --git a/scrapy/tests/__init__.py b/scrapy/tests/__init__.py index c77772af5..afd463740 100644 --- a/scrapy/tests/__init__.py +++ b/scrapy/tests/__init__.py @@ -6,9 +6,6 @@ To run all Scrapy unittests go to Scrapy main dir and type: bin/runtests.sh If you're in windows use runtests.bat instead. - -Keep in mind that some tests may be skipped if you don't have some (optional) -modules available like MySQLdb or simplejson, but that's not a problem. """ import os diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index 48cc90c46..140108b42 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,10 +1,10 @@ -import cPickle as pickle +import unittest, cPickle as pickle from cStringIO import StringIO -from twisted.trial import unittest - from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode +from scrapy.utils.py26 import json +from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter from scrapy.contrib.exporter import BaseItemExporter, PprintItemExporter, \ PickleItemExporter, CsvItemExporter, XmlItemExporter @@ -149,22 +149,10 @@ class XmlItemExporterTest(BaseItemExporterTest): class JsonLinesItemExporterTest(BaseItemExporterTest): - def setUp(self): - try: - import json - except ImportError: - try: - import simplejson - except ImportError: - raise unittest.SkipTest("simplejson module not available") - super(JsonLinesItemExporterTest, self).setUp() - def _get_exporter(self, **kwargs): - from scrapy.contrib.exporter.jsonlines import JsonLinesItemExporter return JsonLinesItemExporter(self.output, **kwargs) def _check_output(self): - from scrapy.contrib.exporter.jsonlines import json exported = json.loads(self.output.getvalue().strip()) self.assertEqual(exported, dict(self.i)) diff --git a/scrapy/utils/py26.py b/scrapy/utils/py26.py index f7e6e6768..c8a9cd7ea 100644 --- a/scrapy/utils/py26.py +++ b/scrapy/utils/py26.py @@ -97,3 +97,11 @@ else: errors.extend((src, dst, str(why))) if errors: raise Error, errors + +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + import scrapy.xlib.simplejson as json diff --git a/scrapy/xlib/simplejson/__init__.py b/scrapy/xlib/simplejson/__init__.py new file mode 100644 index 000000000..dcfd5413b --- /dev/null +++ b/scrapy/xlib/simplejson/__init__.py @@ -0,0 +1,437 @@ +r"""JSON (JavaScript Object Notation) is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. + +Encoding basic Python object hierarchies:: + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print json.dumps("\"foo\bar") + "\"foo\bar" + >>> print json.dumps(u'\u1234') + "\u1234" + >>> print json.dumps('\\') + "\\" + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> json.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> json.load(io)[0] == 'streaming API' + True + +Specializing JSON object decoding:: + + >>> import simplejson as json + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + >>> from decimal import Decimal + >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') + True + +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) + '[2.0, 1.0]' + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) + '[2.0, 1.0]' + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' + + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) +""" +__version__ = '2.1.1' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', + 'OrderedDict', +] + +__author__ = 'Bob Ippolito ' + +from decimal import Decimal + +from decoder import JSONDecoder, JSONDecodeError +from encoder import JSONEncoder +def _import_OrderedDict(): + import collections + try: + return collections.OrderedDict + except AttributeError: + import ordered_dict + return ordered_dict.OrderedDict +OrderedDict = _import_OrderedDict() + +def _import_c_make_encoder(): + try: + from simplejson._speedups import make_encoder + return make_encoder + except ImportError: + return None + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + use_decimal=False, +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If *indent* is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + default=default, use_decimal=use_decimal, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not use_decimal + and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, default=default, + use_decimal=use_decimal, **kw).encode(obj) + + +_default_decoder = JSONDecoder(encoding=None, object_hook=None, + object_pairs_hook=None) + + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + use_decimal=use_decimal, **kw) + + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and object_pairs_hook is None + and not use_decimal and not kw): + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + if object_pairs_hook is not None: + kw['object_pairs_hook'] = object_pairs_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant + if use_decimal: + if parse_float is not None: + raise TypeError("use_decimal=True implies parse_float=Decimal") + kw['parse_float'] = Decimal + return cls(encoding=encoding, **kw).decode(s) + + +def _toggle_speedups(enabled): + import simplejson.decoder as dec + import simplejson.encoder as enc + import simplejson.scanner as scan + c_make_encoder = _import_c_make_encoder() + if enabled: + dec.scanstring = dec.c_scanstring or dec.py_scanstring + enc.c_make_encoder = c_make_encoder + enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or + enc.py_encode_basestring_ascii) + scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner + else: + dec.scanstring = dec.py_scanstring + enc.c_make_encoder = None + enc.encode_basestring_ascii = enc.py_encode_basestring_ascii + scan.make_scanner = scan.py_make_scanner + dec.make_scanner = scan.make_scanner + global _default_decoder + _default_decoder = JSONDecoder( + encoding=None, + object_hook=None, + object_pairs_hook=None, + ) + global _default_encoder + _default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + ) diff --git a/scrapy/xlib/simplejson/decoder.py b/scrapy/xlib/simplejson/decoder.py new file mode 100644 index 000000000..4cf4015f6 --- /dev/null +++ b/scrapy/xlib/simplejson/decoder.py @@ -0,0 +1,421 @@ +"""Implementation of JSONDecoder +""" +import re +import sys +import struct + +from simplejson.scanner import make_scanner +def _import_c_scanstring(): + try: + from simplejson._speedups import scanstring + return scanstring + except ImportError: + return None +c_scanstring = _import_c_scanstring() + +__all__ = ['JSONDecoder'] + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + # The struct module in Python 2.4 would get frexp() out of range here + # when an endian is specified in the format string. Fixed in Python 2.5+ + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + + +class JSONDecodeError(ValueError): + """Subclass of ValueError with the following additional properties: + + msg: The unformatted error message + doc: The JSON document being parsed + pos: The start index of doc where parsing failed + end: The end index of doc where parsing failed (may be None) + lineno: The line corresponding to pos + colno: The column corresponding to pos + endlineno: The line corresponding to end (may be None) + endcolno: The column corresponding to end (may be None) + + """ + def __init__(self, msg, doc, pos, end=None): + ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) + self.msg = msg + self.doc = doc + self.pos = pos + self.end = end + self.lineno, self.colno = linecol(doc, pos) + if end is not None: + self.endlineno, self.endcolno = linecol(doc, pos) + else: + self.endlineno, self.endcolno = None, None + + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + + +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups + lineno, colno = linecol(doc, pos) + if end is None: + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, +} + +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def py_scanstring(s, end, encoding=None, strict=True, + _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + end = chunk.end() + content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows + if terminator == '"': + break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise JSONDecodeError(msg, s, end) + else: + _append(terminator) + continue + try: + esc = s[end] + except IndexError: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + # If not a unicode escape sequence, must be in the lookup table + if esc != 'u': + try: + char = _b[esc] + except KeyError: + msg = "Invalid \\escape: " + repr(esc) + raise JSONDecodeError(msg, s, end) + end += 1 + else: + # Unicode escape sequence + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise JSONDecodeError(msg, s, end) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise JSONDecodeError(msg, s, end) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise JSONDecodeError(msg, s, end) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) + return u''.join(chunks), end + + +# Use speedup if available +scanstring = c_scanstring or py_scanstring + +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, + object_pairs_hook, memo=None, + _w=WHITESPACE.match, _ws=WHITESPACE_STR): + # Backwards compatibility + if memo is None: + memo = {} + memo_get = memo.setdefault + pairs = [] + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty + nextchar = s[end:end + 1] + # Normally we expect nextchar == '"' + if nextchar != '"': + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + 1 + elif nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end) + end += 1 + while True: + key, end = scanstring(s, end, encoding, strict) + key = memo_get(key, key) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". + if s[end:end + 1] != ':': + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise JSONDecodeError("Expecting : delimiter", s, end) + + end += 1 + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + pairs.append((key, value)) + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 + + if nextchar == '}': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end - 1) + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + + end += 1 + if nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end - 1) + + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = dict(pairs) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + values = [] + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array + if nextchar == ']': + return values, end + 1 + _append = values.append + while True: + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + _append(value) + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end) + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + return values, end + +class JSONDecoder(object): + """Simple JSON decoder + + Performs the following translations in decoding by default: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + + """ + + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True, + object_pairs_hook=None): + """ + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + *strict* controls the parser's behavior when it encounters an + invalid control character in a string. The default setting of + ``True`` means that unescaped control characters are parse errors, if + ``False`` then control characters will be allowed in strings. + + """ + self.encoding = encoding + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.memo = {} + self.scan_once = make_scanner(self) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise JSONDecodeError("Extra data", s, end, len(s)) + return obj + + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` + beginning with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + + """ + try: + obj, end = self.scan_once(s, idx) + except StopIteration: + raise JSONDecodeError("No JSON object could be decoded", s, idx) + return obj, end diff --git a/scrapy/xlib/simplejson/encoder.py b/scrapy/xlib/simplejson/encoder.py new file mode 100644 index 000000000..cab845653 --- /dev/null +++ b/scrapy/xlib/simplejson/encoder.py @@ -0,0 +1,501 @@ +"""Implementation of JSONEncoder +""" +import re +from decimal import Decimal + +def _import_speedups(): + try: + from simplejson import _speedups + return _speedups.encode_basestring_ascii, _speedups.make_encoder + except ImportError: + return None, None +c_encode_basestring_ascii, c_make_encoder = _import_speedups() + +from simplejson.decoder import PosInf + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +FLOAT_REPR = repr + +def encode_basestring(s): + """Return a JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + return ESCAPE_DCT[match.group(0)] + return u'"' + ESCAPE.sub(replace, s) + u'"' + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + #return '\\u{0:04x}'.format(n) + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii) + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8', default=None, + use_decimal=False): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + + If use_decimal is true (not the default), ``decimal.Decimal`` will + be supported directly by the encoder. For the inverse, decode JSON + with ``parse_float=decimal.Decimal``. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.use_decimal = use_decimal + if isinstance(indent, (int, long)): + indent = ' ' * indent + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + if default is not None: + self.default = default + self.encoding = encoding + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from simplejson import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + if self.ensure_ascii: + return ''.join(chunks) + else: + return u''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, + _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on + # the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + key_memo = {} + if (_one_shot and c_make_encoder is not None + and not self.indent and not self.sort_keys): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, key_memo, self.use_decimal) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, self.use_decimal) + try: + return _iterencode(o, 0) + finally: + key_memo.clear() + + +class JSONEncoderForHTML(JSONEncoder): + """An encoder that produces JSON safe to embed in HTML. + + To embed JSON content in, say, a script tag on a web page, the + characters &, < and > should be escaped. They cannot be escaped + with the usual entities (e.g. &) because they are not expanded + within """ + +PARSED3 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1}, + {'end': 26, 'start': 16}, + {'attributes': {}, 'end': 31, 'start': 26, 'tag': u'h1', 'tag_type': 2}, + {'attributes': {}, 'end': 34, 'start': 31, 'tag': u'p', 'tag_type': 1}, + {'end': 51, 'start': 34}, + {'attributes': {}, 'end': 55, 'start': 51, 'tag': u'p', 'tag_type': 2}, + {'end': 70, 'start': 55}, + {'attributes': {u'type': u'text/javascript'}, 'end': 101, 'start': 70, 'tag': u'script', 'tag_type': 1}, + {'end': 124, 'start': 101}, + {'attributes': {}, 'end': 133, 'start': 124, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 140, 'start': 133, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 147, 'start': 140, 'tag': u'html', 'tag_type': 2} +] + +# for testing tags inside scripts +PAGE4 = u"""

    Konnichiwa!!

    hellohello again""" + +PARSED4 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1}, + {'end': 28,'start': 16}, + {'attributes': {}, 'end': 33, 'start': 28, 'tag': u'h1', 'tag_type': 2}, + {'end': 38, 'start': 33}, + {'attributes': {u'type': u'text/javascript'}, 'end': 69, 'start': 38, 'tag': u'script', 'tag_type': 1}, + {'end': 130, 'start': 69}, + {'attributes': {}, 'end': 139, 'start': 130, 'tag': u'script', 'tag_type': 2}, + {'end': 150, 'start': 139}, + {'attributes': {}, 'end': 157, 'start': 150, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 164, 'start': 157, 'tag': u'html', 'tag_type': 2}, +] + +# Test sucessive cleaning elements +PAGE5 = u"""""" + +PARSED5 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1}, + {'end': 25, 'start': 20}, + {'attributes': {}, 'end': 34, 'start': 25, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 42, 'start': 34, 'tag': u'script', 'tag_type': 1}, + {'end': 45, 'start': 42}, + {'attributes': {}, 'end': 54, 'start': 45, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 61, 'start': 54, 'tag': u'body', 'tag_type': 2}, + {'end': 91, 'start': 61}, + {'attributes': {}, 'end': 98, 'start': 91, 'tag': u'html', 'tag_type': 2}, +] + +# Test sucessive cleaning elements variant 2 +PAGE6 = u"""all\n\n""" + +PARSED6 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1}, + {'end': 40, 'start': 20}, + {'attributes': {}, 'end': 49, 'start': 40, 'tag': u'script', 'tag_type': 2}, + {'end': 52, 'start': 49}, + {'attributes': {}, 'end': 60, 'start': 52, 'tag': u'script', 'tag_type': 1}, + {'end': 63, 'start': 60}, + {'attributes': {}, 'end': 72, 'start': 63, 'tag': u'script', 'tag_type': 2}, + {'end': 74, 'start': 72}, + {'attributes': {}, 'end': 81, 'start': 74, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 88, 'start': 81, 'tag': u'html', 'tag_type': 2}, +] diff --git a/scrapy/tests/test_contrib_ibl/test_pageparsing.py b/scrapy/tests/test_contrib_ibl/test_pageparsing.py new file mode 100644 index 000000000..545e9d46b --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/test_pageparsing.py @@ -0,0 +1,283 @@ +""" +Unit tests for pageparsing +""" +import os +from cStringIO import StringIO +from gzip import GzipFile + +from unittest import TestCase +from scrapy.utils.python import str_to_unicode +from scrapy.utils.py26 import json + +from scrapy.contrib.ibl.htmlpage import HtmlPage +from scrapy.contrib.ibl.extraction.pageparsing import (InstanceLearningParser, + TemplatePageParser, ExtractionPageParser) +from scrapy.contrib.ibl.extraction.pageobjects import TokenDict, TokenType +from scrapy.tests.test_contrib_ibl import path + +SIMPLE_PAGE = u""" +

    this is a test

    +""" + +LABELLED_PAGE1 = u""" + +

    Some Product

    +

    some stuff

    +

    +This is such a nice item
    +Everybody likes it. +

    +

    +\xa310.00 +
    +

    +Old fashioned product +

    +For exigent individuals +

    click here for other items

    + +""" + +BROKEN_PAGE = u""" +

    html parser cannot parse this

    +""" + +LABELLED_PAGE2 = u""" + +

    A product

    +
    +

    A very nice product for all intelligent people

    + +
    +
    +\xa310.00

    13

    +
    + + + +
    + + + + + + +""" + +LABELLED_PAGE3 = u""" + +

    A product

    +
    +

    A very nice product for all intelligent people

    + +
    +
    +\xa310.00

    13

    + + + + + +
    Description 1Description 2Description 3Description 4
    +
    + +""" + +LABELLED_PAGE4 = u""" + +

    A product

    +
    +

    A very nice product for all intelligent people

    + +
    +
    +\xa310.00

    13

    + + + + + +
    Description 1Description 2Description 3 +Price \xa310.00
    +
    + +""" + +LABELLED_PAGE5 = u""" + +
      +
    • Option A
      • +
    • Option I
      • +
    • Option B
      • +
    + +""" + +LABELLED_PAGE6 = u""" + +Text A +

    +65.00pounds

    +

    Description: +Text B

    +Text C + +""" + +LABELLED_PAGE7 = u""" + +
    +Item Id +Description +
    + +""" + +LABELLED_PAGE8 = u""" + +
    +Item Id +Description +
    + +""" + +def _parse_page(parser_class, pagetext): + htmlpage = HtmlPage(None, {}, pagetext) + parser = parser_class(TokenDict()) + parser.feed(htmlpage) + return parser + +def _tags(pp, predicate): + return [pp.token_dict.token_string(s) for s in pp.token_list \ + if predicate(s)] + +class TestPageParsing(TestCase): + + def test_instance_parsing(self): + pp = _parse_page(InstanceLearningParser, SIMPLE_PAGE) + # all tags + self.assertEqual(_tags(pp, bool), ['', '

    ', '

    ', '']) + + # open/closing tag handling + openp = lambda x: pp.token_dict.token_type(x) == TokenType.OPEN_TAG + self.assertEqual(_tags(pp, openp), ['', '

    ']) + closep = lambda x: pp.token_dict.token_type(x) == TokenType.CLOSE_TAG + self.assertEqual(_tags(pp, closep), ['

    ', '']) + + def _validate_annotation(self, parser, lable_region, name, start_tag, end_tag): + assert lable_region.surrounds_attribute == name + start_token = parser.token_list[lable_region.start_index] + assert parser.token_dict.token_string(start_token) == start_tag + end_token = parser.token_list[lable_region.end_index] + assert parser.token_dict.token_string(end_token) == end_tag + + def test_template_parsing(self): + lp = _parse_page(TemplatePageParser, LABELLED_PAGE1) + self.assertEqual(len(lp.annotations), 5) + self._validate_annotation(lp, lp.annotations[0], + 'name', '

    ', '

    ') + self.assertEqual(lp.annotations[0].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[1], + 'description', '

    ', '

    ') + self.assertEqual(lp.annotations[1].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[2], + 'price', '

    ', '

    ') + self.assertEqual(lp.annotations[2].match_common_prefix, True) + self._validate_annotation(lp, lp.annotations[3], + 'short_description', '

    ', '

    ') + self.assertEqual(lp.annotations[3].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[4], + 'short_description', '

    ', '

    ') + self.assertEqual(lp.annotations[4].match_common_prefix, False) + + # all tags were closed + self.assertEqual(len(lp.labelled_tag_stacks), 0) + + def test_extraction_page_parsing(self): + epp = _parse_page(ExtractionPageParser, SIMPLE_PAGE) + ep = epp.to_extraction_page() + assert len(ep.page_tokens) == 4 + assert ep.token_html(0) == '' + assert ep.token_html(1) == '

    ' + + assert ep.html_between_tokens(1, 2) == 'this is a test' + assert ep.html_between_tokens(1, 3) == 'this is a test

    ' + + def test_invalid_html(self): + p = _parse_page(InstanceLearningParser, BROKEN_PAGE) + assert p + + def test_ignore_region(self): + """Test ignored regions""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE2) + self.assertEqual(p.ignored_regions, [(7,12),(15,17),(19,26),(21,22),(27,28),(28,29),(29,None),(30,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_ignore_regions2(self): + """Test ignore-beneath regions""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE3) + self.assertEqual(p.ignored_regions, [(7,12),(15,17),(22,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_ignore_regions3(self): + """Test ignore-beneath with annotation inside region""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE4) + self.assertEqual(p.ignored_regions, [(15,17),(22,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_replacement(self): + """Test parsing of replacement tags""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE5) + self.assertEqual(_tags(p, bool), ['', '', '', '', '']) + + def test_partial(self): + """Test partial annotation parsing""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE6) + text = p.annotations[0].annotation_text + self.assertEqual(text.start_text, '') + self.assertEqual(text.follow_text, 'pounds') + text = p.annotations[1].annotation_text + self.assertEqual(text.start_text, "Description: ") + self.assertEqual(text.follow_text, '') + + def test_ignored_partial(self): + """Test ignored region declared on partial annotation""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE7) + self.assertEqual(p.ignored_regions, [(2, 3)]) + + def test_extra_required(self): + """Test parsing of extra required attributes""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE8) + self.assertEqual(p.extra_required_attrs, ["description"]) + + def test_site_pages(self): + """ + Tests from real pages. More reliable and easy to build for more complicated structures + """ + samples_file = open(os.path.join(path, "samples_pageparsing.json.gz"), "r") + samples = [] + for line in GzipFile(fileobj=StringIO(samples_file.read())).readlines(): + samples.append(json.loads(line)) + for sample in samples: + source = sample["annotated"] + annotations = sample["annotations"] + template = HtmlPage(body=str_to_unicode(source)) + parser = TemplatePageParser(TokenDict()) + parser.feed(template) + for annotation in parser.annotations: + test_annotation = annotations.pop(0) + for s in annotation.__slots__: + if s == "tag_attributes": + for pair in getattr(annotation, s): + self.assertEqual(list(pair), test_annotation[s].pop(0)) + else: + self.assertEqual(getattr(annotation, s), test_annotation[s]) + self.assertEqual(annotations, []) From c87a29eb9edda23d70861ab041e1816efbdfa01a Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 14 May 2010 14:48:34 -0300 Subject: [PATCH 63/68] improved docstring --- scrapy/contrib/ibl/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/ibl/__init__.py b/scrapy/contrib/ibl/__init__.py index 3179c6e63..52c6e08b2 100644 --- a/scrapy/contrib/ibl/__init__.py +++ b/scrapy/contrib/ibl/__init__.py @@ -2,9 +2,15 @@ This contrib implements an automatic extraction library based on an Instance Based Learning (IBL) algorithm, as described in the following papers: -A hierarchical approach to wrapper induction + A hierarchical approach to wrapper induction http://portal.acm.org/citation.cfm?id=301191 -Extracting web data using instance based learning + Extracting web data using instance based learning http://portal.acm.org/citation.cfm?id=1265174 + +This code has some additional dependencies too: + +* numpy +* nltk + """ From b2f58207a4365d9886e6293539b1aad9c5ca8183 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 14 May 2010 20:09:07 +0100 Subject: [PATCH 64/68] avoid different behaviour in urljoin between pytho2.5 and python2.6+. see http://bugs.python.org/issue1432 --- scrapy/utils/url.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index e24773e2c..196c03727 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -106,7 +106,7 @@ def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='=', remove=False, u else: querylist.append(ksv) seen.add(k) - return urlparse.urljoin(base, '?'+sep.join(querylist)) + return '?'.join([base, sep.join(querylist)]) if querylist else base def add_or_replace_parameter(url, name, new_value, sep='&', url_is_quoted=False): """Add or remove a parameter to a given url""" From 02b7ca7e8c87cc94034593910213d9ca6ee7b8f4 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 14 May 2010 16:30:50 -0300 Subject: [PATCH 65/68] bumped embedded BeautifulSoup to 3.0.8.1 --- scrapy/xlib/BeautifulSoup.py | 229 +++++++++++++++++++++-------------- 1 file changed, 138 insertions(+), 91 deletions(-) diff --git a/scrapy/xlib/BeautifulSoup.py b/scrapy/xlib/BeautifulSoup.py index 0e214630c..748e6fe4b 100644 --- a/scrapy/xlib/BeautifulSoup.py +++ b/scrapy/xlib/BeautifulSoup.py @@ -42,7 +42,7 @@ http://www.crummy.com/software/BeautifulSoup/documentation.html Here, have some legalese: -Copyright (c) 2004-2008, Leonard Richardson +Copyright (c) 2004-2010, Leonard Richardson All rights reserved. @@ -79,8 +79,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. from __future__ import generators __author__ = "Leonard Richardson (leonardr@segfault.org)" -__version__ = "3.0.7a" -__copyright__ = "Copyright (c) 2004-2008 Leonard Richardson" +__version__ = "3.0.8.1" +__copyright__ = "Copyright (c) 2004-2010 Leonard Richardson" __license__ = "New-style BSD" from sgmllib import SGMLParser, SGMLParseError @@ -104,9 +104,13 @@ markupbase._declname_match = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*\s*').match DEFAULT_OUTPUT_ENCODING = "utf-8" +def _match_css_class(str): + """Build a RE to match the given CSS class.""" + return re.compile(r"(^|.*\s)%s($|\s)" % str) + # First, the classes that represent markup elements. -class PageElement: +class PageElement(object): """Contains the navigational information for some part of the page (either a tag or a piece of text)""" @@ -124,10 +128,11 @@ class PageElement: def replaceWith(self, replaceWith): oldParent = self.parent - myIndex = self.parent.contents.index(self) - if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent: + myIndex = self.parent.index(self) + if hasattr(replaceWith, "parent")\ + and replaceWith.parent is self.parent: # We're replacing this element with one of its siblings. - index = self.parent.contents.index(replaceWith) + index = replaceWith.parent.index(replaceWith) if index and index < myIndex: # Furthermore, it comes before this element. That # means that when we extract it, the index of this @@ -136,11 +141,20 @@ class PageElement: self.extract() oldParent.insert(myIndex, replaceWith) + def replaceWithChildren(self): + myParent = self.parent + myIndex = self.parent.index(self) + self.extract() + reversedChildren = list(self.contents) + reversedChildren.reverse() + for child in reversedChildren: + myParent.insert(myIndex, child) + def extract(self): """Destructively rips this element out of the tree.""" if self.parent: try: - self.parent.contents.remove(self) + del self.parent.contents[self.parent.index(self)] except ValueError: pass @@ -173,18 +187,17 @@ class PageElement: return lastChild def insert(self, position, newChild): - if (isinstance(newChild, basestring) - or isinstance(newChild, unicode)) \ + if isinstance(newChild, basestring) \ and not isinstance(newChild, NavigableString): newChild = NavigableString(newChild) position = min(position, len(self.contents)) - if hasattr(newChild, 'parent') and newChild.parent != None: + if hasattr(newChild, 'parent') and newChild.parent is not None: # We're 'inserting' an element that's already one # of this object's children. - if newChild.parent == self: - index = self.find(newChild) - if index and index < position: + if newChild.parent is self: + index = self.index(newChild) + if index > position: # Furthermore we're moving it further down the # list of this object's children. That means that # when we extract this element, our target index @@ -322,8 +335,21 @@ class PageElement: if isinstance(name, SoupStrainer): strainer = name + # (Possibly) special case some findAll*(...) searches + elif text is None and not limit and not attrs and not kwargs: + # findAll*(True) + if name is True: + return [element for element in generator() + if isinstance(element, Tag)] + # findAll*('tag-name') + elif isinstance(name, basestring): + return [element for element in generator() + if isinstance(element, Tag) and + element.name == name] + else: + strainer = SoupStrainer(name, attrs, text, **kwargs) + # Build a SoupStrainer else: - # Build a SoupStrainer strainer = SoupStrainer(name, attrs, text, **kwargs) results = ResultSet(strainer) g = generator() @@ -344,31 +370,31 @@ class PageElement: #NavigableStrings and Tags. def nextGenerator(self): i = self - while i: + while i is not None: i = i.next yield i def nextSiblingGenerator(self): i = self - while i: + while i is not None: i = i.nextSibling yield i def previousGenerator(self): i = self - while i: + while i is not None: i = i.previous yield i def previousSiblingGenerator(self): i = self - while i: + while i is not None: i = i.previousSibling yield i def parentGenerator(self): i = self - while i: + while i is not None: i = i.parent yield i @@ -503,7 +529,7 @@ class Tag(PageElement): self.parserClass = parser.__class__ self.isSelfClosing = parser.isSelfClosingTag(name) self.name = name - if attrs == None: + if attrs is None: attrs = [] self.attrs = attrs self.contents = [] @@ -521,12 +547,49 @@ class Tag(PageElement): val)) self.attrs = map(convert, self.attrs) + def getString(self): + if (len(self.contents) == 1 + and isinstance(self.contents[0], NavigableString)): + return self.contents[0] + + def setString(self, string): + """Replace the contents of the tag with a string""" + self.clear() + self.append(string) + + string = property(getString, setString) + + def getText(self, separator=u""): + if not len(self.contents): + return u"" + stopNode = self._lastRecursiveChild().next + strings = [] + current = self.contents[0] + while current is not stopNode: + if isinstance(current, NavigableString): + strings.append(current.strip()) + current = current.next + return separator.join(strings) + + text = property(getText) + def get(self, key, default=None): """Returns the value of the 'key' attribute for the tag, or the value given for 'default' if it doesn't have that attribute.""" return self._getAttrMap().get(key, default) + def clear(self): + """Extract all children.""" + for child in self.contents[:]: + child.extract() + + def index(self, element): + for i, child in enumerate(self.contents): + if child is element: + return i + raise ValueError("Tag.index: element not in tag") + def has_key(self, key): return self._getAttrMap().has_key(key) @@ -595,6 +658,8 @@ class Tag(PageElement): NOTE: right now this will return false if two tags have the same attributes in a different order. Should this be fixed?""" + if other is self: + return True if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): return False for i in range(0, len(self.contents)): @@ -638,7 +703,7 @@ class Tag(PageElement): if self.attrs: for key, val in self.attrs: fmt = '%s="%s"' - if isString(val): + if isinstance(val, basestring): if self.containsSubstitutions and '%SOUP-ENCODING%' in val: val = self.substituteEncoding(val, encoding) @@ -710,13 +775,20 @@ class Tag(PageElement): def decompose(self): """Recursively destroys the contents of this tree.""" - contents = [i for i in self.contents] - for i in contents: - if isinstance(i, Tag): - i.decompose() - else: - i.extract() self.extract() + if len(self.contents) == 0: + return + current = self.contents[0] + while current is not None: + next = current.next + if isinstance(current, Tag): + del current.contents[:] + current.parent = None + current.previous = None + current.previousSibling = None + current.next = None + current.nextSibling = None + current = next def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): return self.__str__(encoding, True) @@ -795,24 +867,18 @@ class Tag(PageElement): #Generator methods def childGenerator(self): - for i in range(0, len(self.contents)): - yield self.contents[i] - raise StopIteration + # Just use the iterator from the contents + return iter(self.contents) def recursiveChildGenerator(self): - stack = [(self, 0)] - while stack: - tag, start = stack.pop() - if isinstance(tag, Tag): - for i in range(start, len(tag.contents)): - a = tag.contents[i] - yield a - if isinstance(a, Tag) and tag.contents: - if i < len(tag.contents) - 1: - stack.append((tag, i+1)) - stack.append((a, 0)) - break - raise StopIteration + if not len(self.contents): + raise StopIteration + stopNode = self._lastRecursiveChild().next + current = self.contents[0] + while current is not stopNode: + yield current + current = current.next + # Next, a couple classes to represent queries and their results. class SoupStrainer: @@ -821,8 +887,8 @@ class SoupStrainer: def __init__(self, name=None, attrs={}, text=None, **kwargs): self.name = name - if isString(attrs): - kwargs['class'] = attrs + if isinstance(attrs, basestring): + kwargs['class'] = _match_css_class(attrs) attrs = None if kwargs: if attrs: @@ -881,7 +947,8 @@ class SoupStrainer: found = None # If given a list of items, scan it for a text element that # matches. - if isList(markup) and not isinstance(markup, Tag): + if hasattr(markup, "__iter__") \ + and not isinstance(markup, Tag): for element in markup: if isinstance(element, NavigableString) \ and self.search(element): @@ -894,7 +961,7 @@ class SoupStrainer: found = self.searchTag(markup) # If it's text, make sure the text matches. elif isinstance(markup, NavigableString) or \ - isString(markup): + isinstance(markup, basestring): if self._matches(markup, self.text): found = markup else: @@ -905,8 +972,8 @@ class SoupStrainer: def _matches(self, markup, matchAgainst): #print "Matching %s against %s" % (markup, matchAgainst) result = False - if matchAgainst == True and type(matchAgainst) == types.BooleanType: - result = markup != None + if matchAgainst is True: + result = markup is not None elif callable(matchAgainst): result = matchAgainst(markup) else: @@ -914,17 +981,17 @@ class SoupStrainer: #other ways of matching match the tag name as a string. if isinstance(markup, Tag): markup = markup.name - if markup and not isString(markup): + if markup and not isinstance(markup, basestring): markup = unicode(markup) #Now we know that chunk is either a string, or None. if hasattr(matchAgainst, 'match'): # It's a regexp object. result = markup and matchAgainst.search(markup) - elif isList(matchAgainst): + elif hasattr(matchAgainst, '__iter__'): # list-like result = markup in matchAgainst elif hasattr(matchAgainst, 'items'): result = markup.has_key(matchAgainst) - elif matchAgainst and isString(markup): + elif matchAgainst and isinstance(markup, basestring): if isinstance(markup, unicode): matchAgainst = unicode(matchAgainst) else: @@ -943,20 +1010,6 @@ class ResultSet(list): # Now, some helper functions. -def isList(l): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is listlike.""" - return hasattr(l, '__iter__') \ - or (type(l) in (types.ListType, types.TupleType)) - -def isString(s): - """Convenience method that works with all 2.x versions of Python - to determine whether or not something is stringlike.""" - try: - return isinstance(s, unicode) or isinstance(s, basestring) - except NameError: - return isinstance(s, str) - def buildTagMap(default, *args): """Turns a list of maps, lists, or scalars into a single map. Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and @@ -967,7 +1020,7 @@ def buildTagMap(default, *args): #It's a map. Merge it. for k,v in portion.items(): built[k] = v - elif isList(portion): + elif hasattr(portion, '__iter__'): # is a list #It's a list. Map each item to the default. for k in portion: built[k] = default @@ -1116,7 +1169,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): self.declaredHTMLEncoding = dammit.declaredHTMLEncoding if markup: if self.markupMassage: - if not isList(self.markupMassage): + if not hasattr(self.markupMassage, "__iter__"): self.markupMassage = self.MARKUP_MASSAGE for fix, m in self.markupMassage: markup = fix.sub(m, markup) @@ -1139,10 +1192,10 @@ class BeautifulStoneSoup(Tag, SGMLParser): superclass or the Tag superclass, depending on the method name.""" #print "__getattr__ called on %s.%s" % (self.__class__, methodName) - if methodName.find('start_') == 0 or methodName.find('end_') == 0 \ - or methodName.find('do_') == 0: + if methodName.startswith('start_') or methodName.startswith('end_') \ + or methodName.startswith('do_'): return SGMLParser.__getattr__(self, methodName) - elif methodName.find('__') != 0: + elif not methodName.startswith('__'): return Tag.__getattr__(self, methodName) else: raise AttributeError @@ -1165,12 +1218,6 @@ class BeautifulStoneSoup(Tag, SGMLParser): def popTag(self): tag = self.tagStack.pop() - # Tags with just one string-owning child get the child as a - # 'string' property, so that soup.tag.string is shorthand for - # soup.tag.contents[0] - if len(self.currentTag.contents) == 1 and \ - isinstance(self.currentTag.contents[0], NavigableString): - self.currentTag.string = self.currentTag.contents[0] #print "Pop", tag.name if self.tagStack: @@ -1259,9 +1306,9 @@ class BeautifulStoneSoup(Tag, SGMLParser): #last occurance. popTo = name break - if (nestingResetTriggers != None + if (nestingResetTriggers is not None and p.name in nestingResetTriggers) \ - or (nestingResetTriggers == None and isResetNesting + or (nestingResetTriggers is None and isResetNesting and self.RESET_NESTING_TAGS.has_key(p.name)): #If we encounter one of the nesting reset triggers @@ -1280,7 +1327,7 @@ class BeautifulStoneSoup(Tag, SGMLParser): if self.quoteStack: #This is not a real tag. #print "<%s> is not real!" % name - attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs)) + attrs = ''.join([' %s="%s"' % (x, y) for x, y in attrs]) self.handle_data('<%s%s>' % (name, attrs)) return self.endData() @@ -1470,8 +1517,8 @@ class BeautifulSoup(BeautifulStoneSoup): BeautifulStoneSoup.__init__(self, *args, **kwargs) SELF_CLOSING_TAGS = buildTagMap(None, - ['br' , 'hr', 'input', 'img', 'meta', - 'spacer', 'link', 'frame', 'base']) + ('br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base', 'col')) PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) @@ -1480,13 +1527,13 @@ class BeautifulSoup(BeautifulStoneSoup): #According to the HTML standard, each of these inline tags can #contain another tag of the same type. Furthermore, it's common #to actually use these tags this way. - NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', - 'center'] + NESTABLE_INLINE_TAGS = ('span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', + 'center') #According to the HTML standard, these block tags can contain #another tag of the same type. Furthermore, it's common #to actually use these tags this way. - NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del'] + NESTABLE_BLOCK_TAGS = ('blockquote', 'div', 'fieldset', 'ins', 'del') #Lists can contain other lists, but there are restrictions. NESTABLE_LIST_TAGS = { 'ol' : [], @@ -1506,7 +1553,7 @@ class BeautifulSoup(BeautifulStoneSoup): 'tfoot' : ['table'], } - NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre'] + NON_NESTABLE_BLOCK_TAGS = ('address', 'form', 'p', 'pre') #If one of these tags is encountered, all tags up to the next tag of #this type are popped. @@ -1597,11 +1644,11 @@ class ICantBelieveItsBeautifulSoup(BeautifulSoup): wouldn't be.""" I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ - ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', + ('em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', - 'big'] + 'big') - I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript'] + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ('noscript',) NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, @@ -1752,7 +1799,7 @@ class UnicodeDammit: """Changes a MS smart quote character to an XML or HTML entity.""" sub = self.MS_CHARS.get(orig) - if type(sub) == types.TupleType: + if isinstance(sub, tuple): if self.smartQuotesTo == 'xml': sub = '&#x%s;' % sub[1] else: From 1868ede5492219fbd3eca90dc897c4293eae6aa0 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 14 May 2010 16:38:04 -0300 Subject: [PATCH 66/68] bumped embedded pydispatch to 2.0.1 --- scrapy/xlib/pydispatch/dispatcher.py | 45 ++++++++++++++-------------- scrapy/xlib/pydispatch/saferef.py | 4 +-- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py index d5c5b0034..8fe4d1ab6 100644 --- a/scrapy/xlib/pydispatch/dispatcher.py +++ b/scrapy/xlib/pydispatch/dispatcher.py @@ -30,7 +30,7 @@ import types, weakref from scrapy.xlib.pydispatch import saferef, robustapply, errors __author__ = "Patrick K. O'Brien " -__cvsid__ = "$Id$" +__cvsid__ = "$Id: dispatcher.py,v 1.1.1.1 2006/07/07 15:59:38 mcfletch Exp $" __version__ = "$Revision: 1.1.1.1 $"[11:-2] try: @@ -377,28 +377,29 @@ def _removeReceiver(receiver): # During module cleanup the mapping will be replaced with None return False backKey = id(receiver) - for senderkey in sendersBack.get(backKey,()): - try: - signals = connections[senderkey].keys() - except KeyError,err: - pass - else: - for signal in signals: - try: - receivers = connections[senderkey][signal] - except KeyError: - pass - else: - try: - receivers.remove( receiver ) - except Exception, err: - pass - _cleanupConnections(senderkey, signal) try: - del sendersBack[ backKey ] - except KeyError: - pass - + backSet = sendersBack.pop(backKey) + except KeyError, err: + return False + else: + for senderkey in backSet: + try: + signals = connections[senderkey].keys() + except KeyError,err: + pass + else: + for signal in signals: + try: + receivers = connections[senderkey][signal] + except KeyError: + pass + else: + try: + receivers.remove( receiver ) + except Exception, err: + pass + _cleanupConnections(senderkey, signal) + def _cleanupConnections(senderkey, signal): """Delete any empty signals for senderkey. Delete senderkey if empty.""" try: diff --git a/scrapy/xlib/pydispatch/saferef.py b/scrapy/xlib/pydispatch/saferef.py index 6b3eda1d3..d28229a9b 100644 --- a/scrapy/xlib/pydispatch/saferef.py +++ b/scrapy/xlib/pydispatch/saferef.py @@ -22,7 +22,7 @@ def safeRef(target, onDelete = None): onDelete=onDelete ) return reference - if callable(onDelete): + if onDelete is not None: return weakref.ref(target, onDelete) else: return weakref.ref( target ) @@ -120,7 +120,7 @@ class BoundMethodWeakref(object): self.key = self.calculateKey( target ) self.weakSelf = weakref.ref(target.im_self, remove) self.weakFunc = weakref.ref(target.im_func, remove) - self.selfName = str(target.im_self) + self.selfName = target.im_self.__class__.__name__ self.funcName = str(target.im_func.__name__) def calculateKey( cls, target ): """Calculate the reference key for this reference From d42e5fdbac0ad458c3208236616370fecc7c2c5b Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Fri, 2 Apr 2010 19:45:30 +0800 Subject: [PATCH 67/68] linkextractor: unique after urljoin_rfc Now, '/foo.html' and 'http://example.org/foo.html' are considered as the same and only one is kept. Signed-off-by: Ping Yin --- scrapy/contrib/linkextractors/sgml.py | 6 +++--- .../sample_data/link_extractor/sgml_linkextractor.html | 2 +- scrapy/tests/test_contrib_linkextractors.py | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 9ec664bda..5510396c0 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -25,16 +25,16 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): self.feed(response_text) self.close() - links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links - ret = [] base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url - for link in links: + for link in self.links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) link.text = str_to_unicode(link.text, response_encoding) ret.append(link) + ret = unique_list(ret, key=lambda link: link.url) if self.unique else ret + return ret def extract_links(self, response): diff --git a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html index 602ea04cd..fecd86563 100644 --- a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -9,7 +9,7 @@ sample 2 -sample 3 text +sample 3 text sample 3 repetition diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 65ffad714..a5e57bde0 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -110,6 +110,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition') ]) + lx = SgmlLinkExtractor(allow=('sample', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) + lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), From f2363afe6f7045dc3b09ff39ef7e80db780462bf Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Tue, 27 Apr 2010 14:58:11 +0800 Subject: [PATCH 68/68] LinkExtractor: split _process_links from _extract_links Separate the extraction and process logic, so we can override in subclass easier. Signed-off-by: Ping Yin --- scrapy/contrib/linkextractors/sgml.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 5510396c0..17877031d 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -21,6 +21,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): self.unique = unique def _extract_links(self, response_text, response_url, response_encoding): + """ Do the real extraction work """ self.reset() self.feed(response_text) self.close() @@ -33,13 +34,21 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): link.text = str_to_unicode(link.text, response_encoding) ret.append(link) - ret = unique_list(ret, key=lambda link: link.url) if self.unique else ret - return ret + def _process_links(self, links): + """ Normalize and filter extracted links + + The subclass should override it if neccessary + """ + links = unique_list(links, key=lambda link: link.url) if self.unique else links + return links + def extract_links(self, response): # wrapper needed to allow to work directly with text - return self._extract_links(response.body, response.url, response.encoding) + links = self._extract_links(response.body, response.url, response.encoding) + links = self._process_links(links) + return links def reset(self): FixedSGMLParser.reset(self) @@ -93,12 +102,16 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): def extract_links(self, response): if self.restrict_xpaths: hxs = HtmlXPathSelector(response) - html_slice = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ + html = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ for xpath_expr in self.restrict_xpaths) - links = self._extract_links(html_slice, response.url, response.encoding) else: - links = BaseSgmlLinkExtractor.extract_links(self, response) + html = response.body + links = self._extract_links(html, response.url, response.encoding) + links = self._process_links(links) + return links + + def _process_links(self, links): links = [link for link in links if _is_valid_url(link.url)] if self.allow_res: @@ -114,6 +127,7 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): for link in links: link.url = canonicalize_url(link.url) + links = BaseSgmlLinkExtractor._process_links(self, links) return links def matches(self, url):