From f46a4500803dac7bd64284862c3b5937bb525a64 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 27 Aug 2015 16:44:34 +0500 Subject: [PATCH 1/6] refactor test_linkextractors * rename LinkExtractorTestCase to BaseSgmlLinkExtractorTestCase * add BaseLinkExtractorTestCase link extractor tests can inherit from and decouple it from SgmlLinkExtractor * add an extra check for deny_extensions * xfail test_restrict_xpaths_with_html_entities for LxmlLinkExtractor explicitly --- tests/test_linkextractors.py | 88 ++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d78b25f25..3e202bf02 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,5 +1,8 @@ import re import unittest + +import pytest + from scrapy.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -9,7 +12,7 @@ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata -class LinkExtractorTestCase(unittest.TestCase): +class BaseSgmlLinkExtractorTestCase(unittest.TestCase): def test_basic(self): html = """Page title<title> <body><p><a href="item/12.html">Item 12</a></p> @@ -92,30 +95,21 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), True) - def test_link_nofollow(self): - html = """ - <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a> - <a href="about.html">About us</a> - """ - response = HtmlResponse("http://example.org/page.html", body=html) - lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), - Link(url='http://example.org/about.html', text=u'About us', nofollow=False), - ]) - -class SgmlLinkExtractorTestCase(unittest.TestCase): - extractor_cls = SgmlLinkExtractor +class BaseLinkExtractorTestCase(unittest.TestCase): + extractor_cls = None def setUp(self): + if self.extractor_cls is None: + raise unittest.SkipTest() body = get_testdata('link_extractor', 'sgml_linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def test_urls_type(self): - '''Test that the resulting urls are regular strings and not a unicode objects''' + ''' Test that the resulting urls are str objects ''' lx = self.extractor_cls() - self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response))) + self.assertTrue(all(isinstance(link.url, str) + for link in lx.extract_links(self.response))) def test_extraction(self): '''Test the extractor's behaviour among different situations''' @@ -271,7 +265,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): def test_restrict_xpaths_with_html_entities(self): html = '<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>' response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') - links = SgmlLinkExtractor(restrict_xpaths='//p').extract_links(response) + links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')]) @@ -326,7 +320,8 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), ]) - def test_deny_extensions(self): + def test_ignored_extensions(self): + # jpg is ignored by default html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" response = HtmlResponse("http://example.org/", body=html) lx = self.extractor_cls() @@ -334,9 +329,10 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.org/page.html', text=u'asd'), ]) - lx = SgmlLinkExtractor(deny_extensions="jpg") + # override denied extensions + lx = self.extractor_cls(deny_extensions=['html']) self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), + Link(url='http://example.org/photo.jpg'), ]) def test_process_value(self): @@ -388,13 +384,6 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): lx = self.extractor_cls(attrs=None) self.assertEqual(lx.extract_links(self.response), []) - html = """<html><area href="sample1.html"></area><a ref="sample2.html">sample text 2</a></html>""" - response = HtmlResponse("http://example.com/index.html", body=html) - lx = SgmlLinkExtractor(attrs=("href")) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - ]) - def test_tags(self): html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" response = HtmlResponse("http://example.com/index.html", body=html) @@ -505,7 +494,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): ]) -class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase): +class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase): extractor_cls = LxmlLinkExtractor def test_link_wrong_href(self): @@ -521,6 +510,10 @@ class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase): Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + @pytest.mark.xfail + def test_restrict_xpaths_with_html_entities(self): + super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + class HtmlParserLinkExtractorTestCase(unittest.TestCase): @@ -552,6 +545,39 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): ]) +class SgmlLinkExtractorTestCase(BaseLinkExtractorTestCase): + extractor_cls = SgmlLinkExtractor + + def test_deny_extensions(self): + html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" + response = HtmlResponse("http://example.org/", body=html) + lx = SgmlLinkExtractor(deny_extensions="jpg") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/page.html', text=u'asd'), + ]) + + def test_attrs_sgml(self): + html = """<html><area href="sample1.html"></area> + <a ref="sample2.html">sample text 2</a></html>""" + response = HtmlResponse("http://example.com/index.html", body=html) + lx = SgmlLinkExtractor(attrs="href") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/sample1.html', text=u''), + ]) + + def test_link_nofollow(self): + html = """ + <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a> + <a href="about.html">About us</a> + """ + response = HtmlResponse("http://example.org/page.html", body=html) + lx = SgmlLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), + Link(url='http://example.org/about.html', text=u'About us', nofollow=False), + ]) + + class RegexLinkExtractorTestCase(unittest.TestCase): def setUp(self): @@ -579,7 +605,3 @@ class RegexLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) - - -if __name__ == "__main__": - unittest.main() From f2edbd05deda6d22649bdc753e9dd48995ee6aba Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Thu, 27 Aug 2015 17:35:29 +0500 Subject: [PATCH 2/6] PY3 port LinkExtractor * tests for other link extractors are moved to test_linkextractors_deprecated.py * in Python 3 Link is converted to use native strings for urls * minor cleanups --- scrapy/link.py | 9 +- scrapy/linkextractors/__init__.py | 11 +- scrapy/linkextractors/lxmlhtml.py | 23 +- tests/py3-ignores.txt | 2 +- tests/test_link.py | 18 +- tests/test_linkextractors.py | 880 ++++++++++-------------- tests/test_linkextractors_deprecated.py | 190 +++++ 7 files changed, 577 insertions(+), 556 deletions(-) create mode 100644 tests/test_linkextractors_deprecated.py diff --git a/scrapy/link.py b/scrapy/link.py index 8bdcce761..dc6e64adc 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,8 +4,8 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ +from scrapy.utils.python import to_native_str -import six class Link(object): """Link objects represent an extracted link by the LinkExtractor.""" @@ -13,11 +13,10 @@ class Link(object): __slots__ = ['url', 'text', 'fragment', 'nofollow'] def __init__(self, url, text='', fragment='', nofollow=False): - if isinstance(url, six.text_type): + if not isinstance(url, str): import warnings - warnings.warn("Do not instantiate Link objects with unicode urls. " - "Assuming utf-8 encoding (which could be wrong)") - url = url.encode('utf-8') + warnings.warn("Link urls must be str objects.") + url = to_native_str(url) self.url = url self.text = text self.fragment = fragment diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index bb799e572..64efa0c55 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -39,7 +39,7 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any((r.search(url) for r in regexs)) -_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file']) +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} class FilteringLinkExtractor(object): @@ -51,8 +51,10 @@ class FilteringLinkExtractor(object): self.link_extractor = link_extractor - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) + for x in arg_to_iter(deny)] self.allow_domains = set(arg_to_iter(allow_domains)) self.deny_domains = set(arg_to_iter(deny_domains)) @@ -64,7 +66,7 @@ class FilteringLinkExtractor(object): self.canonicalize = canonicalize if deny_extensions is None: deny_extensions = IGNORED_EXTENSIONS - self.deny_extensions = set(['.' + e for e in arg_to_iter(deny_extensions)]) + self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} def _link_allowed(self, link): if not _is_valid_url(link.url): @@ -104,5 +106,6 @@ class FilteringLinkExtractor(object): def _extract_links(self, *args, **kwargs): return self.link_extractor._extract_links(*args, **kwargs) + # Top-level imports from .lxmlhtml import LxmlLinkExtractor as LinkExtractor diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index e9fa521f3..e39c9950e 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,16 +1,14 @@ """ Link extractor based on lxml.html """ - -import re +import six from six.moves.urllib.parse import urlparse, urljoin import lxml.etree as etree -from scrapy.selector import Selector from scrapy.link import Link from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import unique as unique_list +from scrapy.utils.python import unique as unique_list, to_native_str from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.response import get_base_url @@ -20,8 +18,9 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" _collect_string_content = etree.XPath("string()") + def _nons(tag): - if isinstance(tag, basestring): + if isinstance(tag, six.string_types): if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag @@ -57,16 +56,13 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - if isinstance(url, unicode): - url = url.encode(response_encoding) + url = to_native_str(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', nofollow=True if el.get('rel') == 'nofollow' else False) links.append(link) - - return unique_list(links, key=lambda link: link.url) \ - if self.unique else links + return self._deduplicate_if_needed(links) def extract_links(self, response): base_url = get_base_url(response) @@ -77,7 +73,11 @@ class LxmlParserLinkExtractor(object): The subclass should override it if neccessary """ - links = unique_list(links, key=lambda link: link.url) if self.unique else links + return self._deduplicate_if_needed(links) + + def _deduplicate_if_needed(self, links): + if self.unique: + return unique_list(links, key=lambda link: link.url) return links @@ -110,4 +110,3 @@ class LxmlLinkExtractor(FilteringLinkExtractor): links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) return unique_list(all_links) - diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index f380e6679..b40293f57 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -3,7 +3,7 @@ tests/test_command_fetch.py tests/test_command_shell.py tests/test_commands.py tests/test_exporters.py -tests/test_linkextractors.py +tests/test_linkextractors_deprecated.py tests/test_crawl.py tests/test_crawler.py tests/test_downloader_handlers.py diff --git a/tests/test_link.py b/tests/test_link.py index 0b79e47cd..c8487698f 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,8 +1,10 @@ import unittest import warnings +import six from scrapy.link import Link + class LinkTest(unittest.TestCase): def _assert_same_links(self, link1, link2): @@ -43,9 +45,15 @@ class LinkTest(unittest.TestCase): l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_unicode_url(self): + def test_non_str_url(self): with warnings.catch_warnings(record=True) as w: - link = Link(u"http://www.example.com/\xa3") - self.assertIsInstance(link.url, bytes) - self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3') - assert len(w) == 1, "warning not issued" + if six.PY2: + link = Link(u"http://www.example.com/\xa3") + self.assertIsInstance(link.url, str) + self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3') + else: + link = Link(b"http://www.example.com/\xc2\xa3") + self.assertIsInstance(link.url, str) + self.assertEqual(link.url, u'http://www.example.com/\xa3') + + assert len(w) == 1, "warning not issued" diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 3e202bf02..5966a3caf 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -3,190 +3,360 @@ import unittest import pytest -from scrapy.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link -from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor -from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata -class BaseSgmlLinkExtractorTestCase(unittest.TestCase): - def test_basic(self): - html = """<html><head><title>Page title<title> - <body><p><a href="item/12.html">Item 12</a></p> - <p><a href="/about.html">About us</a></p> - <img src="/logo.png" alt="Company logo (not a link)" /> - <p><a href="../othercat.html">Other category</a></p> - <p><a href="/">>></a></p> - <p><a href="/" /></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) +# a hack to skip base class tests in pytest +class Base: + class LinkExtractorTestCase(unittest.TestCase): + extractor_cls = None - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), - Link(url='http://example.org/about.html', text='About us'), - Link(url='http://example.org/othercat.html', text='Other category'), - Link(url='http://example.org/', text='>>'), - Link(url='http://example.org/', text='')]) + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) - def test_base_url(self): - html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> - <body><p><a href="item/12.html">Item 12</a></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) + def test_urls_type(self): + ''' Test that the resulting urls are str objects ''' + lx = self.extractor_cls() + self.assertTrue(all(isinstance(link.url, str) + for link in lx.extract_links(self.response))) - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + def test_extraction(self): + '''Test the extractor's behaviour among different situations''' - # base url is an absolute path and relative to host - html = """<html><head><title>Page title<title><base href="/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://example.org/item/12.html', text='Item 12')]) + lx = self.extractor_cls() + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'), + ]) - # base url has no scheme - html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + lx = self.extractor_cls(allow=('sample', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) - def test_link_text_wrong_encoding(self): - html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>""" - response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'), - ]) + lx = self.extractor_cls(allow=('sample', ), unique=False) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + ]) - def test_extraction_encoding(self): - body = get_testdata('link_extractor', 'linkextractor_noenc.html') - response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) - response_noenc = HtmlResponse(url='http://example.com/noenc', body=body) - body = get_testdata('link_extractor', 'linkextractor_latin1.html') - response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) + lx = self.extractor_cls(allow=('sample', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response_utf8), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) + lx = self.extractor_cls(allow=('sample', ), deny=('3', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) - self.assertEqual(lx.extract_links(response_noenc), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) + lx = self.extractor_cls(allow_domains=('google.com', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://www.google.com/something', text=u''), + ]) - self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%F1.html', text=''), - Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')), - ]) + def test_extraction_using_single_values(self): + '''Test the extractor's behaviour among different situations''' - def test_matches(self): - url1 = 'http://lotsofstuff.com/stuff1/index' - url2 = 'http://evenmorestuff.com/uglystuff/index' + lx = self.extractor_cls(allow='sample') + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), True) + lx = self.extractor_cls(allow='sample', deny='3') + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) + lx = self.extractor_cls(allow_domains='google.com') + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://www.google.com/something', text=u''), + ]) -class BaseLinkExtractorTestCase(unittest.TestCase): - extractor_cls = None + lx = self.extractor_cls(deny_domains='example.com') + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://www.google.com/something', text=u''), + ]) - def setUp(self): - if self.extractor_cls is None: - raise unittest.SkipTest() - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) + def test_nofollow(self): + '''Test the extractor's behaviour for links with rel="nofollow"''' - def test_urls_type(self): - ''' Test that the resulting urls are str objects ''' - lx = self.extractor_cls() - self.assertTrue(all(isinstance(link.url, str) - for link in lx.extract_links(self.response))) + html = b"""<html><head><title>Page title<title> + <body> + <div class='links'> + <p><a href="/about.html">About us</a></p> + </div> + <div> + <p><a href="/follow.html">Follow this link</a></p> + </div> + <div> + <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p> + </div> + <div> + <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p> + </div> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html) - def test_extraction(self): - '''Test the extractor's behaviour among different situations''' + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/about.html', text=u'About us'), + Link(url='http://example.org/follow.html', text=u'Follow this link'), + Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), + Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), + ]) - lx = self.extractor_cls() - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - ]) + def test_matches(self): + url1 = 'http://lotsofstuff.com/stuff1/index' + url2 = 'http://evenmorestuff.com/uglystuff/index' - lx = self.extractor_cls(allow=('sample', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - ]) + lx = self.extractor_cls(allow=(r'stuff1', )) + self.assertEqual(lx.matches(url1), True) + self.assertEqual(lx.matches(url2), False) - lx = self.extractor_cls(allow=('sample', ), unique=False) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - ]) + lx = self.extractor_cls(deny=(r'uglystuff', )) + self.assertEqual(lx.matches(url1), True) + self.assertEqual(lx.matches(url2), False) - lx = self.extractor_cls(allow=('sample', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - ]) + lx = self.extractor_cls(allow_domains=('evenmorestuff.com', )) + self.assertEqual(lx.matches(url1), False) + self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow=('sample', ), deny=('3', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - ]) + lx = self.extractor_cls(deny_domains=('lotsofstuff.com', )) + self.assertEqual(lx.matches(url1), False) + self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow_domains=('google.com', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), - ]) + lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), + allow_domains=('blah1.com',), + deny_domains=('blah2.com',)) + self.assertEqual(lx.matches('http://blah1.com/blah1'), True) + self.assertEqual(lx.matches('http://blah1.com/blah2'), False) + self.assertEqual(lx.matches('http://blah2.com/blah1'), False) + self.assertEqual(lx.matches('http://blah2.com/blah2'), False) - def test_extraction_using_single_values(self): - '''Test the extractor's behaviour among different situations''' + def test_restrict_xpaths(self): + lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) - lx = self.extractor_cls(allow='sample') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - ]) + def test_restrict_xpaths_encoding(self): + """Test restrict_xpaths with encodings""" + html = b"""<html><head><title>Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <div class='links'> + <p><a href="/about.html">About us\xa3</a></p> + </div> + <div> + <p><a href="/nofollow.html">This shouldn't be followed</a></p> + </div> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') - lx = self.extractor_cls(allow='sample', deny='3') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - ]) + lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/about.html', text=u'About us\xa3')]) - lx = self.extractor_cls(allow_domains='google.com') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), - ]) + def test_restrict_xpaths_with_html_entities(self): + html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>' + response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') + links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) + self.assertEqual(links, + [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')]) - lx = self.extractor_cls(deny_domains='example.com') - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://www.google.com/something', text=u''), - ]) + def test_restrict_xpaths_concat_in_handle_data(self): + """html entities cause SGMLParser to call handle_data hook twice""" + body = b"""<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>""" + response = HtmlResponse("http://example.org", body=body, encoding='gb18030') + lx = self.extractor_cls(restrict_xpaths="//div") + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + fragment='', nofollow=False)]) - def test_nofollow(self): - '''Test the extractor's behaviour for links with rel="nofollow"''' + def test_restrict_css(self): + lx = self.extractor_cls(restrict_css=('#subwrapper a',)) + self.assertEqual(lx.extract_links(self.response), [ + Link(url='http://example.com/sample2.html', text=u'sample 2') + ]) - html = """<html><head><title>Page title<title> - <body> + def test_restrict_css_and_restrict_xpaths_together(self): + lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), + restrict_css=('#subwrapper + a', )) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) + + def test_area_tag_with_unicode_present(self): + body = b"""<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>""" + response = HtmlResponse("http://example.org", body=body, encoding='utf-8') + lx = self.extractor_cls() + lx.extract_links(response) + lx.extract_links(response) + lx.extract_links(response) + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/foo', text=u'', + fragment='', nofollow=False)]) + + def test_encoded_url(self): + body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>""" + response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + ]) + + def test_encoded_url_in_restricted_xpath(self): + body = b"""<html><body><div><a href="?page=2">BinB</a></body></html>""" + response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') + lx = self.extractor_cls(restrict_xpaths="//div") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), + ]) + + def test_ignored_extensions(self): + # jpg is ignored by default + html = b"""<a href="page.html">asd</a> and <a href="photo.jpg">""" + response = HtmlResponse("http://example.org/", body=html) + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/page.html', text=u'asd'), + ]) + + # override denied extensions + lx = self.extractor_cls(deny_extensions=['html']) + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/photo.jpg'), + ]) + + def test_process_value(self): + """Test restrict_xpaths with encodings""" + html = b""" + <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a> + <a href="/about.html">About us</a> + """ + response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') + + def process_value(value): + m = re.search("javascript:goToPage\('(.*?)'", value) + if m: + return m.group(1) + + lx = self.extractor_cls(process_value=process_value) + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/other/page.html', text='Link text')]) + + def test_base_url_with_restrict_xpaths(self): + html = b"""<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html) + lx = self.extractor_cls(restrict_xpaths="//p") + self.assertEqual(lx.extract_links(response), + [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + + def test_attrs(self): + lx = self.extractor_cls(attrs="href") + self.assertEqual(lx.extract_links(self.response), [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'), + ]) + + lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) + self.assertEqual(lx.extract_links(self.response), [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample2.jpg', text=u''), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'), + ]) + + lx = self.extractor_cls(attrs=None) + self.assertEqual(lx.extract_links(self.response), []) + + def test_tags(self): + html = b"""<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" + response = HtmlResponse("http://example.com/index.html", body=html) + + lx = self.extractor_cls(tags=None) + self.assertEqual(lx.extract_links(response), []) + + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) + + lx = self.extractor_cls(tags="area") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/sample1.html', text=u''), + ]) + + lx = self.extractor_cls(tags="a") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) + + lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=()) + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample2.jpg', text=u''), + ]) + + def test_tags_attrs(self): + html = b""" + <html><body> + <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div> + <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div> + </body></html> + """ + response = HtmlResponse("http://example.com/index.html", body=html) + + lx = self.extractor_cls(tags='div', attrs='data-url') + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + ]) + + lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + ]) + + def test_xhtml(self): + xhtml = b""" + <?xml version="1.0"?> + <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <title>XHTML document title + +

About us

@@ -199,306 +369,49 @@ class BaseLinkExtractorTestCase(unittest.TestCase):

Choose to follow or not

- """ - response = HtmlResponse("http://example.org/somepage/index.html", body=html) + + + """ - lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/about.html', text=u'About us'), - Link(url='http://example.org/follow.html', text=u'Follow this link'), - Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), - Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), - ]) + response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) - def test_matches(self): - url1 = 'http://lotsofstuff.com/stuff1/index' - url2 = 'http://evenmorestuff.com/uglystuff/index' + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)] + ) - lx = self.extractor_cls(allow=(r'stuff1', )) - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), False) + response = XmlResponse("http://example.com/index.xhtml", body=xhtml) - lx = self.extractor_cls(deny=(r'uglystuff', )) - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), False) + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), + Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), + Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), + Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)] + ) - lx = self.extractor_cls(allow_domains=('evenmorestuff.com', )) - self.assertEqual(lx.matches(url1), False) - self.assertEqual(lx.matches(url2), True) - - lx = self.extractor_cls(deny_domains=('lotsofstuff.com', )) - self.assertEqual(lx.matches(url1), False) - self.assertEqual(lx.matches(url2), True) - - lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), - allow_domains=('blah1.com',), - deny_domains=('blah2.com',)) - self.assertEqual(lx.matches('http://blah1.com/blah1'), True) - self.assertEqual(lx.matches('http://blah1.com/blah2'), False) - self.assertEqual(lx.matches('http://blah2.com/blah1'), False) - self.assertEqual(lx.matches('http://blah2.com/blah2'), False) - - def test_restrict_xpaths(self): - lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - ]) - - def test_restrict_xpaths_encoding(self): - """Test restrict_xpaths with encodings""" - html = """Page title<title> - <body><p><a href="item/12.html">Item 12</a></p> - <div class='links'> - <p><a href="/about.html">About us\xa3</a></p> - </div> - <div> - <p><a href="/nofollow.html">This shouldn't be followed</a></p> - </div> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') - - lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/about.html', text=u'About us\xa3')]) - - def test_restrict_xpaths_with_html_entities(self): - html = '<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>' - response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') - links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) - self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')]) - - def test_restrict_xpaths_concat_in_handle_data(self): - """html entities cause SGMLParser to call handle_data hook twice""" - body = """<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>""" - response = HtmlResponse("http://example.org", body=body, encoding='gb18030') - lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', - fragment='', nofollow=False)]) - - def test_restrict_css(self): - lx = self.extractor_cls(restrict_css=('#subwrapper a',)) - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2') - ]) - - def test_restrict_css_and_restrict_xpaths_together(self): - lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ), - restrict_css=('#subwrapper + a', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - ]) - - def test_area_tag_with_unicode_present(self): - body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>""" - response = HtmlResponse("http://example.org", body=body, encoding='utf-8') - lx = self.extractor_cls() - lx.extract_links(response) - lx.extract_links(response) - lx.extract_links(response) - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/foo', text=u'', - fragment='', nofollow=False)]) - - def test_encoded_url(self): - body = """<html><body><div><a href="?page=2">BinB</a></body></html>""" - response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') - lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), - ]) - - def test_encoded_url_in_restricted_xpath(self): - body = """<html><body><div><a href="?page=2">BinB</a></body></html>""" - response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') - lx = self.extractor_cls(restrict_xpaths="//div") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), - ]) - - def test_ignored_extensions(self): - # jpg is ignored by default - html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" - response = HtmlResponse("http://example.org/", body=html) - lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), - ]) - - # override denied extensions - lx = self.extractor_cls(deny_extensions=['html']) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/photo.jpg'), - ]) - - def test_process_value(self): - """Test restrict_xpaths with encodings""" - html = """ - <a href="javascript:goToPage('../other/page.html','photo','width=600,height=540,scrollbars'); return false">Link text</a> - <a href="/about.html">About us</a> - """ - response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') - - def process_value(value): - m = re.search("javascript:goToPage\('(.*?)'", value) - if m: - return m.group(1) - - lx = self.extractor_cls(process_value=process_value) - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/other/page.html', text='Link text')]) - - def test_base_url_with_restrict_xpaths(self): - html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> - <body><p><a href="item/12.html">Item 12</a></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - lx = self.extractor_cls(restrict_xpaths="//p") - self.assertEqual(lx.extract_links(response), - [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) - - def test_attrs(self): - lx = self.extractor_cls(attrs="href") - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - ]) - - lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - ]) - - lx = self.extractor_cls(attrs=None) - self.assertEqual(lx.extract_links(self.response), []) - - def test_tags(self): - html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" - response = HtmlResponse("http://example.com/index.html", body=html) - - lx = self.extractor_cls(tags=None) - self.assertEqual(lx.extract_links(response), []) - - lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - ]) - - lx = self.extractor_cls(tags="area") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - ]) - - lx = self.extractor_cls(tags="a") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - ]) - - lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=()) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample2.jpg', text=u''), - ]) - - def test_tags_attrs(self): - html = """ - <html><body> - <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div> - <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div> - </body></html> - """ - response = HtmlResponse("http://example.com/index.html", body=html) - - lx = self.extractor_cls(tags='div', attrs='data-url') - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) - ]) - - lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), - Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) - ]) - - def test_xhtml(self): - xhtml = """ -<?xml version="1.0"?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head> - <title>XHTML document title - - -
-

About us

-
-
-

Follow this link

-
-
-

Dont follow this one

-
-
-

Choose to follow or not

-
- - - """ - - response = HtmlResponse("http://example.com/index.xhtml", body=xhtml) - - lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)] - ) - - response = XmlResponse("http://example.com/index.xhtml", body=xhtml) - - lx = self.extractor_cls() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False), - Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False), - Link(url='http://example.com/nofollow.html', text=u'Dont follow this one', fragment='', nofollow=True), - Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not', fragment='', nofollow=False)] - ) - - def test_link_wrong_href(self): - html = """ - Item 1 - Item 2 - Item 3 - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = self.extractor_cls() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) + def test_link_wrong_href(self): + html = b""" + Item 1 + Item 2 + Item 3 + """ + response = HtmlResponse("http://example.org/index.html", body=html) + lx = self.extractor_cls() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + ]) -class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase): +class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor def test_link_wrong_href(self): - html = """ + html = b""" Item 1 Item 2 Item 3 @@ -514,94 +427,3 @@ class LxmlLinkExtractorTestCase(BaseLinkExtractorTestCase): def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() - -class HtmlParserLinkExtractorTestCase(unittest.TestCase): - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = HtmlParserLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) - - def test_link_wrong_href(self): - html = """ - Item 1 - Item 2 - Item 3 - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = HtmlParserLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - -class SgmlLinkExtractorTestCase(BaseLinkExtractorTestCase): - extractor_cls = SgmlLinkExtractor - - def test_deny_extensions(self): - html = """asd and """ - response = HtmlResponse("http://example.org/", body=html) - lx = SgmlLinkExtractor(deny_extensions="jpg") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), - ]) - - def test_attrs_sgml(self): - html = """ - sample text 2""" - response = HtmlResponse("http://example.com/index.html", body=html) - lx = SgmlLinkExtractor(attrs="href") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - ]) - - def test_link_nofollow(self): - html = """ - Printer-friendly page - About us - """ - response = HtmlResponse("http://example.org/page.html", body=html) - lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), - Link(url='http://example.org/about.html', text=u'About us', nofollow=False), - ]) - - -class RegexLinkExtractorTestCase(unittest.TestCase): - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = RegexLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) - - def test_link_wrong_href(self): - html = """ - Item 1 - Item 2 - Item 3 - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py new file mode 100644 index 000000000..fd5a78879 --- /dev/null +++ b/tests/test_linkextractors_deprecated.py @@ -0,0 +1,190 @@ +import unittest +from scrapy.linkextractors.regex import RegexLinkExtractor +from scrapy.http import HtmlResponse +from scrapy.link import Link +from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor +from tests import get_testdata + +from tests.test_linkextractors import Base + + +class BaseSgmlLinkExtractorTestCase(unittest.TestCase): + # XXX: should we move some of these tests to base link extractor tests? + + def test_basic(self): + html = """Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/">>></a></p> + <p><a href="/" /></p> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html) + + lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), + Link(url='http://example.org/about.html', text='About us'), + Link(url='http://example.org/othercat.html', text='Other category'), + Link(url='http://example.org/', text='>>'), + Link(url='http://example.org/', text='')]) + + def test_base_url(self): + html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html) + + lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href + self.assertEqual(lx.extract_links(response), + [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title><base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://example.org/item/12.html', text='Item 12')]) + + # base url has no scheme + html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + + def test_link_text_wrong_encoding(self): + html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>""" + response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') + lx = BaseSgmlLinkExtractor() + self.assertEqual(lx.extract_links(response), [ + Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'), + ]) + + def test_extraction_encoding(self): + body = get_testdata('link_extractor', 'linkextractor_noenc.html') + response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) + response_noenc = HtmlResponse(url='http://example.com/noenc', body=body) + body = get_testdata('link_extractor', 'linkextractor_latin1.html') + response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) + + lx = BaseSgmlLinkExtractor() + self.assertEqual(lx.extract_links(response_utf8), [ + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), + ]) + + self.assertEqual(lx.extract_links(response_noenc), [ + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), + ]) + + self.assertEqual(lx.extract_links(response_latin1), [ + Link(url='http://example.com/sample_%F1.html', text=''), + Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')), + ]) + + def test_matches(self): + url1 = 'http://lotsofstuff.com/stuff1/index' + url2 = 'http://evenmorestuff.com/uglystuff/index' + + lx = BaseSgmlLinkExtractor() + self.assertEqual(lx.matches(url1), True) + self.assertEqual(lx.matches(url2), True) + + +class HtmlParserLinkExtractorTestCase(unittest.TestCase): + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = HtmlParserLinkExtractor() + self.assertEqual(lx.extract_links(self.response), + [Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'),]) + + def test_link_wrong_href(self): + html = """ + <a href="http://example.org/item1.html">Item 1</a> + <a href="http://[example.org/item2.html">Item 2</a> + <a href="http://example.org/item3.html">Item 3</a> + """ + response = HtmlResponse("http://example.org/index.html", body=html) + lx = HtmlParserLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + ]) + + +class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): + extractor_cls = SgmlLinkExtractor + + def test_deny_extensions(self): + html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" + response = HtmlResponse("http://example.org/", body=html) + lx = SgmlLinkExtractor(deny_extensions="jpg") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/page.html', text=u'asd'), + ]) + + def test_attrs_sgml(self): + html = """<html><area href="sample1.html"></area> + <a ref="sample2.html">sample text 2</a></html>""" + response = HtmlResponse("http://example.com/index.html", body=html) + lx = SgmlLinkExtractor(attrs="href") + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/sample1.html', text=u''), + ]) + + def test_link_nofollow(self): + html = """ + <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a> + <a href="about.html">About us</a> + """ + response = HtmlResponse("http://example.org/page.html", body=html) + lx = SgmlLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), + Link(url='http://example.org/about.html', text=u'About us', nofollow=False), + ]) + + +class RegexLinkExtractorTestCase(unittest.TestCase): + # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten + # not to depend on SgmlLinkExractor. Its speed is also much worse + # than it should be. + + def setUp(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + self.response = HtmlResponse(url='http://example.com/index', body=body) + + def test_extraction(self): + # Default arguments + lx = RegexLinkExtractor() + self.assertEqual(lx.extract_links(self.response), + [Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'),]) + + def test_link_wrong_href(self): + html = """ + <a href="http://example.org/item1.html">Item 1</a> + <a href="http://[example.org/item2.html">Item 2</a> + <a href="http://example.org/item3.html">Item 3</a> + """ + response = HtmlResponse("http://example.org/index.html", body=html) + lx = RegexLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), + Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), + ]) From d5984bbea99f81765596e1aa57d03aff51612576 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Fri, 28 Aug 2015 02:12:36 +0500 Subject: [PATCH 3/6] PY3 port scrapy.spiders --- scrapy/spiders/crawl.py | 9 +++++++-- scrapy/spiders/init.py | 1 + scrapy/spiders/sitemap.py | 14 +++++++++----- scrapy/utils/gz.py | 6 ++++-- tests/py3-ignores.txt | 1 - tests/test_spider.py | 22 ++++++++++++++-------- tests/test_utils_gz.py | 2 +- 7 files changed, 36 insertions(+), 19 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 77551753e..031f649d6 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,14 +6,17 @@ See documentation in docs/topics/spiders.rst """ import copy +import six from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output from scrapy.spiders import Spider + def identity(x): return x + class Rule(object): def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity): @@ -27,6 +30,7 @@ class Rule(object): else: self.follow = follow + class CrawlSpider(Spider): rules = () @@ -49,7 +53,8 @@ class CrawlSpider(Spider): return seen = set() for n, rule in enumerate(self._rules): - links = [l for l in rule.link_extractor.extract_links(response) if l not in seen] + links = [lnk for lnk in rule.link_extractor.extract_links(response) + if lnk not in seen] if links and rule.process_links: links = rule.process_links(links) for link in links: @@ -77,7 +82,7 @@ class CrawlSpider(Spider): def get_method(method): if callable(method): return method - elif isinstance(method, basestring): + elif isinstance(method, six.string_types): return getattr(self, method, None) self._rules = [copy.copy(r) for r in self.rules] diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 7717c8819..2efb1a869 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,6 +1,7 @@ from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output + class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 5aa0b944d..eede467a8 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,5 +1,6 @@ import re import logging +import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse @@ -20,13 +21,14 @@ class SitemapSpider(Spider): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: - if isinstance(c, basestring): + if isinstance(c, six.string_types): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow] def start_requests(self): - return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls) + for url in self.sitemap_urls: + yield Request(url, self._parse_sitemap) def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): @@ -52,8 +54,8 @@ class SitemapSpider(Spider): break def _get_sitemap_body(self, response): - """Return the sitemap body contained in the given response, or None if the - response is not a sitemap. + """Return the sitemap body contained in the given response, + or None if the response is not a sitemap. """ if isinstance(response, XmlResponse): return response.body @@ -64,11 +66,13 @@ class SitemapSpider(Spider): elif response.url.endswith('.xml.gz'): return gunzip(response.body) + def regex(x): - if isinstance(x, basestring): + if isinstance(x, six.string_types): return re.compile(x) return x + def iterloc(it, alt=False): for d in it: yield d['loc'] diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 741948359..7fa4bba57 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,6 +7,7 @@ except ImportError: from gzip import GzipFile + def gunzip(data): """Gunzip the given data and return as much data as possible. @@ -31,7 +32,8 @@ def gunzip(data): raise return output + def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', '') - return ctype in ('application/x-gzip', 'application/gzip') + ctype = response.headers.get('Content-Type', b'') + return ctype in (b'application/x-gzip', b'application/gzip') diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index b40293f57..2eb22f149 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -28,7 +28,6 @@ tests/test_spidermiddleware_depth.py tests/test_spidermiddleware_httperror.py tests/test_spidermiddleware_offsite.py tests/test_spidermiddleware_referer.py -tests/test_spider.py tests/test_utils_iterators.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_spider.py b/tests/test_spider.py index f2dfd2dce..4d5d4b07e 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -301,26 +301,32 @@ class SitemapSpiderTest(SpiderTest): g.close() GZBODY = f.getvalue() - def test_get_sitemap_body(self): + def assertSitemapBody(self, response, body): spider = self.spider_class("example.com") + self.assertEqual(spider._get_sitemap_body(response), body) + def test_get_sitemap_body(self): r = XmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + self.assertSitemapBody(r, self.BODY) r = HtmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), None) + self.assertSitemapBody(r, None) r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), None) + self.assertSitemapBody(r, None) - r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"}) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + def test_get_sitemap_body_gzip_headers(self): + r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, + headers={"content-type": "application/gzip"}) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url(self): r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url_compressed(self): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + self.assertSitemapBody(r, self.BODY) class BaseSpiderDeprecationTest(unittest.TestCase): diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 94e7b71be..8fb1e414d 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -7,7 +7,7 @@ from tests import tests_datadir SAMPLEDIR = join(tests_datadir, 'compressed') -class GzTest(unittest.TestCase): +class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: From ff24cbbc477e8bd5459034e702bafd0c5ea1fc43 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Fri, 28 Aug 2015 02:52:17 +0500 Subject: [PATCH 4/6] PY3 depth, offsite and referer spider middlewares; Crawler --- scrapy/spidermiddlewares/depth.py | 14 +++++++++----- scrapy/spidermiddlewares/offsite.py | 1 + tests/py3-ignores.txt | 4 ---- tests/test_spidermiddleware_referer.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 795b60eb4..e2f039146 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -35,14 +35,18 @@ class DepthMiddleware(object): if self.prio: request.priority -= depth * self.prio if self.maxdepth and depth > self.maxdepth: - logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s ", - {'maxdepth': self.maxdepth, 'requrl': request.url}, - extra={'spider': spider}) + logger.debug( + "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", + {'maxdepth': self.maxdepth, 'requrl': request.url}, + extra={'spider': spider} + ) return False elif self.stats: if self.verbose_stats: - self.stats.inc_value('request_depth_count/%s' % depth, spider=spider) - self.stats.max_value('request_depth_max', depth, spider=spider) + self.stats.inc_value('request_depth_count/%s' % depth, + spider=spider) + self.stats.max_value('request_depth_max', depth, + spider=spider) return True # base case (depth=0) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index a90f9f1e0..ea1c9270f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) + class OffsiteMiddleware(object): def __init__(self, stats): diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 2eb22f149..759eeffff 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -5,7 +5,6 @@ tests/test_commands.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_crawler.py tests/test_downloader_handlers.py tests/test_downloadermiddleware_ajaxcrawlable.py tests/test_downloadermiddleware_defaultheaders.py @@ -24,10 +23,7 @@ tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py -tests/test_spidermiddleware_depth.py tests/test_spidermiddleware_httperror.py -tests/test_spidermiddleware_offsite.py -tests/test_spidermiddleware_referer.py tests/test_utils_iterators.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index d773ea8d3..bd7673efb 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -17,5 +17,5 @@ class TestRefererMiddleware(TestCase): out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEquals(out[0].headers.get('Referer'), - 'http://scrapytest.org') + b'http://scrapytest.org') From f7052413e092346b7d460d589d23fecaa4351930 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Fri, 28 Aug 2015 23:04:02 +0500 Subject: [PATCH 5/6] PY3 raise an exception if bytes are passed as url to Link constructor --- scrapy/link.py | 15 +++++++++++---- tests/test_link.py | 38 ++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/scrapy/link.py b/scrapy/link.py index dc6e64adc..2c8301680 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,7 +4,10 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -from scrapy.utils.python import to_native_str +import warnings +import six + +from scrapy.utils.python import to_bytes class Link(object): @@ -14,9 +17,13 @@ class Link(object): def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): - import warnings - warnings.warn("Link urls must be str objects.") - url = to_native_str(url) + if six.PY2: + warnings.warn("Link urls must be str objects. " + "Assuming utf-8 encoding (which could be wrong)") + url = to_bytes(url, encoding='utf8') + else: + got = url.__class__.__name__ + raise TypeError("Link urls must be str objects, got %s" % got) self.url = url self.text = text self.fragment = fragment diff --git a/tests/test_link.py b/tests/test_link.py index c8487698f..955430b37 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -16,44 +16,42 @@ class LinkTest(unittest.TestCase): self.assertNotEqual(hash(link1), hash(link2)) def test_eq_and_hash(self): - l1 = Link(b"http://www.example.com") - l2 = Link(b"http://www.example.com/other") - l3 = Link(b"http://www.example.com") + l1 = Link("http://www.example.com") + l2 = Link("http://www.example.com/other") + l3 = Link("http://www.example.com") self._assert_same_links(l1, l1) self._assert_different_links(l1, l2) self._assert_same_links(l1, l3) - l4 = Link(b"http://www.example.com", text="test") - l5 = Link(b"http://www.example.com", text="test2") - l6 = Link(b"http://www.example.com", text="test") + l4 = Link("http://www.example.com", text="test") + l5 = Link("http://www.example.com", text="test2") + l6 = Link("http://www.example.com", text="test") self._assert_same_links(l4, l4) self._assert_different_links(l4, l5) self._assert_same_links(l4, l6) - l7 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False) - l8 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=False) - l9 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True) - l10 = Link(b"http://www.example.com", text="test", fragment='other', nofollow=False) + l7 = Link("http://www.example.com", text="test", fragment='something', nofollow=False) + l8 = Link("http://www.example.com", text="test", fragment='something', nofollow=False) + l9 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) + l10 = Link("http://www.example.com", text="test", fragment='other', nofollow=False) self._assert_same_links(l7, l8) self._assert_different_links(l7, l9) self._assert_different_links(l7, l10) def test_repr(self): - l1 = Link(b"http://www.example.com", text="test", fragment='something', nofollow=True) + l1 = Link("http://www.example.com", text="test", fragment='something', nofollow=True) l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_non_str_url(self): - with warnings.catch_warnings(record=True) as w: - if six.PY2: + def test_non_str_url_py2(self): + if six.PY2: + with warnings.catch_warnings(record=True) as w: link = Link(u"http://www.example.com/\xa3") self.assertIsInstance(link.url, str) self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3') - else: - link = Link(b"http://www.example.com/\xc2\xa3") - self.assertIsInstance(link.url, str) - self.assertEqual(link.url, u'http://www.example.com/\xa3') - - assert len(w) == 1, "warning not issued" + assert len(w) == 1, "warning not issued" + else: + with self.assertRaises(TypeError): + Link(b"http://www.example.com/\xc2\xa3") From 44bfcbcf0f26a469c96feed35c40715b8b58c6a2 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov <kmike84@gmail.com> Date: Mon, 31 Aug 2015 00:49:38 +0500 Subject: [PATCH 6/6] TST split LinkExtractorTestCase.test_extraction into several methods; remove duplicated test --- tests/test_linkextractors.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e6db6a400..129336d14 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -24,9 +24,7 @@ class Base: self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response))) - def test_extraction(self): - '''Test the extractor's behaviour among different situations''' - + def test_extract_all_links(self): lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), @@ -36,6 +34,7 @@ class Base: Link(url='http://example.com/innertag.html', text=u'inner tag'), ]) + def test_extract_filter_allow(self): lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), @@ -43,6 +42,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) + def test_extract_filter_allow_with_duplicates(self): lx = self.extractor_cls(allow=('sample', ), unique=False) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), @@ -51,19 +51,14 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), ]) - lx = self.extractor_cls(allow=('sample', )) - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - ]) - + def test_extract_filter_allow_and_deny(self): lx = self.extractor_cls(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), ]) + def test_extract_filter_allowed_domains(self): lx = self.extractor_cls(allow_domains=('google.com', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://www.google.com/something', text=u''),