From e379f58cad0289d725a5241606542d0e20ecd73d Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 29 Oct 2015 14:52:31 +0800 Subject: [PATCH 1/3] fixed: Issue #1564 (Incorrectly picked URL in `scrapy.linkextractors.regex.RegexLinkExtractor` when there is a `` tag. ) --- scrapy/linkextractors/regex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index b6f8d5d30..0fc7b079f 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,7 +1,7 @@ import re from six.moves.urllib.parse import urljoin -from w3lib.html import remove_tags, replace_entities, replace_escape_chars +from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link from .sgml import SgmlLinkExtractor @@ -31,7 +31,7 @@ class RegexLinkExtractor(SgmlLinkExtractor): return clean_url if base_url is None: - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url + base_url = get_base_url(response_text, response_url, response_encoding) links_text = linkre.findall(response_text) return [Link(clean_url(url).encode(response_encoding), From 94486bb294a6e765efe14affddb1df355a6c298b Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Mon, 2 Nov 2015 23:00:42 +0800 Subject: [PATCH 2/3] added: Test case for the fix. --- tests/test_linkextractors_deprecated.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index e3664f8d8..89dcb75c2 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -190,3 +190,20 @@ class RegexLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + + def test_html_base_href(self): + html = """ + + + + + + + + + """ + response = HtmlResponse("http://a.com/", body=html) + lx = RegexLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://b.com/test.html', text=u'', nofollow=False), + ]) From 751155eadeb3b053666804061608b0ceb41f15f6 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 19 Nov 2015 00:38:05 +0800 Subject: [PATCH 3/3] removed: Unused code --- scrapy/linkextractors/sgml.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 7084d0180..9938e071f 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -124,9 +124,6 @@ class SgmlLinkExtractor(FilteringLinkExtractor): restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, canonicalize=canonicalize, deny_extensions=deny_extensions) - # FIXME: was added to fix a RegexLinkExtractor testcase - self.base_url = None - def extract_links(self, response): base_url = None if self.restrict_xpaths: