RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed SgmlLinkExtractor bug which failed to recognize <base> tags when using restrict_xpaths

This commit is contained in:
Pablo Hoffman 2010-11-23 17:28:29 -02:00
parent 426b6fa100
commit d59ef48231
2 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/contrib/linkextractors/sgml.py
View File

@ -9,6 +9,7 @@ from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode
from scrapy.utils.url import safe_url_string, urljoin_rfc, canonicalize_url, url_is_from_any_domain
from scrapy.utils.response import get_base_url
class BaseSgmlLinkExtractor(FixedSGMLParser):
@ -20,14 +21,15 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
self.current_link = None
self.unique = unique
def _extract_links(self, response_text, response_url, response_encoding):
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
""" Do the real extraction work """
self.reset()
self.feed(response_text)
self.close()
ret = []
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
if base_url is None:
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
link.url = safe_url_string(link.url, response_encoding)
@ -100,14 +102,16 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
unique=unique, process_value=process_value)
def extract_links(self, response):
base_url = None
if self.restrict_xpaths:
hxs = HtmlXPathSelector(response)
html = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \
for xpath_expr in self.restrict_xpaths)
base_url = get_base_url(response)
else:
html = response.body
links = self._extract_links(html, response.url, response.encoding)
links = self._extract_links(html, response.url, response.encoding, base_url)
links = self._process_links(links)
return links

10
scrapy/tests/test_contrib_linkextractors.py
View File

@ -221,6 +221,16 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/other/page.html', text='Link text')])
def test_base_url_with_restrict_xpaths(self):
html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
<body><p><a href="item/12.html">Item 12</a></p>
</body></html>"""
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
lx = SgmlLinkExtractor(restrict_xpaths="//p")
self.assertEqual(lx.extract_links(response),
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
class HTMLImageLinkExtractorTestCase(unittest.TestCase):
def setUp(self):
body = get_testdata('link_extractor', 'image_linkextractor.html')