RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Strip leading/trailing whitespaces in link extractors. Fixes GH-838.

This commit is contained in:
Mikhail Korobov 2017-02-08 17:03:11 +05:00
parent ad36a4a6ae
commit d079e15fe2
8 changed files with 66 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/topics/link-extractors.rst
View File

@ -51,7 +51,7 @@ LxmlLinkExtractor
:synopsis: lxml's HTMLParser-based link extractors
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True)
LxmlLinkExtractor is the recommended link extractor with handy filtering
options. It is implemented using lxml's robust HTMLParser.
@ -132,4 +132,12 @@ LxmlLinkExtractor
:type process_value: callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
must be stripped from ``href`` attributes of ``<a>`` and ``<area>``
elements, so LinkExtractor strips them by default. Set ``strip=False``
to turn it off (e.g. if you're extracting urls from elements or
attributes which allow leading/trailing whitespaces).
:type strip: boolean
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

8
scrapy/linkextractors/htmlparser.py
View File

@ -1,7 +1,6 @@
"""
HTMLParser-based link extractor
"""
import warnings
import six
from six.moves.html_parser import HTMLParser
@ -11,12 +10,14 @@ from w3lib.url import safe_url_string
from scrapy.link import Link
from scrapy.utils.python import unique as unique_list
from scrapy.utils.url import trim_href_attribute
from scrapy.exceptions import ScrapyDeprecationWarning
class HtmlParserLinkExtractor(HTMLParser):
def __init__(self, tag="a", attr="href", process=None, unique=False):
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True):
HTMLParser.__init__(self)
warnings.warn(
@ -29,6 +30,7 @@ class HtmlParserLinkExtractor(HTMLParser):
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.process_attr = process if callable(process) else lambda v: v
self.unique = unique
self.strip = strip
def _extract_links(self, response_text, response_url, response_encoding):
self.reset()
@ -70,6 +72,8 @@ class HtmlParserLinkExtractor(HTMLParser):
for attr, value in attrs:
if self.scan_attr(attr):
url = self.process_attr(value)
if self.strip:
url = trim_href_attribute(url)
link = Link(url=url)
self.links.append(link)
self.current_link = link

16
scrapy/linkextractors/lxmlhtml.py
View File

@ -9,8 +9,9 @@ import lxml.etree as etree
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list, to_native_str
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.response import get_base_url
from scrapy.utils.url import trim_href_attribute
from scrapy.linkextractors import FilteringLinkExtractor
# from lxml/src/lxml/html/__init__.py
@ -27,11 +28,13 @@ def _nons(tag):
class LxmlParserLinkExtractor(object):
def __init__(self, tag="a", attr="href", process=None, unique=False):
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True):
self.scan_tag = tag if callable(tag) else lambda t: t == tag
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.process_attr = process if callable(process) else lambda v: v
self.unique = unique
self.strip = strip
def _iter_links(self, document):
for el in document.iter(etree.Element):
@ -49,9 +52,11 @@ class LxmlParserLinkExtractor(object):
for el, attr, attr_val in self._iter_links(selector.root):
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
try:
if self.strip:
attr_val = trim_href_attribute(attr_val)
attr_val = urljoin(base_url, attr_val)
except ValueError:
continue # skipping bogus links
continue # skipping bogus links
else:
url = self.process_attr(attr_val)
if url is None:
@ -85,12 +90,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=True,
unique=True, process_value=None, deny_extensions=None, restrict_css=()):
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
strip=True):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
unique=unique, process=process_value)
unique=unique, process=process_value, strip=strip)
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,

13
scrapy/linkextractors/sgml.py
View File

@ -7,18 +7,19 @@ import warnings
from sgmllib import SGMLParser
from w3lib.url import safe_url_string
from scrapy.selector import Selector
from scrapy.link import Link
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list, to_unicode
from scrapy.utils.response import get_base_url
from scrapy.utils.url import trim_href_attribute
from scrapy.exceptions import ScrapyDeprecationWarning
class BaseSgmlLinkExtractor(SGMLParser):
def __init__(self, tag="a", attr="href", unique=False, process_value=None):
def __init__(self, tag="a", attr="href", unique=False, process_value=None,
strip=True):
warnings.warn(
"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
"Please use scrapy.linkextractors.LinkExtractor",
@ -30,6 +31,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
self.process_value = (lambda v: v) if process_value is None else process_value
self.current_link = None
self.unique = unique
self.strip = strip
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
""" Do the real extraction work """
@ -81,6 +83,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
if self.scan_attr(attr):
url = self.process_value(value)
if url is not None:
if self.strip:
url = trim_href_attribute(url)
link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
self.links.append(link)
self.current_link = link
@ -103,7 +107,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
process_value=None, deny_extensions=None, restrict_css=()):
process_value=None, deny_extensions=None, restrict_css=(),
strip=True):
warnings.warn(
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
@ -118,7 +123,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
with warnings.catch_warnings():
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
unique=unique, process_value=process_value)
unique=unique, process_value=process_value, strip=strip)
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,

13
scrapy/utils/url.py
View File

@ -103,3 +103,16 @@ def guess_scheme(url):
return any_to_uri(url)
else:
return add_http_if_no_scheme(url)
def trim_href_attribute(href):
"""
Process href attribute of ``a`` or ``area`` elements according to HTML5
standards (strip all leading and trailing whitespaces). References:
* https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements
* https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces
* https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace
* https://www.w3.org/TR/html5/infrastructure.html#space-character
"""
return href.strip(' \t\n\r\x0c')

1
tests/sample_data/link_extractor/sgml_linkextractor.html
View File

@ -13,6 +13,7 @@
<a href='sample3.html'>sample 3 repetition</a>
<a href='http://www.google.com/something'></a>
<a href='http://example.com/innertag.html'><b>inner</b> tag</a>
<a href=' page 4.html '>href with whitespaces</a>
</div>
</body>
</html>

3
tests/test_linkextractors.py
View File

@ -32,6 +32,7 @@ class Base:
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
def test_extract_filter_allow(self):
@ -281,6 +282,7 @@ class Base:
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
@ -291,6 +293,7 @@ class Base:
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=None)

20
tests/test_linkextractors_deprecated.py
View File

@ -117,12 +117,14 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
def test_extraction(self):
# Default arguments
lx = HtmlParserLinkExtractor()
self.assertEqual(lx.extract_links(self.response),
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
def test_link_wrong_href(self):
html = """
@ -220,3 +222,9 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://b.com/test.html', text=u'', nofollow=False),
])
@unittest.expectedFailure
def test_extraction(self):
# RegexLinkExtractor doesn't parse URLs with leading/trailing
# whitespaces correctly.
super(RegexLinkExtractorTestCase, self).test_extraction()