RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2547 from scrapy/linkextractor-strip-whitespaces 

[MRG+1] LinkExtractors: strip whitespaces
This commit is contained in:
Daniel Graña 2017-02-20 11:08:32 -03:00 committed by GitHub
parent 58a18e3007 d09eed7674
commit 4a93be4ad8
11 changed files with 61 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
docs/topics/link-extractors.rst
View File

@ -51,7 +51,7 @@ LxmlLinkExtractor
:synopsis: lxml's HTMLParser-based link extractors
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True)
LxmlLinkExtractor is the recommended link extractor with handy filtering
options. It is implemented using lxml's robust HTMLParser.
@ -132,4 +132,13 @@ LxmlLinkExtractor
:type process_value: callable
:param strip: whether to strip whitespaces from extracted attributes.
According to HTML5 standard, leading and trailing whitespaces
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
elements, etc., so LinkExtractor strips space chars by default.
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

2
requirements-py3.txt
View File

@ -3,5 +3,5 @@ lxml>=3.2.4
pyOpenSSL>=0.13.1
cssselect>=0.9
queuelib>=1.1.1
w3lib>=1.14.2
w3lib>=1.17.0
service_identity

2
requirements.txt
View File

@ -2,7 +2,7 @@ Twisted>=13.1.0
lxml
pyOpenSSL
cssselect>=0.9
w3lib>=1.15.0
w3lib>=1.17.0
queuelib
six>=1.5.2
PyDispatcher>=2.0.5

8
scrapy/linkextractors/htmlparser.py
View File

@ -1,13 +1,13 @@
"""
HTMLParser-based link extractor
"""
import warnings
import six
from six.moves.html_parser import HTMLParser
from six.moves.urllib.parse import urljoin
from w3lib.url import safe_url_string
from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
from scrapy.utils.python import unique as unique_list
@ -16,7 +16,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
class HtmlParserLinkExtractor(HTMLParser):
def __init__(self, tag="a", attr="href", process=None, unique=False):
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True):
HTMLParser.__init__(self)
warnings.warn(
@ -29,6 +30,7 @@ class HtmlParserLinkExtractor(HTMLParser):
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.process_attr = process if callable(process) else lambda v: v
self.unique = unique
self.strip = strip
def _extract_links(self, response_text, response_url, response_encoding):
self.reset()
@ -69,6 +71,8 @@ class HtmlParserLinkExtractor(HTMLParser):
if self.scan_tag(tag):
for attr, value in attrs:
if self.scan_attr(attr):
if self.strip:
value = strip_html5_whitespace(value)
url = self.process_attr(value)
link = Link(url=url)
self.links.append(link)

18
scrapy/linkextractors/lxmlhtml.py
View File

@ -2,15 +2,16 @@
Link extractor based on lxml.html
"""
import six
from six.moves.urllib.parse import urlparse, urljoin
from six.moves.urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
from scrapy.utils.python import unique as unique_list, to_native_str
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.response import get_base_url
from scrapy.linkextractors import FilteringLinkExtractor
# from lxml/src/lxml/html/__init__.py
@ -27,11 +28,13 @@ def _nons(tag):
class LxmlParserLinkExtractor(object):
def __init__(self, tag="a", attr="href", process=None, unique=False):
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True):
self.scan_tag = tag if callable(tag) else lambda t: t == tag
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.process_attr = process if callable(process) else lambda v: v
self.unique = unique
self.strip = strip
def _iter_links(self, document):
for el in document.iter(etree.Element):
@ -49,9 +52,11 @@ class LxmlParserLinkExtractor(object):
for el, attr, attr_val in self._iter_links(selector.root):
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
try:
if self.strip:
attr_val = strip_html5_whitespace(attr_val)
attr_val = urljoin(base_url, attr_val)
except ValueError:
continue # skipping bogus links
continue # skipping bogus links
else:
url = self.process_attr(attr_val)
if url is None:
@ -85,12 +90,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=True,
unique=True, process_value=None, deny_extensions=None, restrict_css=()):
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
strip=True):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
unique=unique, process=process_value)
unique=unique, process=process_value, strip=strip)
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,

3
scrapy/linkextractors/regex.py
View File

@ -10,9 +10,10 @@ linkre = re.compile(
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
re.DOTALL | re.IGNORECASE)
def clean_link(link_text):
"""Remove leading and trailing whitespace and punctuation"""
return link_text.strip("\t\r\n '\"")
return link_text.strip("\t\r\n '\"\x0c")
class RegexLinkExtractor(SgmlLinkExtractor):

14
scrapy/linkextractors/sgml.py
View File

@ -7,7 +7,8 @@ import warnings
from sgmllib import SGMLParser
from w3lib.url import safe_url_string
from scrapy.selector import Selector
from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
from scrapy.linkextractors import FilteringLinkExtractor
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@ -18,7 +19,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
class BaseSgmlLinkExtractor(SGMLParser):
def __init__(self, tag="a", attr="href", unique=False, process_value=None):
def __init__(self, tag="a", attr="href", unique=False, process_value=None,
strip=True):
warnings.warn(
"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
"Please use scrapy.linkextractors.LinkExtractor",
@ -30,6 +32,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
self.process_value = (lambda v: v) if process_value is None else process_value
self.current_link = None
self.unique = unique
self.strip = strip
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
""" Do the real extraction work """
@ -79,6 +82,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
if self.scan_tag(tag):
for attr, value in attrs:
if self.scan_attr(attr):
if self.strip and value is not None:
value = strip_html5_whitespace(value)
url = self.process_value(value)
if url is not None:
link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
@ -103,7 +108,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
process_value=None, deny_extensions=None, restrict_css=()):
process_value=None, deny_extensions=None, restrict_css=(),
strip=True):
warnings.warn(
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
@ -118,7 +124,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
with warnings.catch_warnings():
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
unique=unique, process_value=process_value)
unique=unique, process_value=process_value, strip=strip)
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,

2
setup.py
View File

@ -43,7 +43,7 @@ setup(
],
install_requires=[
'Twisted>=13.1.0',
'w3lib>=1.15.0',
'w3lib>=1.17.0',
'queuelib',
'lxml',
'pyOpenSSL',

1
tests/sample_data/link_extractor/sgml_linkextractor.html
View File

@ -13,6 +13,7 @@
<a href='sample3.html'>sample 3 repetition</a>
<a href='http://www.google.com/something'></a>
<a href='http://example.com/innertag.html'><b>inner</b> tag</a>
<a href=' page 4.html '>href with whitespaces</a>
</div>
</body>
</html>

3
tests/test_linkextractors.py
View File

@ -32,6 +32,7 @@ class Base:
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
def test_extract_filter_allow(self):
@ -281,6 +282,7 @@ class Base:
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
@ -291,6 +293,7 @@ class Base:
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
lx = self.extractor_cls(attrs=None)

20
tests/test_linkextractors_deprecated.py
View File

@ -117,12 +117,14 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
def test_extraction(self):
# Default arguments
lx = HtmlParserLinkExtractor()
self.assertEqual(lx.extract_links(self.response),
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
self.assertEqual(lx.extract_links(self.response), [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
])
def test_link_wrong_href(self):
html = """
@ -220,3 +222,9 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
self.assertEqual([link for link in lx.extract_links(response)], [
Link(url='http://b.com/test.html', text=u'', nofollow=False),
])
@unittest.expectedFailure
def test_extraction(self):
# RegexLinkExtractor doesn't parse URLs with leading/trailing
# whitespaces correctly.
super(RegexLinkExtractorTestCase, self).test_extraction()