use w3lib.html.strip_html5_whitespace function; expand docs; strip consistently before calling process_value

docs/topics/link-extractors.rst

@@ -134,10 +134,11 @@ LxmlLinkExtractor
 
     :param strip: whether to strip whitespaces from extracted attributes.
         According to HTML5 standard, leading and trailing whitespaces
-        must be stripped from ``href`` attributes of ``<a>`` and ``<area>``
-        elements, so LinkExtractor strips them by default. Set ``strip=False``
-        to turn it off (e.g. if you're extracting urls from elements or
-        attributes which allow leading/trailing whitespaces).
+        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
+        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
+        elements, etc., so LinkExtractor strips space chars by default.
+        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
+        from elements or attributes which allow leading/trailing whitespaces).
     :type strip: boolean
 
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

scrapy/linkextractors/htmlparser.py

@@ -7,10 +7,10 @@ from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
 from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.utils.python import unique as unique_list
-from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -71,9 +71,9 @@ class HtmlParserLinkExtractor(HTMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
-                    url = self.process_attr(value)
                     if self.strip:
-                        url = trim_href_attribute(url)
+                        value = strip_html5_whitespace(value)
+                    url = self.process_attr(value)
                     link = Link(url=url)
                     self.links.append(link)
                     self.current_link = link

scrapy/linkextractors/lxmlhtml.py

@@ -2,15 +2,15 @@
 Link extractor based on lxml.html
 """
 import six
-from six.moves.urllib.parse import urlparse, urljoin
+from six.moves.urllib.parse import urljoin
 
 import lxml.etree as etree
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_native_str
 from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
 from scrapy.linkextractors import FilteringLinkExtractor
 
 
@@ -53,7 +53,7 @@ class LxmlParserLinkExtractor(object):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
             try:
                 if self.strip:
-                    attr_val = trim_href_attribute(attr_val)
+                    attr_val = strip_html5_whitespace(attr_val)
                 attr_val = urljoin(base_url, attr_val)
             except ValueError:
                 continue  # skipping bogus links

scrapy/linkextractors/sgml.py

@@ -7,12 +7,13 @@ import warnings
 from sgmllib import SGMLParser
 
 from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
+
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -81,10 +82,10 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
+                    if self.strip and value is not None:
+                        value = strip_html5_whitespace(value)
                     url = self.process_value(value)
                     if url is not None:
-                        if self.strip:
-                            url = trim_href_attribute(url)
                         link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
                         self.links.append(link)
                         self.current_link = link

scrapy/utils/url.py

@@ -103,16 +103,3 @@ def guess_scheme(url):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
-
-
-def trim_href_attribute(href):
-    """
-    Process href attribute of ``a`` or ``area`` elements according to HTML5
-    standards (strip all leading and trailing whitespaces). References:
-
-    * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements
-    * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces
-    * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace
-    * https://www.w3.org/TR/html5/infrastructure.html#space-character
-    """
-    return href.strip(' \t\n\r\x0c')