From d09eed7674b5df2a1883a7c7abad40fdfd062c74 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 8 Feb 2017 23:44:55 +0500
Subject: [PATCH] use w3lib.html.strip_html5_whitespace function; expand docs;
 strip consistently before calling process_value

---
 docs/topics/link-extractors.rst     |  9 +++++----
 scrapy/linkextractors/htmlparser.py |  6 +++---
 scrapy/linkextractors/lxmlhtml.py   |  6 +++---
 scrapy/linkextractors/sgml.py       |  7 ++++---
 scrapy/utils/url.py                 | 13 -------------
 5 files changed, 15 insertions(+), 26 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 2486e0982..75bdb4142 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -134,10 +134,11 @@ LxmlLinkExtractor
 
     :param strip: whether to strip whitespaces from extracted attributes.
         According to HTML5 standard, leading and trailing whitespaces
-        must be stripped from ``href`` attributes of ``<a>`` and ``<area>``
-        elements, so LinkExtractor strips them by default. Set ``strip=False``
-        to turn it off (e.g. if you're extracting urls from elements or
-        attributes which allow leading/trailing whitespaces).
+        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
+        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
+        elements, etc., so LinkExtractor strips space chars by default.
+        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
+        from elements or attributes which allow leading/trailing whitespaces).
     :type strip: boolean
 
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 4841e4a54..27978a8a1 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -7,10 +7,10 @@ from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
 from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.utils.python import unique as unique_list
-from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -71,9 +71,9 @@ class HtmlParserLinkExtractor(HTMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
-                    url = self.process_attr(value)
                     if self.strip:
-                        url = trim_href_attribute(url)
+                        value = strip_html5_whitespace(value)
+                    url = self.process_attr(value)
                     link = Link(url=url)
                     self.links.append(link)
                     self.current_link = link
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index f753033ab..c284f1905 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -2,15 +2,15 @@
 Link extractor based on lxml.html
 """
 import six
-from six.moves.urllib.parse import urlparse, urljoin
+from six.moves.urllib.parse import urljoin
 
 import lxml.etree as etree
+from w3lib.html import strip_html5_whitespace
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_native_str
 from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
 from scrapy.linkextractors import FilteringLinkExtractor
 
 
@@ -53,7 +53,7 @@ class LxmlParserLinkExtractor(object):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
             try:
                 if self.strip:
-                    attr_val = trim_href_attribute(attr_val)
+                    attr_val = strip_html5_whitespace(attr_val)
                 attr_val = urljoin(base_url, attr_val)
             except ValueError:
                 continue  # skipping bogus links
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 6ecfd52aa..11ff7a261 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -7,12 +7,13 @@ import warnings
 from sgmllib import SGMLParser
 
 from w3lib.url import safe_url_string
+from w3lib.html import strip_html5_whitespace
+
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
-from scrapy.utils.url import trim_href_attribute
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -81,10 +82,10 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
+                    if self.strip and value is not None:
+                        value = strip_html5_whitespace(value)
                     url = self.process_value(value)
                     if url is not None:
-                        if self.strip:
-                            url = trim_href_attribute(url)
                         link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
                         self.links.append(link)
                         self.current_link = link
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 090f65f80..dc1cce4ac 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -103,16 +103,3 @@ def guess_scheme(url):
         return any_to_uri(url)
     else:
         return add_http_if_no_scheme(url)
-
-
-def trim_href_attribute(href):
-    """
-    Process href attribute of ``a`` or ``area`` elements according to HTML5
-    standards (strip all leading and trailing whitespaces). References:
-
-    * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements
-    * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces
-    * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace
-    * https://www.w3.org/TR/html5/infrastructure.html#space-character
-    """
-    return href.strip(' \t\n\r\x0c')