From 6672883df876803780e5e25e5585d29f1cfcf5cb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 8 Apr 2016 23:25:50 +0200 Subject: [PATCH 1/2] Fix link extractor tests for non-ASCII characters from latin1 document URL path component should use UTF-8 before percent-encoding (that's what browsers do when you open scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html and follow the links) This matches current w3lib v1.14.1 --- tests/test_linkextractors_deprecated.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 89dcb75c2..7759575f3 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import unittest from scrapy.linkextractors.regex import RegexLinkExtractor from scrapy.http import HtmlResponse @@ -81,9 +82,14 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), ]) + # document encoding does not affect URL path component, only query part + # >>> u'sample_ñ.html'.encode('utf8') + # 'sample_\xc3\xb1.html' + # >>> u"sample_á.html".encode('utf8') + # 'sample_\xc3\xa1.html' self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%F1.html', text=''), - Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')), + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), ]) def test_matches(self): From a1a799249b85b3039c6818d04194fbeddadb5b7c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sat, 9 Apr 2016 15:15:01 +0200 Subject: [PATCH 2/2] Add link extractor test for non-ASCII characters in query part of URL --- .../link_extractor/linkextractor_latin1.html | 25 +++++++++++-------- tests/test_linkextractors_deprecated.py | 11 ++++++-- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html index 68609d8d3..fc31d7e5d 100644 --- a/tests/sample_data/link_extractor/linkextractor_latin1.html +++ b/tests/sample_data/link_extractor/linkextractor_latin1.html @@ -1,15 +1,18 @@ - - - -Sample page with links for testing RegexLinkExtractor - + + + + Sample page with links for testing RegexLinkExtractor + -
-
- -
-sample á text -
+
+
+ +
+ sample á text +
+ +
+
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 7759575f3..36dfe174f 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -84,12 +84,19 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase): # document encoding does not affect URL path component, only query part # >>> u'sample_ñ.html'.encode('utf8') - # 'sample_\xc3\xb1.html' + # b'sample_\xc3\xb1.html' # >>> u"sample_á.html".encode('utf8') - # 'sample_\xc3\xa1.html' + # b'sample_\xc3\xa1.html' + # >>> u"sample_ö.html".encode('utf8') + # b'sample_\xc3\xb6.html' + # >>> u"£32".encode('latin1') + # b'\xa332' + # >>> u"µ".encode('latin1') + # b'\xb5' self.assertEqual(lx.extract_links(response_latin1), [ Link(url='http://example.com/sample_%C3%B1.html', text=''), Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), + Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''), ]) def test_matches(self):