RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1921 from scrapy/backport-1.1-pr1913 

[backport][1.1] Fix link extractor tests for non-ASCII characters from latin1 document (PR #1913)
This commit is contained in:
Paul Tremberth 2016-04-11 17:34:47 +02:00
parent 160bcbacfc a1a799249b
commit d30dfada11
2 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
tests/sample_data/link_extractor/linkextractor_latin1.html
View File

@ -1,15 +1,18 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=latin-1">
<base href='http://example.com' />
<title>Sample page with links for testing RegexLinkExtractor</title>
</head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=latin-1">
<base href='http://example.com' />
<title>Sample page with links for testing RegexLinkExtractor</title>
</head>
<body>
<div id='wrapper'>
<div id='subwrapper'>
<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
</div>
<a href='sample_á.html' title='sample á'>sample á text</a>
</div>
<div id='wrapper'>
<div id='subwrapper'>
<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
</div>
<a href='sample_á.html' title='sample á'>sample á text</a>
<div id='subwrapper2'>
<a href='sample_ö.html?price=£32&µ=unit'><img src='sample3.jpg'/></a>
</div>
</div>
</body>
</html>

17
tests/test_linkextractors_deprecated.py
View File

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
import unittest
from scrapy.linkextractors.regex import RegexLinkExtractor
from scrapy.http import HtmlResponse
@ -81,9 +82,21 @@ class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
])
# document encoding does not affect URL path component, only query part
# >>> u'sample_ñ.html'.encode('utf8')
# b'sample_\xc3\xb1.html'
# >>> u"sample_á.html".encode('utf8')
# b'sample_\xc3\xa1.html'
# >>> u"sample_ö.html".encode('utf8')
# b'sample_\xc3\xb6.html'
# >>> u"£32".encode('latin1')
# b'\xa332'
# >>> u"µ".encode('latin1')
# b'\xb5'
self.assertEqual(lx.extract_links(response_latin1), [
Link(url='http://example.com/sample_%F1.html', text=''),
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
])
def test_matches(self):