mirror of https://github.com/scrapy/scrapy.git
Remove the u prefix from strings
This commit is contained in:
Adrián Chaves
parent
5e2d1bd187
commit
890b2138a6
|
|
@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
|
|||
def is_setting_index(node):
|
||||
if node.tagname == 'index':
|
||||
# index entries for setting directives look like:
|
||||
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
|
||||
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
|
||||
entry_type, info, refid = node['entries'][0][:3]
|
||||
return entry_type == 'pair' and info.endswith('; setting')
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -237,10 +237,10 @@ metadata. Here is an example::
|
|||
|
||||
>>> from scrapy.loader import ItemLoader
|
||||
>>> il = ItemLoader(item=Product())
|
||||
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
|
||||
>>> il.add_value('price', [u'€', u'<span>1000</span>'])
|
||||
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
|
||||
>>> il.add_value('price', ['€', '<span>1000</span>'])
|
||||
>>> il.load_item()
|
||||
{'name': u'Welcome to my website', 'price': u'1000'}
|
||||
{'name': 'Welcome to my website', 'price': '1000'}
|
||||
|
||||
The precedence order, for both input and output processors, is as follows:
|
||||
|
||||
|
|
|
|||
|
|
@ -734,7 +734,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
|
|||
Example selecting links in list item with a "class" attribute ending with a digit:
|
||||
|
||||
>>> from scrapy import Selector
|
||||
>>> doc = u"""
|
||||
>>> doc = """
|
||||
... <div>
|
||||
... <ul>
|
||||
... <li class="item-0"><a href="link1.html">first item</a></li>
|
||||
|
|
@ -765,7 +765,7 @@ extracting text elements for example.
|
|||
Example extracting microdata (sample content taken from https://schema.org/Product)
|
||||
with groups of itemscopes and corresponding itemprops::
|
||||
|
||||
>>> doc = u"""
|
||||
>>> doc = """
|
||||
... <div itemscope itemtype="http://schema.org/Product">
|
||||
... <span itemprop="name">Kenmore White 17" Microwave</span>
|
||||
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def main():
|
|||
_contents = None
|
||||
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
|
|||
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
|
||||
namespaces={
|
||||
"re": "http://exslt.org/regular-expressions"})
|
||||
values = [(k, u'' if v is None else v)
|
||||
values = [(k, '' if v is None else v)
|
||||
for k, v in (_value(e) for e in inputs)
|
||||
if k and k not in formdata_keys]
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ def _select_value(ele, n, v):
|
|||
# This is a workround to bug in lxml fixed 2.3.1
|
||||
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
|
||||
selected_options = ele.xpath('.//option[@selected]')
|
||||
v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
|
||||
v = [(o.get('value') or o.text or '').strip() for o in selected_options]
|
||||
return n, v
|
||||
|
||||
|
||||
|
|
@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):
|
|||
|
||||
# We didn't find it, so now we build an XPath expression out of the other
|
||||
# arguments, because they can be used as such
|
||||
xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
|
||||
xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
|
||||
el = form.xpath(xpath)
|
||||
if len(el) == 1:
|
||||
return (el[0].get('name'), el[0].get('value') or '')
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
|
|||
url = safe_url_string(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
link = Link(url, _collect_string_content(el) or '',
|
||||
nofollow=rel_has_nofollow(el.get('rel')))
|
||||
links.append(link)
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class LogFormatter:
|
|||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
'level': logging.INFO, # lowering the level from logging.WARNING
|
||||
'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'args': {
|
||||
'exception': exception,
|
||||
'item': item,
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):
|
|||
'EXTENSIONS=' + json.dumps(EXTENSIONS))
|
||||
# XXX: There's gotta be a smarter way to do this...
|
||||
self.assertNotIn("...", settingsstr)
|
||||
for char in ("'", "<", ">", 'u"'):
|
||||
for char in ("'", "<", ">"):
|
||||
settingsstr = settingsstr.replace(char, '"')
|
||||
settingsdict = json.loads(settingsstr)
|
||||
self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
|
||||
|
|
|
|||
|
|
@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
|
||||
def test_default_mediatype(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(response.text, '\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "iso-8859-7")
|
||||
|
||||
|
|
@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
|
||||
def test_text_charset(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(response.text, '\u038e\u03a3\u038e')
|
||||
self.assertEqual(response.body, b'\xbe\xd3\xbe')
|
||||
self.assertEqual(response.encoding, "iso-8859-7")
|
||||
|
||||
|
|
@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):
|
|||
|
||||
def test_mediatype_parameters(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
|
||||
self.assertEqual(response.text, '\u038e\u03a3\u038e')
|
||||
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
|
||||
self.assertEqual(response.encoding, "utf-8")
|
||||
|
||||
|
|
|
|||
|
|
@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):
|
|||
|
||||
def test_request_cookies_encoding(self):
|
||||
# 1) UTF8-encoded bytes
|
||||
req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
|
||||
req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 2) Non UTF8-encoded bytes
|
||||
req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
|
||||
req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 3) Unicode string
|
||||
req3 = Request('http://example.org', cookies={'a': u'á'})
|
||||
# 3) String
|
||||
req3 = Request('http://example.org', cookies={'a': 'á'})
|
||||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
def test_request_headers_cookie_encoding(self):
|
||||
# 1) UTF8-encoded bytes
|
||||
req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
|
||||
req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 2) Non UTF8-encoded bytes
|
||||
req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
|
||||
req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 3) Unicode string
|
||||
req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
|
||||
# 3) String
|
||||
req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
|
||||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):
|
|||
|
||||
def test_proxy_auth_encoding(self):
|
||||
# utf-8 encoding
|
||||
os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
|
||||
os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
|
||||
mw = HttpProxyMiddleware(auth_encoding='utf-8')
|
||||
req = Request('http://scrapytest.org')
|
||||
assert mw.process_request(req, spider) is None
|
||||
|
|
@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):
|
|||
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
|
||||
|
||||
# proxy from request.meta
|
||||
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
|
||||
req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
|
||||
|
|
@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):
|
|||
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
|
||||
|
||||
# proxy from request.meta, latin-1 encoding
|
||||
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
|
||||
req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
|
||||
def test_latin1_location(self):
|
||||
req = Request('http://scrapytest.org/first')
|
||||
latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1
|
||||
latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1
|
||||
resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
|
||||
req_result = self.mw.process_response(req, resp, self.spider)
|
||||
perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
|
||||
|
|
@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
|
||||
def test_utf8_location(self):
|
||||
req = Request('http://scrapytest.org/first')
|
||||
utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding
|
||||
utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding
|
||||
resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
|
||||
req_result = self.mw.process_response(req, resp, self.spider)
|
||||
perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
|
||||
|
|
@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
|
|||
self.mw = MetaRefreshMiddleware.from_crawler(crawler)
|
||||
|
||||
def _body(self, interval=5, url='http://example.org/newpage'):
|
||||
html = u"""<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
|
||||
html = """<html><head><meta http-equiv="refresh" content="{0};url={1}"/></head></html>"""
|
||||
return html.format(interval, url).encode('utf-8')
|
||||
|
||||
def test_priority_adjust(self):
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
|
|||
def _get_successful_crawler(self):
|
||||
crawler = self.crawler
|
||||
crawler.settings.set('ROBOTSTXT_OBEY', True)
|
||||
ROBOTS = u"""
|
||||
ROBOTS = """
|
||||
User-Agent: *
|
||||
Disallow: /admin/
|
||||
Disallow: /static/
|
||||
|
|
@ -56,7 +56,7 @@ Disallow: /some/randome/page.html
|
|||
self.assertIgnored(Request('http://site.local/admin/main'), middleware),
|
||||
self.assertIgnored(Request('http://site.local/static/'), middleware),
|
||||
self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
|
||||
self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
|
||||
self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
|
||||
], fireOnOneErrback=True)
|
||||
|
||||
def test_robotstxt_ready_parser(self):
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
def setUp(self):
|
||||
if self.item_class is None:
|
||||
raise unittest.SkipTest("item class is None")
|
||||
self.i = self.item_class(name=u'John\xa3', age=u'22')
|
||||
self.i = self.item_class(name='John\xa3', age='22')
|
||||
self.output = BytesIO()
|
||||
self.ie = self._get_exporter()
|
||||
|
||||
|
|
@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase):
|
|||
def test_serialize_field(self):
|
||||
a = ItemAdapter(self.i)
|
||||
res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
|
||||
self.assertEqual(res, u'John\xa3')
|
||||
self.assertEqual(res, 'John\xa3')
|
||||
|
||||
res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
|
||||
self.assertEqual(res, u'22')
|
||||
self.assertEqual(res, '22')
|
||||
|
||||
def test_fields_to_export(self):
|
||||
ie = self._get_exporter(fields_to_export=['name'])
|
||||
self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
|
||||
self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
|
||||
|
||||
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
|
||||
_, name = list(ie._get_serialized_fields(self.i))[0]
|
||||
assert isinstance(name, str)
|
||||
self.assertEqual(name, u'John\xa3')
|
||||
self.assertEqual(name, 'John\xa3')
|
||||
|
||||
def test_field_custom_serializer(self):
|
||||
i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
|
||||
i = self.custom_field_item_class(name='John\xa3', age='22')
|
||||
a = ItemAdapter(i)
|
||||
ie = self._get_exporter()
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
|
||||
|
||||
|
||||
|
|
@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):
|
|||
PythonItemExporter(invalid_option='something')
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = dict(name=u'Maria', age=i1)
|
||||
i3 = self.item_class(name=u'Jesus', age=i2)
|
||||
i1 = self.item_class(name='Joseph', age='22')
|
||||
i2 = dict(name='Maria', age=i1)
|
||||
i3 = self.item_class(name='Jesus', age=i2)
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(type(exported), dict)
|
||||
self.assertEqual(
|
||||
exported,
|
||||
{'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
|
||||
{'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
|
||||
)
|
||||
self.assertEqual(type(exported['age']), dict)
|
||||
self.assertEqual(type(exported['age']['age']), dict)
|
||||
|
||||
def test_export_list(self):
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = self.item_class(name=u'Maria', age=[i1])
|
||||
i3 = self.item_class(name=u'Jesus', age=[i2])
|
||||
i1 = self.item_class(name='Joseph', age='22')
|
||||
i2 = self.item_class(name='Maria', age=[i1])
|
||||
i3 = self.item_class(name='Jesus', age=[i2])
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(
|
||||
exported,
|
||||
{'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
|
||||
{'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
|
||||
)
|
||||
self.assertEqual(type(exported['age'][0]), dict)
|
||||
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
|
||||
|
||||
def test_export_item_dict_list(self):
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = dict(name=u'Maria', age=[i1])
|
||||
i3 = self.item_class(name=u'Jesus', age=[i2])
|
||||
i1 = self.item_class(name='Joseph', age='22')
|
||||
i2 = dict(name='Maria', age=[i1])
|
||||
i3 = self.item_class(name='Jesus', age=[i2])
|
||||
ie = self._get_exporter()
|
||||
exported = ie.export_item(i3)
|
||||
self.assertEqual(
|
||||
exported,
|
||||
{'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
|
||||
{'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
|
||||
)
|
||||
self.assertEqual(type(exported['age'][0]), dict)
|
||||
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
|
||||
|
||||
def test_export_binary(self):
|
||||
exporter = PythonItemExporter(binary=True)
|
||||
value = self.item_class(name=u'John\xa3', age=u'22')
|
||||
value = self.item_class(name='John\xa3', age='22')
|
||||
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
|
||||
self.assertEqual(expected, exporter.export_item(value))
|
||||
|
||||
|
|
@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
|
||||
|
||||
def _check_output(self):
|
||||
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
|
||||
self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
|
||||
|
||||
def assertExportResult(self, item, expected, **kwargs):
|
||||
fp = BytesIO()
|
||||
|
|
@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
|
||||
def test_multivalued_fields(self):
|
||||
self.assertExportResult(
|
||||
self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
|
||||
self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
|
||||
b"""<?xml version="1.0" encoding="utf-8"?>\n
|
||||
<items>
|
||||
<item>
|
||||
|
|
@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
)
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = dict(name=u'foo\xa3hoo', age='22')
|
||||
i2 = dict(name=u'bar', age=i1)
|
||||
i3 = self.item_class(name=u'buz', age=i2)
|
||||
i1 = dict(name='foo\xa3hoo', age='22')
|
||||
i2 = dict(name='bar', age=i1)
|
||||
i3 = self.item_class(name='buz', age=i2)
|
||||
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
|
|
@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
)
|
||||
|
||||
def test_nested_list_item(self):
|
||||
i1 = dict(name=u'foo')
|
||||
i2 = dict(name=u'bar', v2={"egg": ["spam"]})
|
||||
i3 = self.item_class(name=u'buz', age=[i1, i2])
|
||||
i1 = dict(name='foo')
|
||||
i2 = dict(name='bar', v2={"egg": ["spam"]})
|
||||
i3 = self.item_class(name='buz', age=[i1, i2])
|
||||
|
||||
self.assertExportResult(
|
||||
i3,
|
||||
|
|
@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):
|
|||
|
||||
class JsonLinesItemExporterTest(BaseItemExporterTest):
|
||||
|
||||
_expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
|
||||
_expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
return JsonLinesItemExporter(self.output, **kwargs)
|
||||
|
|
@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
|
|||
self.assertEqual(exported, ItemAdapter(self.i).asdict())
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = self.item_class(name=u'Joseph', age='22')
|
||||
i2 = dict(name=u'Maria', age=i1)
|
||||
i3 = self.item_class(name=u'Jesus', age=i2)
|
||||
i1 = self.item_class(name='Joseph', age='22')
|
||||
i2 = dict(name='Maria', age=i1)
|
||||
i3 = self.item_class(name='Jesus', age=i2)
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(i3)
|
||||
self.ie.finish_exporting()
|
||||
|
|
@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
|
|||
self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
|
||||
|
||||
def test_nested_item(self):
|
||||
i1 = self.item_class(name=u'Joseph\xa3', age='22')
|
||||
i2 = self.item_class(name=u'Maria', age=i1)
|
||||
i3 = self.item_class(name=u'Jesus', age=i2)
|
||||
i1 = self.item_class(name='Joseph\xa3', age='22')
|
||||
i2 = self.item_class(name='Maria', age=i1)
|
||||
i3 = self.item_class(name='Jesus', age=i2)
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(i3)
|
||||
self.ie.finish_exporting()
|
||||
exported = json.loads(to_unicode(self.output.getvalue()))
|
||||
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
|
||||
expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
|
||||
self.assertEqual(exported, [expected])
|
||||
|
||||
def test_nested_dict_item(self):
|
||||
i1 = dict(name=u'Joseph\xa3', age='22')
|
||||
i2 = self.item_class(name=u'Maria', age=i1)
|
||||
i3 = dict(name=u'Jesus', age=i2)
|
||||
i1 = dict(name='Joseph\xa3', age='22')
|
||||
i2 = self.item_class(name='Maria', age=i1)
|
||||
i3 = dict(name='Jesus', age=i2)
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(i3)
|
||||
self.ie.finish_exporting()
|
||||
exported = json.loads(to_unicode(self.output.getvalue()))
|
||||
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
|
||||
expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
|
||||
self.assertEqual(exported, [expected])
|
||||
|
||||
def test_nonstring_types_item(self):
|
||||
|
|
@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase):
|
|||
else:
|
||||
return super(CustomItemExporter, self).serialize_field(field, name, value)
|
||||
|
||||
i = self.item_class(name=u'John', age='22')
|
||||
i = self.item_class(name='John', age='22')
|
||||
a = ItemAdapter(i)
|
||||
ie = CustomItemExporter()
|
||||
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
|
||||
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
|
||||
|
||||
i2 = {'name': u'John', 'age': '22'}
|
||||
i2 = {'name': 'John', 'age': '22'}
|
||||
self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
|
||||
self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
|
||||
|
||||
|
|
|
|||
|
|
@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_encoding(self):
|
||||
items = [dict({'foo': u'Test\xd6'})]
|
||||
items = [dict({'foo': 'Test\xd6'})]
|
||||
|
||||
formats = {
|
||||
'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
|
||||
|
|
@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_multiple_configs(self):
|
||||
items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
|
||||
items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
|
||||
|
||||
formats = {
|
||||
'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
|
||||
|
|
@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_multiple_configs(self):
|
||||
items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
|
||||
items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
|
||||
|
||||
formats = {
|
||||
'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
|
||||
|
|
@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_batch_item_count_feeds_setting(self):
|
||||
items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
|
||||
items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
|
||||
formats = {
|
||||
'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
|
||||
'[{"foo": "FOO1"}]'.encode('utf-8')],
|
||||
|
|
|
|||
|
|
@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):
|
|||
assert h.getlist('X-Forwarded-For') is not hlist
|
||||
|
||||
def test_encode_utf8(self):
|
||||
h = Headers({u'key': u'\xa3'}, encoding='utf-8')
|
||||
h = Headers({'key': '\xa3'}, encoding='utf-8')
|
||||
key, val = dict(h).popitem()
|
||||
assert isinstance(key, bytes), key
|
||||
assert isinstance(val[0], bytes), val[0]
|
||||
self.assertEqual(val[0], b'\xc2\xa3')
|
||||
|
||||
def test_encode_latin1(self):
|
||||
h = Headers({u'key': u'\xa3'}, encoding='latin1')
|
||||
h = Headers({'key': '\xa3'}, encoding='latin1')
|
||||
key, val = dict(h).popitem()
|
||||
self.assertEqual(val[0], b'\xa3')
|
||||
|
||||
def test_encode_multiple(self):
|
||||
h = Headers({u'key': [u'\xa3']}, encoding='utf-8')
|
||||
h = Headers({'key': ['\xa3']}, encoding='utf-8')
|
||||
key, val = dict(h).popitem()
|
||||
self.assertEqual(val[0], b'\xc2\xa3')
|
||||
|
||||
|
|
|
|||
|
|
@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):
|
|||
self.assertFalse(p.headers is r.headers)
|
||||
|
||||
# headers must not be unicode
|
||||
h = Headers({'key1': u'val1', u'key2': 'val2'})
|
||||
h[u'newkey'] = u'newval'
|
||||
h = Headers({'key1': 'val1', 'key2': 'val2'})
|
||||
h['newkey'] = 'newval'
|
||||
for k, v in h.items():
|
||||
self.assertIsInstance(k, bytes)
|
||||
for s in v:
|
||||
|
|
@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):
|
|||
self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
|
||||
|
||||
def test_url_encoding(self):
|
||||
r = self.request_class(url=u"http://www.scrapy.org/price/£")
|
||||
r = self.request_class(url="http://www.scrapy.org/price/£")
|
||||
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
|
||||
|
||||
def test_url_encoding_other(self):
|
||||
# encoding affects only query part of URI, not path
|
||||
# path part should always be UTF-8 encoded before percent-escaping
|
||||
r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
|
||||
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
|
||||
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
|
||||
|
||||
r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
|
||||
r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
|
||||
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
|
||||
|
||||
def test_url_encoding_query(self):
|
||||
r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
|
||||
r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
|
||||
self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
|
||||
|
||||
# should be same as above
|
||||
r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
|
||||
r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
|
||||
self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
|
||||
|
||||
def test_url_encoding_query_latin1(self):
|
||||
# encoding is used for encoding query-string before percent-escaping;
|
||||
# path is still UTF-8 encoded before percent-escaping
|
||||
r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
|
||||
r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
|
||||
self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
|
||||
|
||||
def test_url_encoding_nonutf8_untouched(self):
|
||||
|
|
@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):
|
|||
# characters. Otherwise, in the future the IRI will be mapped to
|
||||
# "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
|
||||
# URI from "http://www.example.org/r%E9sum%E9.html".
|
||||
r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
|
||||
r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
|
||||
self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
|
||||
|
||||
r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
|
||||
r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
|
||||
self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
|
||||
|
||||
r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
|
||||
r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
|
||||
self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
|
||||
|
||||
r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
|
||||
r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
|
||||
self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
|
||||
|
||||
def test_body(self):
|
||||
|
|
@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):
|
|||
assert isinstance(r2.body, bytes)
|
||||
self.assertEqual(r2.encoding, 'utf-8') # default encoding
|
||||
|
||||
r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
|
||||
r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
|
||||
assert isinstance(r3.body, bytes)
|
||||
self.assertEqual(r3.body, b"Price: \xc2\xa3100")
|
||||
|
||||
r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
|
||||
r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
|
||||
assert isinstance(r4.body, bytes)
|
||||
self.assertEqual(r4.body, b"Price: \xa3100")
|
||||
|
||||
|
|
@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):
|
|||
r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
|
||||
self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
|
||||
# unicode url
|
||||
r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
|
||||
r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
|
||||
self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
|
||||
|
||||
def test_copy(self):
|
||||
|
|
@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):
|
|||
assert r4.dont_filter is False
|
||||
|
||||
def test_method_always_str(self):
|
||||
r = self.request_class("http://www.example.com", method=u"POST")
|
||||
r = self.request_class("http://www.example.com", method="POST")
|
||||
assert isinstance(r.method, str)
|
||||
|
||||
def test_immutable_attributes(self):
|
||||
|
|
@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_default_encoding_textual_data(self):
|
||||
# using default encoding (utf-8)
|
||||
data = {u'µ one': u'two', u'price': u'£ 100'}
|
||||
data = {'µ one': 'two', 'price': '£ 100'}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data)
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'utf-8')
|
||||
|
|
@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_default_encoding_mixed_data(self):
|
||||
# using default encoding (utf-8)
|
||||
data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
|
||||
data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data)
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'utf-8')
|
||||
|
|
@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):
|
|||
self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
|
||||
|
||||
def test_custom_encoding_textual_data(self):
|
||||
data = {'price': u'£ 100'}
|
||||
data = {'price': '£ 100'}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r3.encoding, 'latin1')
|
||||
self.assertEqual(r3.body, b'price=%A3+100')
|
||||
|
||||
def test_multi_key_values(self):
|
||||
# using multiples values for a single key
|
||||
data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
|
||||
data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data)
|
||||
self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
|
||||
|
||||
|
|
@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):
|
|||
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
|
||||
self.assertEqual(req.url, "http://www.example.com/this/post.php")
|
||||
fs = _qs(req, to_unicode=True)
|
||||
self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
|
||||
self.assertEqual(set(fs[u'one']), {u'two', u'three'})
|
||||
self.assertEqual(fs[u'test2'], [u'xxx µ'])
|
||||
self.assertEqual(fs[u'six'], [u'seven'])
|
||||
self.assertEqual(set(fs['test £']), {'val1', 'val2'})
|
||||
self.assertEqual(set(fs['one']), {'two', 'three'})
|
||||
self.assertEqual(fs['test2'], ['xxx µ'])
|
||||
self.assertEqual(fs['six'], ['seven'])
|
||||
|
||||
def test_from_response_post_nonascii_bytes_latin1(self):
|
||||
response = _buildresponse(
|
||||
|
|
@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):
|
|||
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
|
||||
self.assertEqual(req.url, "http://www.example.com/this/post.php")
|
||||
fs = _qs(req, to_unicode=True, encoding='latin1')
|
||||
self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
|
||||
self.assertEqual(set(fs[u'one']), {u'two', u'three'})
|
||||
self.assertEqual(fs[u'test2'], [u'xxx µ'])
|
||||
self.assertEqual(fs[u'six'], [u'seven'])
|
||||
self.assertEqual(set(fs['test £']), {'val1', 'val2'})
|
||||
self.assertEqual(set(fs['one']), {'two', 'three'})
|
||||
self.assertEqual(fs['test2'], ['xxx µ'])
|
||||
self.assertEqual(fs['six'], ['seven'])
|
||||
|
||||
def test_from_response_post_nonascii_unicode(self):
|
||||
response = _buildresponse(
|
||||
u"""<form action="post.php" method="POST">
|
||||
"""<form action="post.php" method="POST">
|
||||
<input type="hidden" name="test £" value="val1">
|
||||
<input type="hidden" name="test £" value="val2">
|
||||
<input type="hidden" name="test2" value="xxx µ">
|
||||
|
|
@ -490,10 +490,10 @@ class FormRequestTest(RequestTest):
|
|||
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
|
||||
self.assertEqual(req.url, "http://www.example.com/this/post.php")
|
||||
fs = _qs(req, to_unicode=True)
|
||||
self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
|
||||
self.assertEqual(set(fs[u'one']), {u'two', u'three'})
|
||||
self.assertEqual(fs[u'test2'], [u'xxx µ'])
|
||||
self.assertEqual(fs[u'six'], [u'seven'])
|
||||
self.assertEqual(set(fs['test £']), {'val1', 'val2'})
|
||||
self.assertEqual(set(fs['one']), {'two', 'three'})
|
||||
self.assertEqual(fs['test2'], ['xxx µ'])
|
||||
self.assertEqual(fs['six'], ['seven'])
|
||||
|
||||
def test_from_response_duplicate_form_key(self):
|
||||
response = _buildresponse(
|
||||
|
|
@ -685,7 +685,7 @@ class FormRequestTest(RequestTest):
|
|||
<input type="hidden" name="two" value="clicked2">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(
|
||||
response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
|
||||
response, clickdata={'name': 'clickable', 'value': 'clicked2'}
|
||||
)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs[b'clickable'], [b'clicked2'])
|
||||
|
|
@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_from_response_unicode_clickdata(self):
|
||||
response = _buildresponse(
|
||||
u"""<form action="get.php" method="GET">
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="price in \u00a3" value="\u00a3 1000">
|
||||
<input type="submit" name="price in \u20ac" value="\u20ac 2000">
|
||||
<input type="hidden" name="poundsign" value="\u00a3">
|
||||
<input type="hidden" name="eurosign" value="\u20ac">
|
||||
</form>""")
|
||||
req = self.request_class.from_response(
|
||||
response, clickdata={u'name': u'price in \u00a3'}
|
||||
response, clickdata={'name': 'price in \u00a3'}
|
||||
)
|
||||
fs = _qs(req, to_unicode=True)
|
||||
self.assertTrue(fs[u'price in \u00a3'])
|
||||
self.assertTrue(fs['price in \u00a3'])
|
||||
|
||||
def test_from_response_unicode_clickdata_latin1(self):
|
||||
response = _buildresponse(
|
||||
u"""<form action="get.php" method="GET">
|
||||
"""<form action="get.php" method="GET">
|
||||
<input type="submit" name="price in \u00a3" value="\u00a3 1000">
|
||||
<input type="submit" name="price in \u00a5" value="\u00a5 2000">
|
||||
<input type="hidden" name="poundsign" value="\u00a3">
|
||||
|
|
@ -716,10 +716,10 @@ class FormRequestTest(RequestTest):
|
|||
</form>""",
|
||||
encoding='latin1')
|
||||
req = self.request_class.from_response(
|
||||
response, clickdata={u'name': u'price in \u00a5'}
|
||||
response, clickdata={'name': 'price in \u00a5'}
|
||||
)
|
||||
fs = _qs(req, to_unicode=True, encoding='latin1')
|
||||
self.assertTrue(fs[u'price in \u00a5'])
|
||||
self.assertTrue(fs['price in \u00a5'])
|
||||
|
||||
def test_from_response_multiple_forms_clickdata(self):
|
||||
response = _buildresponse(
|
||||
|
|
@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):
|
|||
</form>
|
||||
""")
|
||||
req = self.request_class.from_response(
|
||||
response, formname='form2', clickdata={u'name': u'clickable'}
|
||||
response, formname='form2', clickdata={'name': 'clickable'}
|
||||
)
|
||||
fs = _qs(req)
|
||||
self.assertEqual(fs[b'clickable'], [b'clicked2'])
|
||||
|
|
@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):
|
|||
|
||||
def test_from_response_unicode_xpath(self):
|
||||
response = _buildresponse(b'<form name="\xd1\x8a"></form>')
|
||||
r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
|
||||
r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
|
||||
fs = _qs(r)
|
||||
self.assertEqual(fs, {})
|
||||
|
||||
xpath = u"//form[@name='\u03b1']"
|
||||
xpath = "//form[@name='\u03b1']"
|
||||
self.assertRaisesRegex(ValueError, re.escape(xpath),
|
||||
self.request_class.from_response,
|
||||
response, formxpath=xpath)
|
||||
|
|
@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):
|
|||
self._test_request(params=('value',))
|
||||
self._test_request(params=('username', 'password'), methodname='login')
|
||||
self._test_request(params=('response', ), methodresponse='login')
|
||||
self._test_request(params=(u'pas£',), encoding='utf-8')
|
||||
self._test_request(params=('pas£',), encoding='utf-8')
|
||||
self._test_request(params=(None,), allow_none=1)
|
||||
self.assertRaises(TypeError, self._test_request)
|
||||
self.assertRaises(TypeError, self._test_request, params=(None,))
|
||||
|
||||
def test_latin1(self):
|
||||
self._test_request(params=(u'pas£',), encoding='latin1')
|
||||
self._test_request(params=('pas£',), encoding='latin1')
|
||||
|
||||
|
||||
class JsonRequestTest(RequestTest):
|
||||
|
|
|
|||
|
|
@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):
|
|||
|
||||
def test_unicode_url(self):
|
||||
# instantiate with unicode url without encoding (should set default encoding)
|
||||
resp = self.response_class(u"http://www.example.com/")
|
||||
resp = self.response_class("http://www.example.com/")
|
||||
self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
|
||||
|
||||
# make sure urls are converted to str
|
||||
resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
|
||||
resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
|
||||
assert isinstance(resp.url, str)
|
||||
|
||||
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
|
||||
resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
|
||||
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
|
||||
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
|
||||
resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3",
|
||||
resp = self.response_class("http://www.example.com/price/\xa3",
|
||||
headers={"Content-type": ["text/html; charset=utf-8"]})
|
||||
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3",
|
||||
resp = self.response_class("http://www.example.com/price/\xa3",
|
||||
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
|
||||
|
||||
def test_unicode_body(self):
|
||||
unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
|
||||
'\u0442\u0435\u043a\u0441\u0442')
|
||||
self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
|
||||
self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
|
||||
|
||||
original_string = unicode_string.encode('cp1251')
|
||||
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
|
||||
|
|
@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
def test_encoding(self):
|
||||
r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
|
||||
headers={"Content-type": ["text/html; charset=utf-8"]})
|
||||
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
|
||||
r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
|
||||
r3 = self.response_class("http://www.example.com", body=b"\xa3",
|
||||
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
|
||||
|
|
@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):
|
|||
self.assertEqual(r5._headers_encoding(), None)
|
||||
self._assert_response_encoding(r5, "utf-8")
|
||||
assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
|
||||
self._assert_response_values(r1, 'utf-8', u"\xa3")
|
||||
self._assert_response_values(r2, 'utf-8', u"\xa3")
|
||||
self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
|
||||
self._assert_response_values(r6, 'gb18030', u"\u2015")
|
||||
self._assert_response_values(r7, 'gb18030', u"\u2015")
|
||||
self._assert_response_values(r1, 'utf-8', "\xa3")
|
||||
self._assert_response_values(r2, 'utf-8', "\xa3")
|
||||
self._assert_response_values(r3, 'iso-8859-1', "\xa3")
|
||||
self._assert_response_values(r6, 'gb18030', "\u2015")
|
||||
self._assert_response_values(r7, 'gb18030', "\u2015")
|
||||
|
||||
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
|
||||
self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
|
||||
self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
|
||||
|
||||
def test_declared_encoding_invalid(self):
|
||||
"""Check that unknown declared encodings are ignored"""
|
||||
|
|
@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):
|
|||
headers={"Content-type": ["text/html; charset=UKNOWN"]},
|
||||
body=b"\xc2\xa3")
|
||||
self.assertEqual(r._declared_encoding(), None)
|
||||
self._assert_response_values(r, 'utf-8', u"\xa3")
|
||||
self._assert_response_values(r, 'utf-8', "\xa3")
|
||||
|
||||
def test_utf16(self):
|
||||
"""Test utf-16 because UnicodeDammit is known to have problems with"""
|
||||
r = self.response_class("http://www.example.com",
|
||||
body=b'\xff\xfeh\x00i\x00',
|
||||
encoding='utf-16')
|
||||
self._assert_response_values(r, 'utf-16', u"hi")
|
||||
self._assert_response_values(r, 'utf-16', "hi")
|
||||
|
||||
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
|
||||
r6 = self.response_class("http://www.example.com",
|
||||
|
|
@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):
|
|||
body=b"\xef\xbb\xbfWORD\xe3\xab")
|
||||
self.assertEqual(r6.encoding, 'utf-8')
|
||||
self.assertIn(r6.text, {
|
||||
u'WORD\ufffd\ufffd', # w3lib < 1.19.0
|
||||
u'WORD\ufffd', # w3lib >= 1.19.0
|
||||
'WORD\ufffd\ufffd', # w3lib < 1.19.0
|
||||
'WORD\ufffd', # w3lib >= 1.19.0
|
||||
})
|
||||
|
||||
def test_bom_is_removed_from_body(self):
|
||||
|
|
@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):
|
|||
# Test response without content-type and BOM encoding
|
||||
response = self.response_class(url, body=body)
|
||||
self.assertEqual(response.encoding, 'utf-8')
|
||||
self.assertEqual(response.text, u'WORD')
|
||||
self.assertEqual(response.text, 'WORD')
|
||||
response = self.response_class(url, body=body)
|
||||
self.assertEqual(response.text, u'WORD')
|
||||
self.assertEqual(response.text, 'WORD')
|
||||
self.assertEqual(response.encoding, 'utf-8')
|
||||
|
||||
# Body caching sideeffect isn't triggered when encoding is declared in
|
||||
|
|
@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):
|
|||
# body
|
||||
response = self.response_class(url, headers=headers, body=body)
|
||||
self.assertEqual(response.encoding, 'utf-8')
|
||||
self.assertEqual(response.text, u'WORD')
|
||||
self.assertEqual(response.text, 'WORD')
|
||||
response = self.response_class(url, headers=headers, body=body)
|
||||
self.assertEqual(response.text, u'WORD')
|
||||
self.assertEqual(response.text, 'WORD')
|
||||
self.assertEqual(response.encoding, 'utf-8')
|
||||
|
||||
def test_replace_wrong_encoding(self):
|
||||
"""Test invalid chars are replaced properly"""
|
||||
r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
|
||||
# XXX: Policy for replacing invalid chars may suffer minor variations
|
||||
# but it should always contain the unicode replacement char (u'\ufffd')
|
||||
assert u'\ufffd' in r.text, repr(r.text)
|
||||
assert u'PREFIX' in r.text, repr(r.text)
|
||||
assert u'SUFFIX' in r.text, repr(r.text)
|
||||
# but it should always contain the unicode replacement char ('\ufffd')
|
||||
assert '\ufffd' in r.text, repr(r.text)
|
||||
assert 'PREFIX' in r.text, repr(r.text)
|
||||
assert 'SUFFIX' in r.text, repr(r.text)
|
||||
|
||||
# Do not destroy html tags due to encoding bugs
|
||||
r = self.response_class("http://example.com", encoding='utf-8',
|
||||
body=b'\xf0<span>value</span>')
|
||||
assert u'<span>value</span>' in r.text, repr(r.text)
|
||||
assert '<span>value</span>' in r.text, repr(r.text)
|
||||
|
||||
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
|
||||
# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
|
||||
# assert u'\ufffd' in r.text, repr(r.text)
|
||||
# assert '\ufffd' in r.text, repr(r.text)
|
||||
|
||||
def test_selector(self):
|
||||
body = b"<html><head><title>Some page</title><body></body></html>"
|
||||
|
|
@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):
|
|||
|
||||
self.assertEqual(
|
||||
response.selector.xpath("//title/text()").getall(),
|
||||
[u'Some page']
|
||||
['Some page']
|
||||
)
|
||||
self.assertEqual(
|
||||
response.selector.css("title::text").getall(),
|
||||
[u'Some page']
|
||||
['Some page']
|
||||
)
|
||||
self.assertEqual(
|
||||
response.selector.re("Some (.*)</title>"),
|
||||
[u'page']
|
||||
['page']
|
||||
)
|
||||
|
||||
def test_selector_shortcuts(self):
|
||||
|
|
@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
resp1 = self.response_class(
|
||||
'http://example.com',
|
||||
encoding='utf8',
|
||||
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
|
||||
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
|
||||
)
|
||||
req = self._assert_followed_url(
|
||||
resp1.css('a')[0],
|
||||
|
|
@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):
|
|||
resp2 = self.response_class(
|
||||
'http://example.com',
|
||||
encoding='cp1251',
|
||||
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
|
||||
body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
|
||||
)
|
||||
req = self._assert_followed_url(
|
||||
resp2.css('a')[0],
|
||||
|
|
@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):
|
|||
|
||||
def test_body_as_unicode_deprecation_warning(self):
|
||||
with catch_warnings(record=True) as warnings:
|
||||
r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
|
||||
self.assertEqual(r1.body_as_unicode(), u'Hello')
|
||||
r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
|
||||
self.assertEqual(r1.body_as_unicode(), 'Hello')
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
|
||||
|
||||
|
|
@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):
|
|||
|
||||
self.assertEqual(
|
||||
response.selector.xpath("//elem/text()").getall(),
|
||||
[u'value']
|
||||
['value']
|
||||
)
|
||||
|
||||
def test_selector_shortcuts(self):
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):
|
|||
name = Field()
|
||||
|
||||
i = TestItem()
|
||||
i['name'] = u'name'
|
||||
self.assertEqual(i['name'], u'name')
|
||||
i['name'] = 'name'
|
||||
self.assertEqual(i['name'], 'name')
|
||||
|
||||
def test_init(self):
|
||||
class TestItem(Item):
|
||||
|
|
@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):
|
|||
i = TestItem()
|
||||
self.assertRaises(KeyError, i.__getitem__, 'name')
|
||||
|
||||
i2 = TestItem(name=u'john doe')
|
||||
self.assertEqual(i2['name'], u'john doe')
|
||||
i2 = TestItem(name='john doe')
|
||||
self.assertEqual(i2['name'], 'john doe')
|
||||
|
||||
i3 = TestItem({'name': u'john doe'})
|
||||
self.assertEqual(i3['name'], u'john doe')
|
||||
i3 = TestItem({'name': 'john doe'})
|
||||
self.assertEqual(i3['name'], 'john doe')
|
||||
|
||||
i4 = TestItem(i3)
|
||||
self.assertEqual(i4['name'], u'john doe')
|
||||
self.assertEqual(i4['name'], 'john doe')
|
||||
|
||||
self.assertRaises(KeyError, TestItem, {'name': u'john doe',
|
||||
'other': u'foo'})
|
||||
self.assertRaises(KeyError, TestItem, {'name': 'john doe',
|
||||
'other': 'foo'})
|
||||
|
||||
def test_invalid_field(self):
|
||||
class TestItem(Item):
|
||||
|
|
@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):
|
|||
number = Field()
|
||||
|
||||
i = TestItem()
|
||||
i['name'] = u'John Doe'
|
||||
i['name'] = 'John Doe'
|
||||
i['number'] = 123
|
||||
itemrepr = repr(i)
|
||||
|
||||
|
|
@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):
|
|||
|
||||
i = TestItem()
|
||||
self.assertRaises(KeyError, i.get_name)
|
||||
i['name'] = u'lala'
|
||||
self.assertEqual(i.get_name(), u'lala')
|
||||
i.change_name(u'other')
|
||||
i['name'] = 'lala'
|
||||
self.assertEqual(i.get_name(), 'lala')
|
||||
i.change_name('other')
|
||||
self.assertEqual(i.get_name(), 'other')
|
||||
|
||||
def test_metaclass(self):
|
||||
|
|
@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):
|
|||
values = Field()
|
||||
|
||||
i = TestItem()
|
||||
i['name'] = u'John'
|
||||
i['name'] = 'John'
|
||||
self.assertEqual(list(i.keys()), ['name'])
|
||||
self.assertEqual(list(i.values()), ['John'])
|
||||
|
||||
i['keys'] = u'Keys'
|
||||
i['values'] = u'Values'
|
||||
i['keys'] = 'Keys'
|
||||
i['values'] = 'Values'
|
||||
self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
|
||||
self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
|
||||
self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
|
||||
|
||||
def test_metaclass_with_fields_attribute(self):
|
||||
class TestItem(Item):
|
||||
fields = {'new': Field(default='X')}
|
||||
|
||||
item = TestItem(new=u'New')
|
||||
item = TestItem(new='New')
|
||||
self.assertSortedEqual(list(item.keys()), ['new'])
|
||||
self.assertSortedEqual(list(item.values()), [u'New'])
|
||||
self.assertSortedEqual(list(item.values()), ['New'])
|
||||
|
||||
def test_metaclass_inheritance(self):
|
||||
class ParentItem(Item):
|
||||
|
|
@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):
|
|||
name = Field()
|
||||
|
||||
i = TestItem()
|
||||
i['name'] = u'John'
|
||||
self.assertEqual(dict(i), {'name': u'John'})
|
||||
i['name'] = 'John'
|
||||
self.assertEqual(dict(i), {'name': 'John'})
|
||||
|
||||
def test_copy(self):
|
||||
class TestItem(Item):
|
||||
|
|
|
|||
|
|
@ -31,31 +31,31 @@ class Base:
|
|||
page4_url = 'http://example.com/page%204.html'
|
||||
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url=page4_url, text=u'href with whitespaces'),
|
||||
Link(url='http://www.google.com/something', text=''),
|
||||
Link(url='http://example.com/innertag.html', text='inner tag'),
|
||||
Link(url=page4_url, text='href with whitespaces'),
|
||||
])
|
||||
|
||||
def test_extract_filter_allow(self):
|
||||
lx = self.extractor_cls(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
|
||||
])
|
||||
|
||||
def test_extract_filter_allow_with_duplicates(self):
|
||||
lx = self.extractor_cls(allow=('sample', ), unique=False)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
|
||||
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
|
||||
])
|
||||
|
||||
|
|
@ -63,10 +63,10 @@ class Base:
|
|||
lx = self.extractor_cls(allow=('sample', ), unique=False,
|
||||
canonicalize=True)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
|
||||
])
|
||||
|
||||
|
|
@ -74,22 +74,22 @@ class Base:
|
|||
lx = self.extractor_cls(allow=('sample',), unique=True,
|
||||
canonicalize=True)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
])
|
||||
|
||||
def test_extract_filter_allow_and_deny(self):
|
||||
lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
])
|
||||
|
||||
def test_extract_filter_allowed_domains(self):
|
||||
lx = self.extractor_cls(allow_domains=('google.com', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://www.google.com/something', text=''),
|
||||
])
|
||||
|
||||
def test_extraction_using_single_values(self):
|
||||
|
|
@ -97,27 +97,27 @@ class Base:
|
|||
|
||||
lx = self.extractor_cls(allow='sample')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html#foo',
|
||||
text='sample 3 repetition with fragment')
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(allow='sample', deny='3')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(allow_domains='google.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://www.google.com/something', text=''),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(deny_domains='example.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://www.google.com/something', text=''),
|
||||
])
|
||||
|
||||
def test_nofollow(self):
|
||||
|
|
@ -145,11 +145,11 @@ class Base:
|
|||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/about.html', text=u'About us'),
|
||||
Link(url='http://example.org/follow.html', text=u'Follow this link'),
|
||||
Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
|
||||
Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
Link(url='http://example.org/about.html', text='About us'),
|
||||
Link(url='http://example.org/follow.html', text='Follow this link'),
|
||||
Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
|
||||
Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
|
||||
Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
|
||||
])
|
||||
|
||||
def test_matches(self):
|
||||
|
|
@ -183,8 +183,8 @@ class Base:
|
|||
def test_restrict_xpaths(self):
|
||||
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
])
|
||||
|
||||
def test_restrict_xpaths_encoding(self):
|
||||
|
|
@ -202,14 +202,14 @@ class Base:
|
|||
|
||||
lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
|
||||
[Link(url='http://example.org/about.html', text='About us\xa3')])
|
||||
|
||||
def test_restrict_xpaths_with_html_entities(self):
|
||||
html = b'<html><body><p><a href="/♥/you?c=€">text</a></p></body></html>'
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
|
||||
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
|
||||
self.assertEqual(links,
|
||||
[Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
|
||||
[Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
|
||||
|
||||
def test_restrict_xpaths_concat_in_handle_data(self):
|
||||
"""html entities cause SGMLParser to call handle_data hook twice"""
|
||||
|
|
@ -217,22 +217,22 @@ class Base:
|
|||
response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
|
||||
lx = self.extractor_cls(restrict_xpaths="//div")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
|
||||
[Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
|
||||
fragment='', nofollow=False)])
|
||||
|
||||
def test_restrict_css(self):
|
||||
lx = self.extractor_cls(restrict_css=('#subwrapper a',))
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2')
|
||||
Link(url='http://example.com/sample2.html', text='sample 2')
|
||||
])
|
||||
|
||||
def test_restrict_css_and_restrict_xpaths_together(self):
|
||||
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
|
||||
restrict_css=('#subwrapper + a', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
])
|
||||
|
||||
def test_area_tag_with_unicode_present(self):
|
||||
|
|
@ -243,7 +243,7 @@ class Base:
|
|||
lx.extract_links(response)
|
||||
lx.extract_links(response)
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/foo', text=u'',
|
||||
[Link(url='http://example.org/foo', text='',
|
||||
fragment='', nofollow=False)])
|
||||
|
||||
def test_encoded_url(self):
|
||||
|
|
@ -251,7 +251,7 @@ class Base:
|
|||
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_encoded_url_in_restricted_xpath(self):
|
||||
|
|
@ -259,7 +259,7 @@ class Base:
|
|||
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls(restrict_xpaths="//div")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
|
||||
Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_ignored_extensions(self):
|
||||
|
|
@ -268,7 +268,7 @@ class Base:
|
|||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
Link(url='http://example.org/page.html', text='asd'),
|
||||
])
|
||||
|
||||
# override denied extensions
|
||||
|
|
@ -308,25 +308,25 @@ class Base:
|
|||
page4_url = 'http://example.com/page%204.html'
|
||||
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url=page4_url, text=u'href with whitespaces'),
|
||||
Link(url='http://www.google.com/something', text=''),
|
||||
Link(url='http://example.com/innertag.html', text='inner tag'),
|
||||
Link(url=page4_url, text='href with whitespaces'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u''),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=''),
|
||||
Link(url='http://example.com/sample3.html', text='sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url=page4_url, text=u'href with whitespaces'),
|
||||
Link(url='http://www.google.com/something', text=''),
|
||||
Link(url='http://example.com/innertag.html', text='inner tag'),
|
||||
Link(url=page4_url, text='href with whitespaces'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=None)
|
||||
|
|
@ -344,24 +344,24 @@ class Base:
|
|||
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags="area")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample1.html', text=''),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags="a")
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text='sample 2'),
|
||||
Link(url='http://example.com/sample2.jpg', text=''),
|
||||
])
|
||||
|
||||
def test_tags_attrs(self):
|
||||
|
|
@ -375,14 +375,14 @@ class Base:
|
|||
|
||||
lx = self.extractor_cls(tags='div', attrs='data-url')
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
|
||||
Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
|
||||
Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
|
||||
])
|
||||
|
||||
def test_xhtml(self):
|
||||
|
|
@ -420,13 +420,13 @@ class Base:
|
|||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
|
||||
Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text='Dont follow this one',
|
||||
fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
|
||||
Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
|
||||
fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -436,13 +436,13 @@ class Base:
|
|||
self.assertEqual(
|
||||
lx.extract_links(response),
|
||||
[
|
||||
Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
|
||||
Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
|
||||
Link(url='http://example.com/nofollow.html', text='Dont follow this one',
|
||||
fragment='', nofollow=True),
|
||||
Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
|
||||
Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
|
||||
fragment='', nofollow=False),
|
||||
Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
|
||||
Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -455,8 +455,8 @@ class Base:
|
|||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
def test_ftp_links(self):
|
||||
|
|
@ -467,7 +467,7 @@ class Base:
|
|||
response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
|
||||
Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
|
||||
])
|
||||
|
||||
def test_pickle_extractor(self):
|
||||
|
|
@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
|
|||
response = HtmlResponse("http://example.org/index.html", body=html)
|
||||
lx = self.extractor_cls()
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
|
||||
Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
|
||||
Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
|
||||
])
|
||||
|
||||
def test_link_restrict_text(self):
|
||||
|
|
@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
|
|||
# Simple text inclusion test
|
||||
lx = self.extractor_cls(restrict_text='dog')
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
|
||||
Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
|
||||
])
|
||||
# Unique regex test
|
||||
lx = self.extractor_cls(restrict_text=r'of.*dog')
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
|
||||
Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
|
||||
])
|
||||
# Multiple regex test
|
||||
lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
|
||||
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
|
||||
Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
|
||||
Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
|
||||
])
|
||||
|
||||
def test_restrict_xpaths_with_html_entities(self):
|
||||
|
|
|
|||
|
|
@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_add_value_on_unknown_field(self):
|
||||
il = TestItemLoader()
|
||||
self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
|
||||
self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
|
||||
|
||||
def test_load_item_using_default_loader(self):
|
||||
i = TestItem()
|
||||
i['summary'] = u'lala'
|
||||
i['summary'] = 'lala'
|
||||
il = ItemLoader(item=i)
|
||||
il.add_value('name', u'marta')
|
||||
il.add_value('name', 'marta')
|
||||
item = il.load_item()
|
||||
assert item is i
|
||||
self.assertEqual(item['summary'], [u'lala'])
|
||||
self.assertEqual(item['name'], [u'marta'])
|
||||
self.assertEqual(item['summary'], ['lala'])
|
||||
self.assertEqual(item['name'], ['marta'])
|
||||
|
||||
def test_load_item_using_custom_loader(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
il.add_value('name', 'marta')
|
||||
item = il.load_item()
|
||||
self.assertEqual(item['name'], [u'Marta'])
|
||||
self.assertEqual(item['name'], ['Marta'])
|
||||
|
||||
|
||||
class InitializationTestMixin:
|
||||
|
|
@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
self.assertRaises(RuntimeError, l.get_css, '#name::text')
|
||||
|
||||
def test_init_method_with_selector(self):
|
||||
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
|
||||
sel = Selector(text="<html><body><div>marta</div></body></html>")
|
||||
l = TestItemLoader(selector=sel)
|
||||
self.assertIs(l.selector, sel)
|
||||
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
|
||||
def test_init_method_with_selector_css(self):
|
||||
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
|
||||
sel = Selector(text="<html><body><div>marta</div></body></html>")
|
||||
l = TestItemLoader(selector=sel)
|
||||
self.assertIs(l.selector, sel)
|
||||
|
||||
l.add_css('name', 'div::text')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
|
||||
def test_init_method_with_response(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
|
||||
def test_init_method_with_response_css(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
|
||||
l.add_css('name', 'div::text')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
|
||||
l.add_css('url', 'a::attr(href)')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
|
||||
|
||||
# combining/accumulating CSS selectors and XPath expressions
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
|
||||
|
||||
l.add_xpath('url', '//img/@src')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])
|
||||
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
|
||||
|
||||
def test_add_xpath_re(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
l.add_xpath('name', '//div/text()', re='ma')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Ma'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Ma'])
|
||||
|
||||
def test_replace_xpath(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
l.replace_xpath('name', '//p/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
|
||||
|
||||
l.replace_xpath('name', ['//p/text()', '//div/text()'])
|
||||
self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
|
||||
|
||||
def test_get_xpath(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
|
||||
self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
|
||||
self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
|
||||
self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
|
||||
self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
|
||||
self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
|
||||
|
||||
self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
|
||||
self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
|
||||
|
||||
def test_replace_xpath_multi_fields(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
|
||||
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
|
||||
|
||||
def test_replace_xpath_re(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
l.replace_xpath('name', '//div/text()', re='ma')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Ma'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Ma'])
|
||||
|
||||
def test_add_css_re(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
l.add_css('name', 'div::text', re='ma')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Ma'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Ma'])
|
||||
|
||||
l.add_css('url', 'a::attr(href)', re='http://(.+)')
|
||||
self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
|
||||
|
||||
def test_replace_css(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
l.add_css('name', 'div::text')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
l.replace_css('name', 'p::text')
|
||||
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
|
||||
|
||||
l.replace_css('name', ['p::text', 'div::text'])
|
||||
self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
|
||||
|
||||
l.add_css('url', 'a::attr(href)', re='http://(.+)')
|
||||
self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
|
||||
l.replace_css('url', 'img::attr(src)')
|
||||
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
|
||||
self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
|
||||
|
||||
def test_get_css(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertEqual(l.get_css('p::text'), [u'paragraph'])
|
||||
self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')
|
||||
self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')
|
||||
self.assertEqual(l.get_css('p::text'), ['paragraph'])
|
||||
self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
|
||||
self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
|
||||
|
||||
self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
|
||||
self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
|
||||
self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
|
||||
[u'http://www.scrapy.org', u'/images/logo.png'])
|
||||
['http://www.scrapy.org', '/images/logo.png'])
|
||||
|
||||
def test_replace_css_multi_fields(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
|
||||
self.assertEqual(l.get_output_value('name'), [u'Marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Marta'])
|
||||
l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
|
||||
self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
|
||||
self.assertEqual(l.get_output_value('name'), ['Paragraph'])
|
||||
|
||||
l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
|
||||
l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
|
||||
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
|
||||
self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
|
||||
|
||||
def test_replace_css_re(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
self.assertTrue(l.selector)
|
||||
l.add_css('url', 'a::attr(href)')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
|
||||
l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
|
||||
self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
|
||||
|
||||
|
||||
class SubselectorLoaderTest(unittest.TestCase):
|
||||
|
|
@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl.add_css('name_div', '#id')
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), ['marta'])
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
|
||||
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
|
||||
|
|
@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl.add_css('name_div', '#id')
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name'), ['marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), ['<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), ['marta'])
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
|
||||
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
|
||||
|
|
@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl2 = nl1.nested_xpath('a')
|
||||
|
||||
l.add_xpath('url', '//footer/a/@href')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
|
||||
nl1.replace_xpath('url', 'img/@src')
|
||||
self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
|
||||
self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
|
||||
nl2.replace_xpath('url', '@href')
|
||||
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
|
||||
self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
|
||||
|
||||
def test_nested_ordering(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
|
|
@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
l.add_xpath('url', '//footer/a/@href')
|
||||
|
||||
self.assertEqual(l.get_output_value('url'), [
|
||||
u'/images/logo.png',
|
||||
u'http://www.scrapy.org',
|
||||
u'homepage',
|
||||
u'http://www.scrapy.org',
|
||||
'/images/logo.png',
|
||||
'http://www.scrapy.org',
|
||||
'homepage',
|
||||
'http://www.scrapy.org',
|
||||
])
|
||||
|
||||
def test_nested_load_item(self):
|
||||
|
|
@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
assert item is nl1.item
|
||||
assert item is nl2.item
|
||||
|
||||
self.assertEqual(item['name'], [u'marta'])
|
||||
self.assertEqual(item['url'], [u'http://www.scrapy.org'])
|
||||
self.assertEqual(item['image'], [u'/images/logo.png'])
|
||||
self.assertEqual(item['name'], ['marta'])
|
||||
self.assertEqual(item['url'], ['http://www.scrapy.org'])
|
||||
self.assertEqual(item['image'], ['/images/logo.png'])
|
||||
|
||||
|
||||
# Functions as processors
|
||||
|
|
|
|||
|
|
@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_load_item_using_default_loader(self):
|
||||
i = TestItem()
|
||||
i['summary'] = u'lala'
|
||||
i['summary'] = 'lala'
|
||||
il = ItemLoader(item=i)
|
||||
il.add_value('name', u'marta')
|
||||
il.add_value('name', 'marta')
|
||||
item = il.load_item()
|
||||
assert item is i
|
||||
self.assertEqual(item['summary'], [u'lala'])
|
||||
self.assertEqual(item['name'], [u'marta'])
|
||||
self.assertEqual(item['summary'], ['lala'])
|
||||
self.assertEqual(item['name'], ['marta'])
|
||||
|
||||
def test_load_item_using_custom_loader(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
il.add_value('name', 'marta')
|
||||
item = il.load_item()
|
||||
self.assertEqual(item['name'], [u'Marta'])
|
||||
self.assertEqual(item['name'], ['Marta'])
|
||||
|
||||
def test_load_item_ignore_none_field_values(self):
|
||||
def validate_sku(value):
|
||||
|
|
@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
price_out = Compose(TakeFirst(), float)
|
||||
sku_out = Compose(TakeFirst(), validate_sku)
|
||||
|
||||
valid_fragment = u'SKU: 1234'
|
||||
invalid_fragment = u'SKU: not available'
|
||||
valid_fragment = 'SKU: 1234'
|
||||
invalid_fragment = 'SKU: not available'
|
||||
sku_re = 'SKU: (.+)'
|
||||
|
||||
il = MyLoader(item={})
|
||||
# Should not return "sku: None".
|
||||
il.add_value('sku', [invalid_fragment], re=sku_re)
|
||||
# Should not ignore empty values.
|
||||
il.add_value('name', u'')
|
||||
il.add_value('price', [u'0'])
|
||||
il.add_value('name', '')
|
||||
il.add_value('price', ['0'])
|
||||
self.assertEqual(il.load_item(), {
|
||||
'name': u'',
|
||||
'name': '',
|
||||
'price': 0.0,
|
||||
})
|
||||
|
||||
il.replace_value('sku', [valid_fragment], re=sku_re)
|
||||
self.assertEqual(il.load_item()['sku'], u'1234')
|
||||
self.assertEqual(il.load_item()['sku'], '1234')
|
||||
|
||||
def test_self_referencing_loader(self):
|
||||
class MyLoader(ItemLoader):
|
||||
|
|
@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_add_value(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), [u'Marta'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Marta'])
|
||||
il.add_value('name', u'pepe')
|
||||
self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), ['Marta'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Marta'])
|
||||
il.add_value('name', 'pepe')
|
||||
self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
|
||||
|
||||
# test add object value
|
||||
il.add_value('summary', {'key': 1})
|
||||
self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
|
||||
|
||||
il.add_value(None, u'Jim', lambda x: {'name': x})
|
||||
self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
|
||||
il.add_value(None, 'Jim', lambda x: {'name': x})
|
||||
self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
|
||||
|
||||
def test_add_zero(self):
|
||||
il = NameItemLoader()
|
||||
|
|
@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_replace_value(self):
|
||||
il = TestItemLoader()
|
||||
il.replace_value('name', u'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), [u'Marta'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Marta'])
|
||||
il.replace_value('name', u'pepe')
|
||||
self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Pepe'])
|
||||
il.replace_value('name', 'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), ['Marta'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Marta'])
|
||||
il.replace_value('name', 'pepe')
|
||||
self.assertEqual(il.get_collected_values('name'), ['Pepe'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Pepe'])
|
||||
|
||||
il.replace_value(None, u'Jim', lambda x: {'name': x})
|
||||
self.assertEqual(il.get_collected_values('name'), [u'Jim'])
|
||||
il.replace_value(None, 'Jim', lambda x: {'name': x})
|
||||
self.assertEqual(il.get_collected_values('name'), ['Jim'])
|
||||
|
||||
def test_get_value(self):
|
||||
il = NameItemLoader()
|
||||
self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
|
||||
self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
|
||||
self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
|
||||
self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
|
||||
self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
|
||||
self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
|
||||
|
||||
il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
|
||||
self.assertEqual([u'foo'], il.get_collected_values('name'))
|
||||
il.replace_value('name', u'name:bar', re=u'name:(.*)$')
|
||||
self.assertEqual([u'bar'], il.get_collected_values('name'))
|
||||
il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
|
||||
self.assertEqual(['foo'], il.get_collected_values('name'))
|
||||
il.replace_value('name', 'name:bar', re='name:(.*)$')
|
||||
self.assertEqual(['bar'], il.get_collected_values('name'))
|
||||
|
||||
def test_iter_on_input_processor_input(self):
|
||||
class NameFirstItemLoader(NameItemLoader):
|
||||
name_in = TakeFirst()
|
||||
|
||||
il = NameFirstItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), [u'marta'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), ['marta'])
|
||||
il = NameFirstItemLoader()
|
||||
il.add_value('name', [u'marta', u'jose'])
|
||||
self.assertEqual(il.get_collected_values('name'), [u'marta'])
|
||||
il.add_value('name', ['marta', 'jose'])
|
||||
self.assertEqual(il.get_collected_values('name'), ['marta'])
|
||||
|
||||
il = NameFirstItemLoader()
|
||||
il.replace_value('name', u'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), [u'marta'])
|
||||
il.replace_value('name', 'marta')
|
||||
self.assertEqual(il.get_collected_values('name'), ['marta'])
|
||||
il = NameFirstItemLoader()
|
||||
il.replace_value('name', [u'marta', u'jose'])
|
||||
self.assertEqual(il.get_collected_values('name'), [u'marta'])
|
||||
il.replace_value('name', ['marta', 'jose'])
|
||||
self.assertEqual(il.get_collected_values('name'), ['marta'])
|
||||
|
||||
il = NameFirstItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
il.add_value('name', [u'jose', u'pedro'])
|
||||
self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
|
||||
il.add_value('name', 'marta')
|
||||
il.add_value('name', ['jose', 'pedro'])
|
||||
self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
|
||||
|
||||
def test_map_compose_filter(self):
|
||||
def filter_world(x):
|
||||
|
|
@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'Mart'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['Mart'])
|
||||
item = il.load_item()
|
||||
self.assertEqual(item['name'], [u'Mart'])
|
||||
self.assertEqual(item['name'], ['Mart'])
|
||||
|
||||
def test_default_input_processor(self):
|
||||
il = DefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'mart'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['mart'])
|
||||
|
||||
def test_inherited_default_input_processor(self):
|
||||
class InheritDefaultedItemLoader(DefaultedItemLoader):
|
||||
pass
|
||||
|
||||
il = InheritDefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'mart'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['mart'])
|
||||
|
||||
def test_input_processor_inheritance(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
url_in = MapCompose(lambda v: v.lower())
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value('url', u'HTTP://scrapy.ORG')
|
||||
self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'Marta'])
|
||||
il.add_value('url', 'HTTP://scrapy.ORG')
|
||||
self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['Marta'])
|
||||
|
||||
class ChildChildItemLoader(ChildItemLoader):
|
||||
url_in = MapCompose(lambda v: v.upper())
|
||||
summary_in = MapCompose(lambda v: v)
|
||||
|
||||
il = ChildChildItemLoader()
|
||||
il.add_value('url', u'http://scrapy.org')
|
||||
self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'Marta'])
|
||||
il.add_value('url', 'http://scrapy.org')
|
||||
self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['Marta'])
|
||||
|
||||
def test_empty_map_compose(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_in = MapCompose()
|
||||
|
||||
il = IdentityDefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'marta'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['marta'])
|
||||
|
||||
def test_identity_input_processor(self):
|
||||
class IdentityDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_in = Identity()
|
||||
|
||||
il = IdentityDefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'marta'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['marta'])
|
||||
|
||||
def test_extend_custom_input_processors(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'mARTA'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['mARTA'])
|
||||
|
||||
def test_extend_default_input_processors(self):
|
||||
class ChildDefaultedItemLoader(DefaultedItemLoader):
|
||||
name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
|
||||
|
||||
il = ChildDefaultedItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
self.assertEqual(il.get_output_value('name'), [u'MART'])
|
||||
il.add_value('name', 'marta')
|
||||
self.assertEqual(il.get_output_value('name'), ['MART'])
|
||||
|
||||
def test_output_processor_using_function(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
name_out = u" ".join
|
||||
name_out = " ".join
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), u'Mar Ta')
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), 'Mar Ta')
|
||||
|
||||
def test_output_processor_error(self):
|
||||
class TestItemLoader(ItemLoader):
|
||||
|
|
@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
name_out = MapCompose(float)
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'$10'])
|
||||
il.add_value('name', ['$10'])
|
||||
try:
|
||||
float(u'$10')
|
||||
float('$10')
|
||||
except Exception as e:
|
||||
expected_exc_str = str(e)
|
||||
|
||||
|
|
@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
def test_output_processor_using_classes(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
name_out = Join()
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), u'Mar Ta')
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), 'Mar Ta')
|
||||
|
||||
class TakeFirstItemLoader(TestItemLoader):
|
||||
name_out = Join("<br>")
|
||||
|
||||
il = TakeFirstItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), u'Mar<br>Ta')
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), 'Mar<br>Ta')
|
||||
|
||||
def test_default_output_processor(self):
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
|
||||
|
||||
class LalaItemLoader(TestItemLoader):
|
||||
default_output_processor = Identity()
|
||||
|
||||
il = LalaItemLoader()
|
||||
il.add_value('name', [u'mar', u'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
|
||||
il.add_value('name', ['mar', 'ta'])
|
||||
self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
|
||||
|
||||
def test_loader_context_on_declaration(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
url_in = MapCompose(processor_with_args, key=u'val')
|
||||
url_in = MapCompose(processor_with_args, key='val')
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.add_value('url', u'text')
|
||||
il.add_value('url', 'text')
|
||||
self.assertEqual(il.get_output_value('url'), ['val'])
|
||||
il.replace_value('url', u'text2')
|
||||
il.replace_value('url', 'text2')
|
||||
self.assertEqual(il.get_output_value('url'), ['val'])
|
||||
|
||||
def test_loader_context_on_instantiation(self):
|
||||
class ChildItemLoader(TestItemLoader):
|
||||
url_in = MapCompose(processor_with_args)
|
||||
|
||||
il = ChildItemLoader(key=u'val')
|
||||
il.add_value('url', u'text')
|
||||
il = ChildItemLoader(key='val')
|
||||
il.add_value('url', 'text')
|
||||
self.assertEqual(il.get_output_value('url'), ['val'])
|
||||
il.replace_value('url', u'text2')
|
||||
il.replace_value('url', 'text2')
|
||||
self.assertEqual(il.get_output_value('url'), ['val'])
|
||||
|
||||
def test_loader_context_on_assign(self):
|
||||
|
|
@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
url_in = MapCompose(processor_with_args)
|
||||
|
||||
il = ChildItemLoader()
|
||||
il.context['key'] = u'val'
|
||||
il.add_value('url', u'text')
|
||||
il.context['key'] = 'val'
|
||||
il.add_value('url', 'text')
|
||||
self.assertEqual(il.get_output_value('url'), ['val'])
|
||||
il.replace_value('url', u'text2')
|
||||
il.replace_value('url', 'text2')
|
||||
self.assertEqual(il.get_output_value('url'), ['val'])
|
||||
|
||||
def test_item_passed_to_input_processor_functions(self):
|
||||
|
|
@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
it = TestItem(name='marta')
|
||||
il = ChildItemLoader(item=it)
|
||||
il.add_value('url', u'text')
|
||||
il.add_value('url', 'text')
|
||||
self.assertEqual(il.get_output_value('url'), ['marta'])
|
||||
il.replace_value('url', u'text2')
|
||||
il.replace_value('url', 'text2')
|
||||
self.assertEqual(il.get_output_value('url'), ['marta'])
|
||||
|
||||
def test_compose_processor(self):
|
||||
|
|
@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'marta', u'other'])
|
||||
self.assertEqual(il.get_output_value('name'), u'Mart')
|
||||
il.add_value('name', ['marta', 'other'])
|
||||
self.assertEqual(il.get_output_value('name'), 'Mart')
|
||||
item = il.load_item()
|
||||
self.assertEqual(item['name'], u'Mart')
|
||||
self.assertEqual(item['name'], 'Mart')
|
||||
|
||||
def test_partial_processor(self):
|
||||
def join(values, sep=None, loader_context=None, ignored=None):
|
||||
|
|
@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
summary_out = Compose(partial(join, ignored='foo'))
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', [u'rabbit', u'hole'])
|
||||
il.add_value('url', [u'rabbit', u'hole'])
|
||||
il.add_value('summary', [u'rabbit', u'hole'])
|
||||
il.add_value('name', ['rabbit', 'hole'])
|
||||
il.add_value('url', ['rabbit', 'hole'])
|
||||
il.add_value('summary', ['rabbit', 'hole'])
|
||||
item = il.load_item()
|
||||
self.assertEqual(item['name'], u'rabbit+hole')
|
||||
self.assertEqual(item['url'], u'rabbit.hole')
|
||||
self.assertEqual(item['summary'], u'rabbithole')
|
||||
self.assertEqual(item['name'], 'rabbit+hole')
|
||||
self.assertEqual(item['url'], 'rabbit.hole')
|
||||
self.assertEqual(item['summary'], 'rabbithole')
|
||||
|
||||
def test_error_input_processor(self):
|
||||
class TestItem(Item):
|
||||
|
|
@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
il = TestItemLoader()
|
||||
self.assertRaises(ValueError, il.add_value, 'name',
|
||||
[u'marta', u'other'])
|
||||
['marta', 'other'])
|
||||
|
||||
def test_error_output_processor(self):
|
||||
class TestItem(Item):
|
||||
|
|
@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
name_out = Compose(Join(), float)
|
||||
|
||||
il = TestItemLoader()
|
||||
il.add_value('name', u'marta')
|
||||
il.add_value('name', 'marta')
|
||||
with self.assertRaises(ValueError):
|
||||
il.load_item()
|
||||
|
||||
|
|
@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
|
||||
il = TestItemLoader()
|
||||
self.assertRaises(ValueError, il.add_value, 'name',
|
||||
[u'marta', u'other'], Compose(float))
|
||||
['marta', 'other'], Compose(float))
|
||||
|
||||
|
||||
class InitializationFromDictTest(unittest.TestCase):
|
||||
|
|
@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):
|
|||
def test_join(self):
|
||||
proc = Join()
|
||||
self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
|
||||
self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
|
||||
self.assertEqual(proc(['hello', 'world']), u'hello world')
|
||||
self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
|
||||
self.assertEqual(proc(['hello', 'world']), 'hello world')
|
||||
self.assertIsInstance(proc(['hello', 'world']), str)
|
||||
|
||||
def test_compose(self):
|
||||
|
|
@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):
|
|||
def filter_world(x):
|
||||
return None if x == 'world' else x
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
|
||||
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
|
||||
self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
|
||||
['HELLO', 'THIS', 'IS', 'SCRAPY'])
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
self.assertEqual(proc(None), [])
|
||||
proc = MapCompose(filter_world, str.upper)
|
||||
|
|
|
|||
|
|
@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):
|
|||
|
||||
def test_dropped(self):
|
||||
item = {}
|
||||
exception = Exception(u"\u2018")
|
||||
exception = Exception("\u2018")
|
||||
response = Response("http://www.example.com")
|
||||
logkws = self.formatter.dropped(item, exception, response, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
lines = logline.splitlines()
|
||||
assert all(isinstance(x, str) for x in lines)
|
||||
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
|
||||
self.assertEqual(lines, ["Dropped: \u2018", '{}'])
|
||||
|
||||
def test_item_error(self):
|
||||
# In practice, the complete traceback is shown by passing the
|
||||
|
|
@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):
|
|||
response = Response("http://www.example.com")
|
||||
logkws = self.formatter.item_error(item, exception, response, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
self.assertEqual(logline, u"Error processing {'key': 'value'}")
|
||||
self.assertEqual(logline, "Error processing {'key': 'value'}")
|
||||
|
||||
def test_spider_error(self):
|
||||
# In practice, the complete traceback is shown by passing the
|
||||
|
|
@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):
|
|||
|
||||
def test_scraped(self):
|
||||
item = CustomItem()
|
||||
item['name'] = u'\xa3'
|
||||
item['name'] = '\xa3'
|
||||
response = Response("http://www.example.com")
|
||||
logkws = self.formatter.scraped(item, response, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
lines = logline.splitlines()
|
||||
assert all(isinstance(x, str) for x in lines)
|
||||
self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
|
||||
self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
|
||||
|
||||
|
||||
class LogFormatterSubclass(LogFormatter):
|
||||
def crawled(self, request, response, spider):
|
||||
kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
|
||||
CRAWLEDMSG = (
|
||||
u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
|
||||
"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
|
||||
)
|
||||
log_args = kwargs['args']
|
||||
log_args['flags'] = str(request.flags)
|
||||
|
|
|
|||
|
|
@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):
|
|||
self.catched_msg = dict(**kwargs)
|
||||
|
||||
def test_send_utf8(self):
|
||||
subject = u'sübjèçt'
|
||||
body = u'bödÿ-àéïöñß'
|
||||
subject = 'sübjèçt'
|
||||
body = 'bödÿ-àéïöñß'
|
||||
mailsender = MailSender(debug=True)
|
||||
mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
|
||||
charset='utf-8', _callback=self._catch_mail_sent)
|
||||
|
|
@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):
|
|||
self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
|
||||
|
||||
def test_send_attach_utf8(self):
|
||||
subject = u'sübjèçt'
|
||||
body = u'bödÿ-àéïöñß'
|
||||
subject = 'sübjèçt'
|
||||
body = 'bödÿ-àéïöñß'
|
||||
attach = BytesIO()
|
||||
attach.write(body.encode('utf-8'))
|
||||
attach.seek(0)
|
||||
|
|
|
|||
|
|
@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
mappings = [
|
||||
(b'attachment; filename="data.xml"', XmlResponse),
|
||||
(b'attachment; filename=data.xml', XmlResponse),
|
||||
(u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
|
||||
(u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
|
||||
(u'attachment;filename=data高.doc'.encode('gbk'), Response),
|
||||
(u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
|
||||
(u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
|
||||
('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
|
||||
('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
|
||||
('attachment;filename=data高.doc'.encode('gbk'), Response),
|
||||
('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
|
||||
('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
|
||||
|
||||
]
|
||||
for source, cls in mappings:
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ class BaseRobotParserTest:
|
|||
self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
|
||||
|
||||
def test_unicode_url_and_useragent(self):
|
||||
robotstxt_robotstxt_body = u"""
|
||||
robotstxt_robotstxt_body = """
|
||||
User-Agent: *
|
||||
Disallow: /admin/
|
||||
Disallow: /static/
|
||||
|
|
@ -107,11 +107,11 @@ class BaseRobotParserTest:
|
|||
self.assertTrue(rp.allowed("https://site.local/", "*"))
|
||||
self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
|
||||
self.assertFalse(rp.allowed("https://site.local/static/", "*"))
|
||||
self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
|
||||
self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
|
||||
self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
|
||||
self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
|
||||
self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
|
||||
self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
|
||||
self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
|
||||
self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
|
||||
|
||||
|
||||
class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):
|
|||
)
|
||||
self.assertEqual(
|
||||
[x.get() for x in sel.xpath("//input[@name='a']/@name")],
|
||||
[u'a']
|
||||
['a']
|
||||
)
|
||||
self.assertEqual(
|
||||
[x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
[u'12.0']
|
||||
['12.0']
|
||||
)
|
||||
self.assertEqual(
|
||||
sel.xpath("concat('xpath', 'rules')").getall(),
|
||||
[u'xpathrules']
|
||||
['xpathrules']
|
||||
)
|
||||
self.assertEqual(
|
||||
[x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12']
|
||||
['12']
|
||||
)
|
||||
|
||||
def test_root_base_url(self):
|
||||
|
|
@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):
|
|||
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'xml')
|
||||
self.assertEqual(sel.xpath("//div").getall(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
['<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'html')
|
||||
self.assertEqual(sel.xpath("//div").getall(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
['<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
def test_http_header_encoding_precedence(self):
|
||||
# u'\xa3' = pound symbol in unicode
|
||||
# u'\xc2\xa3' = pound symbol in utf-8
|
||||
# u'\xa3' = pound symbol in latin-1 (iso-8859-1)
|
||||
# '\xa3' = pound symbol in unicode
|
||||
# '\xc2\xa3' = pound symbol in utf-8
|
||||
# '\xa3' = pound symbol in latin-1 (iso-8859-1)
|
||||
|
||||
meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
|
||||
head = u'<head>' + meta + u'</head>'
|
||||
body_content = u'<span id="blank">\xa3</span>'
|
||||
body = u'<body>' + body_content + u'</body>'
|
||||
html = u'<html>' + head + body + u'</html>'
|
||||
meta = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
|
||||
head = '<head>' + meta + '</head>'
|
||||
body_content = '<span id="blank">\xa3</span>'
|
||||
body = '<body>' + body_content + '</body>'
|
||||
html = '<html>' + head + body + '</html>'
|
||||
encoding = 'utf-8'
|
||||
html_utf8 = html.encode(encoding)
|
||||
|
||||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = Selector(response)
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
|
|
@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):
|
|||
|
||||
def test_selector_bad_args(self):
|
||||
with self.assertRaisesRegex(ValueError, 'received both response and text'):
|
||||
Selector(TextResponse(url='http://example.com', body=b''), text=u'')
|
||||
Selector(TextResponse(url='http://example.com', body=b''), text='')
|
||||
|
|
|
|||
|
|
@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):
|
|||
output = list(spider._parse(response))
|
||||
self.assertEqual(len(output), 2, iterator)
|
||||
self.assertEqual(output, [
|
||||
{'loc': [u'http://www.example.com/Special-Offers.html'],
|
||||
'updated': [u'2009-08-16'],
|
||||
'custom': [u'fuu'],
|
||||
'other': [u'bar']},
|
||||
{'loc': ['http://www.example.com/Special-Offers.html'],
|
||||
'updated': ['2009-08-16'],
|
||||
'custom': ['fuu'],
|
||||
'other': ['bar']},
|
||||
{'loc': [],
|
||||
'updated': [u'2009-08-16'],
|
||||
'other': [u'foo'],
|
||||
'updated': ['2009-08-16'],
|
||||
'other': ['foo'],
|
||||
'custom': []},
|
||||
], iterator)
|
||||
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):
|
|||
|
||||
def test_xmliter_unicode(self):
|
||||
# example taken from https://github.com/scrapy/scrapy/issues/1665
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<þingflokkar>
|
||||
<þingflokkur id="26">
|
||||
<heiti />
|
||||
|
|
@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):
|
|||
XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
|
||||
):
|
||||
attrs = []
|
||||
for x in self.xmliter(r, u'þingflokkur'):
|
||||
for x in self.xmliter(r, 'þingflokkur'):
|
||||
attrs.append((x.attrib['id'],
|
||||
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
|
||||
x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
|
||||
x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
|
||||
x.xpath('./tímabil/fyrstaþing/text()').getall()))
|
||||
|
||||
self.assertEqual(attrs,
|
||||
[(u'26', [u'-'], [u'80']),
|
||||
(u'21', [u'Ab'], [u'76']),
|
||||
(u'27', [u'A'], [u'27'])])
|
||||
[('26', ['-'], ['80']),
|
||||
('21', ['Ab'], ['76']),
|
||||
('27', ['A'], ['27'])])
|
||||
|
||||
def test_xmliter_text(self):
|
||||
body = (
|
||||
|
|
@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):
|
|||
)
|
||||
|
||||
self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
|
||||
[[u'one'], [u'two']])
|
||||
[['one'], ['two']])
|
||||
|
||||
def test_xmliter_namespaces(self):
|
||||
body = b"""
|
||||
|
|
@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):
|
|||
response = XmlResponse('http://www.example.com', body=body)
|
||||
self.assertEqual(
|
||||
next(self.xmliter(response, 'item')).get(),
|
||||
u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
|
||||
'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
|
||||
result = [row for row in csv]
|
||||
self.assertEqual(result,
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
# explicit type check cuz' we no like stinkin' autocasting! yarrr
|
||||
for result_row in result:
|
||||
|
|
@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
csv = csviter(response, delimiter='\t')
|
||||
|
||||
self.assertEqual([row for row in csv],
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_quotechar(self):
|
||||
body1 = get_testdata('feeds', 'feed-sample6.csv')
|
||||
|
|
@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
csv1 = csviter(response1, quotechar="'")
|
||||
|
||||
self.assertEqual([row for row in csv1],
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
response2 = TextResponse(url="http://example.com/", body=body2)
|
||||
csv2 = csviter(response2, delimiter="|", quotechar="'")
|
||||
|
||||
self.assertEqual([row for row in csv2],
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_wrong_quotechar(self):
|
||||
body = get_testdata('feeds', 'feed-sample6.csv')
|
||||
|
|
@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
csv = csviter(response)
|
||||
|
||||
self.assertEqual([row for row in csv],
|
||||
[{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
|
||||
{u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
|
||||
{u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
|
||||
{u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
|
||||
[{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
|
||||
{"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
|
||||
{"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
|
||||
{"'id'": "4", "'name'": "'empty'", "'value'": ""}])
|
||||
|
||||
def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
|
||||
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
|
||||
|
|
@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
csv = csviter(response, delimiter='\t')
|
||||
|
||||
self.assertEqual([row for row in csv],
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_headers(self):
|
||||
sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
|
||||
|
|
@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
|
||||
|
||||
self.assertEqual([row for row in csv],
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_falserow(self):
|
||||
body = get_testdata('feeds', 'feed-sample3.csv')
|
||||
|
|
@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
csv = csviter(response)
|
||||
|
||||
self.assertEqual([row for row in csv],
|
||||
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
|
||||
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
|
||||
{u'id': u'4', u'name': u'empty', u'value': u''}])
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_exception(self):
|
||||
body = get_testdata('feeds', 'feed-sample3.csv')
|
||||
|
|
@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual(
|
||||
list(csv),
|
||||
[
|
||||
{u'id': u'1', u'name': u'latin1', u'value': u'test'},
|
||||
{u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
|
||||
{'id': '1', 'name': 'latin1', 'value': 'test'},
|
||||
{'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
|
||||
]
|
||||
)
|
||||
|
||||
|
|
@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual(
|
||||
list(csv),
|
||||
[
|
||||
{u'id': u'1', u'name': u'cp852', u'value': u'test'},
|
||||
{u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
|
||||
{'id': '1', 'name': 'cp852', 'value': 'test'},
|
||||
{'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):
|
|||
|
||||
class ToUnicodeTest(unittest.TestCase):
|
||||
def test_converting_an_utf8_encoded_string_to_unicode(self):
|
||||
self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
|
||||
self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
|
||||
|
||||
def test_converting_a_latin_1_encoded_string_to_unicode(self):
|
||||
self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
|
||||
self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
|
||||
|
||||
def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
|
||||
self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
|
||||
self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
|
||||
|
||||
def test_converting_a_strange_object_should_raise_TypeError(self):
|
||||
self.assertRaises(TypeError, to_unicode, 423)
|
||||
|
|
@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):
|
|||
def test_errors_argument(self):
|
||||
self.assertEqual(
|
||||
to_unicode(b'a\xedb', 'utf-8', errors='replace'),
|
||||
u'a\ufffdb'
|
||||
'a\ufffdb'
|
||||
)
|
||||
|
||||
|
||||
class ToBytesTest(unittest.TestCase):
|
||||
def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
|
||||
self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
|
||||
self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
|
||||
|
||||
def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
|
||||
self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
|
||||
self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
|
||||
|
||||
def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
|
||||
self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
|
||||
|
|
@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):
|
|||
|
||||
def test_errors_argument(self):
|
||||
self.assertEqual(
|
||||
to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
|
||||
to_bytes('a\ufffdb', 'latin-1', errors='replace'),
|
||||
b'a?b'
|
||||
)
|
||||
|
||||
|
|
@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):
|
|||
assert binary_is_text(b"hello")
|
||||
|
||||
def test_utf_16_strings_contain_null_bytes(self):
|
||||
assert binary_is_text(u"hello".encode('utf-16'))
|
||||
assert binary_is_text("hello".encode('utf-16'))
|
||||
|
||||
def test_one_with_encoding(self):
|
||||
assert binary_is_text(b"<div>Price \xa3</div>")
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
method="POST",
|
||||
body=b"some body",
|
||||
headers={'content-encoding': 'text/html; charset=latin-1'},
|
||||
cookies={'currency': u'руб'},
|
||||
cookies={'currency': 'руб'},
|
||||
encoding='latin-1',
|
||||
priority=20,
|
||||
meta={'a': 'b'},
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
|
|||
def test_simple_render(self):
|
||||
|
||||
context = dict(project_name='proj', name='spi', classname='TheSpider')
|
||||
template = u'from ${project_name}.spiders.${name} import ${classname}'
|
||||
rendered = u'from proj.spiders.spi import TheSpider'
|
||||
template = 'from ${project_name}.spiders.${name} import ${classname}'
|
||||
rendered = 'from proj.spiders.spi import TheSpider'
|
||||
|
||||
template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
|
||||
render_path = os.path.join(self.tmp_path, 'templ.py')
|
||||
|
|
|
|||
Loading…
Reference in New Issue