...
Kenmore White 17" Microwave
...

diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py
index 9acfc3b23..95a3f17d5 100755
--- a/docs/utils/linkfix.py
+++ b/docs/utils/linkfix.py
@@ -23,7 +23,7 @@ def main():
_contents = None
# A regex that matches standard linkcheck output lines
- line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+ line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0e6ceef0b..a260798ac 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
namespaces={
"re": "http://exslt.org/regular-expressions"})
- values = [(k, u'' if v is None else v)
+ values = [(k, '' if v is None else v)
for k, v in (_value(e) for e in inputs)
if k and k not in formdata_keys]
@@ -168,7 +168,7 @@ def _select_value(ele, n, v):
# This is a workround to bug in lxml fixed 2.3.1
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
selected_options = ele.xpath('.//option[@selected]')
- v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
+ v = [(o.get('value') or o.text or '').strip() for o in selected_options]
return n, v
@@ -205,7 +205,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
- xpath = u'.//*' + u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
+ xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1615d44d7..8b9f961ee 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
url = safe_url_string(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
- link = Link(url, _collect_string_content(el) or u'',
+ link = Link(url, _collect_string_content(el) or '',
nofollow=rel_has_nofollow(el.get('rel')))
links.append(link)
return self._deduplicate_if_needed(links)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 219145f13..0f9e6f1cb 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -44,7 +44,7 @@ class LogFormatter:
def dropped(self, item, exception, response, spider):
return {
'level': logging.INFO, # lowering the level from logging.WARNING
- 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+ 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
'args': {
'exception': exception,
'item': item,
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index da99a6be8..591075a98 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -59,7 +59,7 @@ class CmdlineTest(unittest.TestCase):
'EXTENSIONS=' + json.dumps(EXTENSIONS))
# XXX: There's gotta be a smarter way to do this...
self.assertNotIn("...", settingsstr)
- for char in ("'", "<", ">", 'u"'):
+ for char in ("'", "<", ">"):
settingsstr = settingsstr.replace(char, '"')
settingsdict = json.loads(settingsstr)
self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 51deb20f4..57d4cdd6b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1110,7 +1110,7 @@ class DataURITestCase(unittest.TestCase):
def test_default_mediatype(self):
def _test(response):
- self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+ self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "iso-8859-7")
@@ -1119,7 +1119,7 @@ class DataURITestCase(unittest.TestCase):
def test_text_charset(self):
def _test(response):
- self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+ self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(response.body, b'\xbe\xd3\xbe')
self.assertEqual(response.encoding, "iso-8859-7")
@@ -1128,7 +1128,7 @@ class DataURITestCase(unittest.TestCase):
def test_mediatype_parameters(self):
def _test(response):
- self.assertEqual(response.text, u'\u038e\u03a3\u038e')
+ self.assertEqual(response.text, '\u038e\u03a3\u038e')
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "utf-8")
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 9ccc2110b..010577415 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -277,33 +277,33 @@ class CookiesMiddlewareTest(TestCase):
def test_request_cookies_encoding(self):
# 1) UTF8-encoded bytes
- req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
+ req1 = Request('http://example.org', cookies={'a': 'á'.encode('utf8')})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
# 2) Non UTF8-encoded bytes
- req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
+ req2 = Request('http://example.org', cookies={'a': 'á'.encode('latin1')})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
- # 3) Unicode string
- req3 = Request('http://example.org', cookies={'a': u'á'})
+ # 3) String
+ req3 = Request('http://example.org', cookies={'a': 'á'})
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
def test_request_headers_cookie_encoding(self):
# 1) UTF8-encoded bytes
- req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
+ req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
assert self.mw.process_request(req1, self.spider) is None
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
# 2) Non UTF8-encoded bytes
- req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
+ req2 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('latin1')})
assert self.mw.process_request(req2, self.spider) is None
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
- # 3) Unicode string
- req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
+ # 3) String
+ req3 = Request('http://example.org', headers={'Cookie': 'a=á'})
assert self.mw.process_request(req3, self.spider) is None
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 9841d7a76..351631eb8 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -88,7 +88,7 @@ class TestHttpProxyMiddleware(TestCase):
def test_proxy_auth_encoding(self):
# utf-8 encoding
- os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+ os.environ['http_proxy'] = 'https://m\u00E1n:pass@proxy:3128'
mw = HttpProxyMiddleware(auth_encoding='utf-8')
req = Request('http://scrapytest.org')
assert mw.process_request(req, spider) is None
@@ -96,7 +96,7 @@ class TestHttpProxyMiddleware(TestCase):
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
# proxy from request.meta
- req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+ req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
@@ -109,7 +109,7 @@ class TestHttpProxyMiddleware(TestCase):
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
# proxy from request.meta, latin-1 encoding
- req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
+ req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
assert mw.process_request(req, spider) is None
self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 919dbed23..131332131 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -184,7 +184,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
def test_latin1_location(self):
req = Request('http://scrapytest.org/first')
- latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1
+ latin1_location = '/ação'.encode('latin1') # HTTP historically supports latin1
resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
req_result = self.mw.process_response(req, resp, self.spider)
perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
@@ -192,7 +192,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
def test_utf8_location(self):
req = Request('http://scrapytest.org/first')
- utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding
+ utf8_location = '/ação'.encode('utf-8') # header using UTF-8 encoding
resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
req_result = self.mw.process_response(req, resp, self.spider)
perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
@@ -207,7 +207,7 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
self.mw = MetaRefreshMiddleware.from_crawler(crawler)
def _body(self, interval=5, url='http://example.org/newpage'):
- html = u"""
"""
+ html = """
"""
return html.format(interval, url).encode('utf-8')
def test_priority_adjust(self):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index b9452a0e7..f9936baba 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -30,7 +30,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def _get_successful_crawler(self):
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
- ROBOTS = u"""
+ ROBOTS = """
User-Agent: *
Disallow: /admin/
Disallow: /static/
@@ -56,7 +56,7 @@ Disallow: /some/randome/page.html
self.assertIgnored(Request('http://site.local/admin/main'), middleware),
self.assertIgnored(Request('http://site.local/static/'), middleware),
self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
- self.assertIgnored(Request(u'http://site.local/wiki/Käyttäjä:'), middleware)
+ self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
], fireOnOneErrback=True)
def test_robotstxt_ready_parser(self):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 25da54a65..660c99ce1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -54,7 +54,7 @@ class BaseItemExporterTest(unittest.TestCase):
def setUp(self):
if self.item_class is None:
raise unittest.SkipTest("item class is None")
- self.i = self.item_class(name=u'John\xa3', age=u'22')
+ self.i = self.item_class(name='John\xa3', age='22')
self.output = BytesIO()
self.ie = self._get_exporter()
@@ -96,25 +96,25 @@ class BaseItemExporterTest(unittest.TestCase):
def test_serialize_field(self):
a = ItemAdapter(self.i)
res = self.ie.serialize_field(a.get_field_meta('name'), 'name', a['name'])
- self.assertEqual(res, u'John\xa3')
+ self.assertEqual(res, 'John\xa3')
res = self.ie.serialize_field(a.get_field_meta('age'), 'age', a['age'])
- self.assertEqual(res, u'22')
+ self.assertEqual(res, '22')
def test_fields_to_export(self):
ie = self._get_exporter(fields_to_export=['name'])
- self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', u'John\xa3')])
+ self.assertEqual(list(ie._get_serialized_fields(self.i)), [('name', 'John\xa3')])
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
_, name = list(ie._get_serialized_fields(self.i))[0]
assert isinstance(name, str)
- self.assertEqual(name, u'John\xa3')
+ self.assertEqual(name, 'John\xa3')
def test_field_custom_serializer(self):
- i = self.custom_field_item_class(name=u'John\xa3', age=u'22')
+ i = self.custom_field_item_class(name='John\xa3', age='22')
a = ItemAdapter(i)
ie = self._get_exporter()
- self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), u'John\xa3')
+ self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
@@ -132,48 +132,48 @@ class PythonItemExporterTest(BaseItemExporterTest):
PythonItemExporter(invalid_option='something')
def test_nested_item(self):
- i1 = self.item_class(name=u'Joseph', age='22')
- i2 = dict(name=u'Maria', age=i1)
- i3 = self.item_class(name=u'Jesus', age=i2)
+ i1 = self.item_class(name='Joseph', age='22')
+ i2 = dict(name='Maria', age=i1)
+ i3 = self.item_class(name='Jesus', age=i2)
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(type(exported), dict)
self.assertEqual(
exported,
- {'age': {'age': {'age': '22', 'name': u'Joseph'}, 'name': u'Maria'}, 'name': 'Jesus'}
+ {'age': {'age': {'age': '22', 'name': 'Joseph'}, 'name': 'Maria'}, 'name': 'Jesus'}
)
self.assertEqual(type(exported['age']), dict)
self.assertEqual(type(exported['age']['age']), dict)
def test_export_list(self):
- i1 = self.item_class(name=u'Joseph', age='22')
- i2 = self.item_class(name=u'Maria', age=[i1])
- i3 = self.item_class(name=u'Jesus', age=[i2])
+ i1 = self.item_class(name='Joseph', age='22')
+ i2 = self.item_class(name='Maria', age=[i1])
+ i3 = self.item_class(name='Jesus', age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(
exported,
- {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+ {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
)
self.assertEqual(type(exported['age'][0]), dict)
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_item_dict_list(self):
- i1 = self.item_class(name=u'Joseph', age='22')
- i2 = dict(name=u'Maria', age=[i1])
- i3 = self.item_class(name=u'Jesus', age=[i2])
+ i1 = self.item_class(name='Joseph', age='22')
+ i2 = dict(name='Maria', age=[i1])
+ i3 = self.item_class(name='Jesus', age=[i2])
ie = self._get_exporter()
exported = ie.export_item(i3)
self.assertEqual(
exported,
- {'age': [{'age': [{'age': '22', 'name': u'Joseph'}], 'name': u'Maria'}], 'name': 'Jesus'}
+ {'age': [{'age': [{'age': '22', 'name': 'Joseph'}], 'name': 'Maria'}], 'name': 'Jesus'}
)
self.assertEqual(type(exported['age'][0]), dict)
self.assertEqual(type(exported['age'][0]['age'][0]), dict)
def test_export_binary(self):
exporter = PythonItemExporter(binary=True)
- value = self.item_class(name=u'John\xa3', age=u'22')
+ value = self.item_class(name='John\xa3', age='22')
expected = {b'name': b'John\xc2\xa3', b'age': b'22'}
self.assertEqual(expected, exporter.export_item(value))
@@ -279,7 +279,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
- self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+ self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()
@@ -396,7 +396,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
def test_multivalued_fields(self):
self.assertExportResult(
- self.item_class(name=[u'John\xa3', u'Doe'], age=[1, 2, 3]),
+ self.item_class(name=['John\xa3', 'Doe'], age=[1, 2, 3]),
b"""\n
-
@@ -408,9 +408,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
def test_nested_item(self):
- i1 = dict(name=u'foo\xa3hoo', age='22')
- i2 = dict(name=u'bar', age=i1)
- i3 = self.item_class(name=u'buz', age=i2)
+ i1 = dict(name='foo\xa3hoo', age='22')
+ i2 = dict(name='bar', age=i1)
+ i3 = self.item_class(name='buz', age=i2)
self.assertExportResult(
i3,
@@ -431,9 +431,9 @@ class XmlItemExporterTest(BaseItemExporterTest):
)
def test_nested_list_item(self):
- i1 = dict(name=u'foo')
- i2 = dict(name=u'bar', v2={"egg": ["spam"]})
- i3 = self.item_class(name=u'buz', age=[i1, i2])
+ i1 = dict(name='foo')
+ i2 = dict(name='bar', v2={"egg": ["spam"]})
+ i3 = self.item_class(name='buz', age=[i1, i2])
self.assertExportResult(
i3,
@@ -475,7 +475,7 @@ class XmlItemExporterDataclassTest(XmlItemExporterTest):
class JsonLinesItemExporterTest(BaseItemExporterTest):
- _expected_nested = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
+ _expected_nested = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': {'name': 'Joseph', 'age': '22'}}}
def _get_exporter(self, **kwargs):
return JsonLinesItemExporter(self.output, **kwargs)
@@ -485,9 +485,9 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.assertEqual(exported, ItemAdapter(self.i).asdict())
def test_nested_item(self):
- i1 = self.item_class(name=u'Joseph', age='22')
- i2 = dict(name=u'Maria', age=i1)
- i3 = self.item_class(name=u'Jesus', age=i2)
+ i1 = self.item_class(name='Joseph', age='22')
+ i2 = dict(name='Maria', age=i1)
+ i3 = self.item_class(name='Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
@@ -542,25 +542,25 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
def test_nested_item(self):
- i1 = self.item_class(name=u'Joseph\xa3', age='22')
- i2 = self.item_class(name=u'Maria', age=i1)
- i3 = self.item_class(name=u'Jesus', age=i2)
+ i1 = self.item_class(name='Joseph\xa3', age='22')
+ i2 = self.item_class(name='Maria', age=i1)
+ i3 = self.item_class(name='Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
- expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
+ expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': ItemAdapter(i1).asdict()}}
self.assertEqual(exported, [expected])
def test_nested_dict_item(self):
- i1 = dict(name=u'Joseph\xa3', age='22')
- i2 = self.item_class(name=u'Maria', age=i1)
- i3 = dict(name=u'Jesus', age=i2)
+ i1 = dict(name='Joseph\xa3', age='22')
+ i2 = self.item_class(name='Maria', age=i1)
+ i3 = dict(name='Jesus', age=i2)
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
exported = json.loads(to_unicode(self.output.getvalue()))
- expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
+ expected = {'name': 'Jesus', 'age': {'name': 'Maria', 'age': i1}}
self.assertEqual(exported, [expected])
def test_nonstring_types_item(self):
@@ -595,14 +595,14 @@ class CustomExporterItemTest(unittest.TestCase):
else:
return super(CustomItemExporter, self).serialize_field(field, name, value)
- i = self.item_class(name=u'John', age='22')
+ i = self.item_class(name='John', age='22')
a = ItemAdapter(i)
ie = CustomItemExporter()
self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John')
self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '23')
- i2 = {'name': u'John', 'age': '22'}
+ i2 = {'name': 'John', 'age': '22'}
self.assertEqual(ie.serialize_field({}, 'name', i2['name']), 'John')
self.assertEqual(ie.serialize_field({}, 'age', i2['age']), '23')
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index c49b2e92f..b57349848 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -874,7 +874,7 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_encoding(self):
- items = [dict({'foo': u'Test\xd6'})]
+ items = [dict({'foo': 'Test\xd6'})]
formats = {
'json': '[{"foo": "Test\\u00d6"}]'.encode('utf-8'),
@@ -919,7 +919,7 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_multiple_configs(self):
- items = [dict({'foo': u'FOO', 'bar': u'BAR'})]
+ items = [dict({'foo': 'FOO', 'bar': 'BAR'})]
formats = {
'json': '[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1393,7 +1393,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_export_multiple_configs(self):
- items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+ items = [dict({'foo': 'FOO', 'bar': 'BAR'}), dict({'foo': 'FOO1', 'bar': 'BAR1'})]
formats = {
'json': ['[\n{"bar": "BAR"}\n]'.encode('utf-8'),
@@ -1442,7 +1442,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_batch_item_count_feeds_setting(self):
- items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
+ items = [dict({'foo': 'FOO'}), dict({'foo': 'FOO1'})]
formats = {
'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
'[{"foo": "FOO1"}]'.encode('utf-8')],
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index cf3fc8496..64ff7a73d 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -39,19 +39,19 @@ class HeadersTest(unittest.TestCase):
assert h.getlist('X-Forwarded-For') is not hlist
def test_encode_utf8(self):
- h = Headers({u'key': u'\xa3'}, encoding='utf-8')
+ h = Headers({'key': '\xa3'}, encoding='utf-8')
key, val = dict(h).popitem()
assert isinstance(key, bytes), key
assert isinstance(val[0], bytes), val[0]
self.assertEqual(val[0], b'\xc2\xa3')
def test_encode_latin1(self):
- h = Headers({u'key': u'\xa3'}, encoding='latin1')
+ h = Headers({'key': '\xa3'}, encoding='latin1')
key, val = dict(h).popitem()
self.assertEqual(val[0], b'\xa3')
def test_encode_multiple(self):
- h = Headers({u'key': [u'\xa3']}, encoding='utf-8')
+ h = Headers({'key': ['\xa3']}, encoding='utf-8')
key, val = dict(h).popitem()
self.assertEqual(val[0], b'\xc2\xa3')
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 63014b22d..f5cf4e798 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -60,8 +60,8 @@ class RequestTest(unittest.TestCase):
self.assertFalse(p.headers is r.headers)
# headers must not be unicode
- h = Headers({'key1': u'val1', u'key2': 'val2'})
- h[u'newkey'] = u'newval'
+ h = Headers({'key1': 'val1', 'key2': 'val2'})
+ h['newkey'] = 'newval'
for k, v in h.items():
self.assertIsInstance(k, bytes)
for s in v:
@@ -89,30 +89,30 @@ class RequestTest(unittest.TestCase):
self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
def test_url_encoding(self):
- r = self.request_class(url=u"http://www.scrapy.org/price/£")
+ r = self.request_class(url="http://www.scrapy.org/price/£")
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
def test_url_encoding_other(self):
# encoding affects only query part of URI, not path
# path part should always be UTF-8 encoded before percent-escaping
- r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="utf-8")
+ r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
- r = self.request_class(url=u"http://www.scrapy.org/price/£", encoding="latin1")
+ r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3")
def test_url_encoding_query(self):
- r1 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ")
+ r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
# should be same as above
- r2 = self.request_class(url=u"http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
+ r2 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8")
self.assertEqual(r2.url, "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5")
def test_url_encoding_query_latin1(self):
# encoding is used for encoding query-string before percent-escaping;
# path is still UTF-8 encoded before percent-escaping
- r3 = self.request_class(url=u"http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
+ r3 = self.request_class(url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1")
self.assertEqual(r3.url, "http://www.scrapy.org/price/%C2%B5?currency=%A3")
def test_url_encoding_nonutf8_untouched(self):
@@ -131,16 +131,16 @@ class RequestTest(unittest.TestCase):
# characters. Otherwise, in the future the IRI will be mapped to
# "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
# URI from "http://www.example.org/r%E9sum%E9.html".
- r1 = self.request_class(url=u"http://www.scrapy.org/price/%a3")
+ r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
self.assertEqual(r1.url, "http://www.scrapy.org/price/%a3")
- r2 = self.request_class(url=u"http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+ r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
self.assertEqual(r2.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
- r3 = self.request_class(url=u"http://www.scrapy.org/résumé/%a3")
+ r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
self.assertEqual(r3.url, "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
- r4 = self.request_class(url=u"http://www.example.org/r%E9sum%E9.html")
+ r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
self.assertEqual(r4.url, "http://www.example.org/r%E9sum%E9.html")
def test_body(self):
@@ -151,11 +151,11 @@ class RequestTest(unittest.TestCase):
assert isinstance(r2.body, bytes)
self.assertEqual(r2.encoding, 'utf-8') # default encoding
- r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
+ r3 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='utf-8')
assert isinstance(r3.body, bytes)
self.assertEqual(r3.body, b"Price: \xc2\xa3100")
- r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
+ r4 = self.request_class(url="http://www.example.com/", body="Price: \xa3100", encoding='latin1')
assert isinstance(r4.body, bytes)
self.assertEqual(r4.body, b"Price: \xa3100")
@@ -164,7 +164,7 @@ class RequestTest(unittest.TestCase):
r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
# unicode url
- r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
+ r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue")
def test_copy(self):
@@ -236,7 +236,7 @@ class RequestTest(unittest.TestCase):
assert r4.dont_filter is False
def test_method_always_str(self):
- r = self.request_class("http://www.example.com", method=u"POST")
+ r = self.request_class("http://www.example.com", method="POST")
assert isinstance(r.method, str)
def test_immutable_attributes(self):
@@ -381,7 +381,7 @@ class FormRequestTest(RequestTest):
def test_default_encoding_textual_data(self):
# using default encoding (utf-8)
- data = {u'µ one': u'two', u'price': u'£ 100'}
+ data = {'µ one': 'two', 'price': '£ 100'}
r2 = self.request_class("http://www.example.com", formdata=data)
self.assertEqual(r2.method, 'POST')
self.assertEqual(r2.encoding, 'utf-8')
@@ -390,7 +390,7 @@ class FormRequestTest(RequestTest):
def test_default_encoding_mixed_data(self):
# using default encoding (utf-8)
- data = {u'\u00b5one': b'two', b'price\xc2\xa3': u'\u00a3 100'}
+ data = {'\u00b5one': b'two', b'price\xc2\xa3': '\u00a3 100'}
r2 = self.request_class("http://www.example.com", formdata=data)
self.assertEqual(r2.method, 'POST')
self.assertEqual(r2.encoding, 'utf-8')
@@ -406,14 +406,14 @@ class FormRequestTest(RequestTest):
self.assertEqual(r2.headers[b'Content-Type'], b'application/x-www-form-urlencoded')
def test_custom_encoding_textual_data(self):
- data = {'price': u'£ 100'}
+ data = {'price': '£ 100'}
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
self.assertEqual(r3.encoding, 'latin1')
self.assertEqual(r3.body, b'price=%A3+100')
def test_multi_key_values(self):
# using multiples values for a single key
- data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
+ data = {'price': '\xa3 100', 'colours': ['red', 'blue', 'green']}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100')
@@ -450,10 +450,10 @@ class FormRequestTest(RequestTest):
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
self.assertEqual(req.url, "http://www.example.com/this/post.php")
fs = _qs(req, to_unicode=True)
- self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
- self.assertEqual(set(fs[u'one']), {u'two', u'three'})
- self.assertEqual(fs[u'test2'], [u'xxx µ'])
- self.assertEqual(fs[u'six'], [u'seven'])
+ self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+ self.assertEqual(set(fs['one']), {'two', 'three'})
+ self.assertEqual(fs['test2'], ['xxx µ'])
+ self.assertEqual(fs['six'], ['seven'])
def test_from_response_post_nonascii_bytes_latin1(self):
response = _buildresponse(
@@ -471,14 +471,14 @@ class FormRequestTest(RequestTest):
self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded')
self.assertEqual(req.url, "http://www.example.com/this/post.php")
fs = _qs(req, to_unicode=True, encoding='latin1')
- self.assertEqual(set(fs[u'test £']), {u'val1', u'val2'})
- self.assertEqual(set(fs[u'one']), {u'two', u'three'})
- self.assertEqual(fs[u'test2'], [u'xxx µ'])
- self.assertEqual(fs[u'six'], [u'seven'])
+ self.assertEqual(set(fs['test £']), {'val1', 'val2'})
+ self.assertEqual(set(fs['one']), {'two', 'three'})
+ self.assertEqual(fs['test2'], ['xxx µ'])
+ self.assertEqual(fs['six'], ['seven'])
def test_from_response_post_nonascii_unicode(self):
response = _buildresponse(
- u"""""")
req = self.request_class.from_response(
- response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
+ response, clickdata={'name': 'clickable', 'value': 'clicked2'}
)
fs = _qs(req)
self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -694,21 +694,21 @@ class FormRequestTest(RequestTest):
def test_from_response_unicode_clickdata(self):
response = _buildresponse(
- u"""""")
req = self.request_class.from_response(
- response, clickdata={u'name': u'price in \u00a3'}
+ response, clickdata={'name': 'price in \u00a3'}
)
fs = _qs(req, to_unicode=True)
- self.assertTrue(fs[u'price in \u00a3'])
+ self.assertTrue(fs['price in \u00a3'])
def test_from_response_unicode_clickdata_latin1(self):
response = _buildresponse(
- u"""""",
encoding='latin1')
req = self.request_class.from_response(
- response, clickdata={u'name': u'price in \u00a5'}
+ response, clickdata={'name': 'price in \u00a5'}
)
fs = _qs(req, to_unicode=True, encoding='latin1')
- self.assertTrue(fs[u'price in \u00a5'])
+ self.assertTrue(fs['price in \u00a5'])
def test_from_response_multiple_forms_clickdata(self):
response = _buildresponse(
@@ -733,7 +733,7 @@ class FormRequestTest(RequestTest):
""")
req = self.request_class.from_response(
- response, formname='form2', clickdata={u'name': u'clickable'}
+ response, formname='form2', clickdata={'name': 'clickable'}
)
fs = _qs(req)
self.assertEqual(fs[b'clickable'], [b'clicked2'])
@@ -1072,11 +1072,11 @@ class FormRequestTest(RequestTest):
def test_from_response_unicode_xpath(self):
response = _buildresponse(b'')
- r = self.request_class.from_response(response, formxpath=u"//form[@name='\u044a']")
+ r = self.request_class.from_response(response, formxpath="//form[@name='\u044a']")
fs = _qs(r)
self.assertEqual(fs, {})
- xpath = u"//form[@name='\u03b1']"
+ xpath = "//form[@name='\u03b1']"
self.assertRaisesRegex(ValueError, re.escape(xpath),
self.request_class.from_response,
response, formxpath=xpath)
@@ -1246,13 +1246,13 @@ class XmlRpcRequestTest(RequestTest):
self._test_request(params=('value',))
self._test_request(params=('username', 'password'), methodname='login')
self._test_request(params=('response', ), methodresponse='login')
- self._test_request(params=(u'pas£',), encoding='utf-8')
+ self._test_request(params=('pas£',), encoding='utf-8')
self._test_request(params=(None,), allow_none=1)
self.assertRaises(TypeError, self._test_request)
self.assertRaises(TypeError, self._test_request, params=(None,))
def test_latin1(self):
- self._test_request(params=(u'pas£',), encoding='latin1')
+ self._test_request(params=('pas£',), encoding='latin1')
class JsonRequestTest(RequestTest):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index e0ca3c0e6..56d017de6 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -318,28 +318,28 @@ class TextResponseTest(BaseResponseTest):
def test_unicode_url(self):
# instantiate with unicode url without encoding (should set default encoding)
- resp = self.response_class(u"http://www.example.com/")
+ resp = self.response_class("http://www.example.com/")
self._assert_response_encoding(resp, self.response_class._DEFAULT_ENCODING)
# make sure urls are converted to str
- resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
+ resp = self.response_class(url="http://www.example.com/", encoding='utf-8')
assert isinstance(resp.url, str)
- resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
+ resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='utf-8')
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
- resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
+ resp = self.response_class(url="http://www.example.com/price/\xa3", encoding='latin-1')
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
- resp = self.response_class(u"http://www.example.com/price/\xa3",
+ resp = self.response_class("http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]})
self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
- resp = self.response_class(u"http://www.example.com/price/\xa3",
+ resp = self.response_class("http://www.example.com/price/\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
def test_unicode_body(self):
unicode_string = ('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 '
'\u0442\u0435\u043a\u0441\u0442')
- self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
+ self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body='unicode body')
original_string = unicode_string.encode('cp1251')
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
@@ -355,7 +355,7 @@ class TextResponseTest(BaseResponseTest):
def test_encoding(self):
r1 = self.response_class("http://www.example.com", body=b"\xc2\xa3",
headers={"Content-type": ["text/html; charset=utf-8"]})
- r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
+ r2 = self.response_class("http://www.example.com", encoding='utf-8', body="\xa3")
r3 = self.response_class("http://www.example.com", body=b"\xa3",
headers={"Content-type": ["text/html; charset=iso-8859-1"]})
r4 = self.response_class("http://www.example.com", body=b"\xa2\xa3")
@@ -376,14 +376,14 @@ class TextResponseTest(BaseResponseTest):
self.assertEqual(r5._headers_encoding(), None)
self._assert_response_encoding(r5, "utf-8")
assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
- self._assert_response_values(r1, 'utf-8', u"\xa3")
- self._assert_response_values(r2, 'utf-8', u"\xa3")
- self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
- self._assert_response_values(r6, 'gb18030', u"\u2015")
- self._assert_response_values(r7, 'gb18030', u"\u2015")
+ self._assert_response_values(r1, 'utf-8', "\xa3")
+ self._assert_response_values(r2, 'utf-8', "\xa3")
+ self._assert_response_values(r3, 'iso-8859-1', "\xa3")
+ self._assert_response_values(r6, 'gb18030', "\u2015")
+ self._assert_response_values(r7, 'gb18030', "\u2015")
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
- self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
+ self.assertRaises(TypeError, self.response_class, "http://www.example.com", body="\xa3")
def test_declared_encoding_invalid(self):
"""Check that unknown declared encodings are ignored"""
@@ -391,14 +391,14 @@ class TextResponseTest(BaseResponseTest):
headers={"Content-type": ["text/html; charset=UKNOWN"]},
body=b"\xc2\xa3")
self.assertEqual(r._declared_encoding(), None)
- self._assert_response_values(r, 'utf-8', u"\xa3")
+ self._assert_response_values(r, 'utf-8', "\xa3")
def test_utf16(self):
"""Test utf-16 because UnicodeDammit is known to have problems with"""
r = self.response_class("http://www.example.com",
body=b'\xff\xfeh\x00i\x00',
encoding='utf-16')
- self._assert_response_values(r, 'utf-16', u"hi")
+ self._assert_response_values(r, 'utf-16', "hi")
def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
r6 = self.response_class("http://www.example.com",
@@ -406,8 +406,8 @@ class TextResponseTest(BaseResponseTest):
body=b"\xef\xbb\xbfWORD\xe3\xab")
self.assertEqual(r6.encoding, 'utf-8')
self.assertIn(r6.text, {
- u'WORD\ufffd\ufffd', # w3lib < 1.19.0
- u'WORD\ufffd', # w3lib >= 1.19.0
+ 'WORD\ufffd\ufffd', # w3lib < 1.19.0
+ 'WORD\ufffd', # w3lib >= 1.19.0
})
def test_bom_is_removed_from_body(self):
@@ -422,9 +422,9 @@ class TextResponseTest(BaseResponseTest):
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
self.assertEqual(response.encoding, 'utf-8')
- self.assertEqual(response.text, u'WORD')
+ self.assertEqual(response.text, 'WORD')
response = self.response_class(url, body=body)
- self.assertEqual(response.text, u'WORD')
+ self.assertEqual(response.text, 'WORD')
self.assertEqual(response.encoding, 'utf-8')
# Body caching sideeffect isn't triggered when encoding is declared in
@@ -432,28 +432,28 @@ class TextResponseTest(BaseResponseTest):
# body
response = self.response_class(url, headers=headers, body=body)
self.assertEqual(response.encoding, 'utf-8')
- self.assertEqual(response.text, u'WORD')
+ self.assertEqual(response.text, 'WORD')
response = self.response_class(url, headers=headers, body=body)
- self.assertEqual(response.text, u'WORD')
+ self.assertEqual(response.text, 'WORD')
self.assertEqual(response.encoding, 'utf-8')
def test_replace_wrong_encoding(self):
"""Test invalid chars are replaced properly"""
r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
# XXX: Policy for replacing invalid chars may suffer minor variations
- # but it should always contain the unicode replacement char (u'\ufffd')
- assert u'\ufffd' in r.text, repr(r.text)
- assert u'PREFIX' in r.text, repr(r.text)
- assert u'SUFFIX' in r.text, repr(r.text)
+ # but it should always contain the unicode replacement char ('\ufffd')
+ assert '\ufffd' in r.text, repr(r.text)
+ assert 'PREFIX' in r.text, repr(r.text)
+ assert 'SUFFIX' in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
r = self.response_class("http://example.com", encoding='utf-8',
body=b'\xf0value')
- assert u'value' in r.text, repr(r.text)
+ assert 'value' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
- # assert u'\ufffd' in r.text, repr(r.text)
+ # assert '\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"Some page"
@@ -466,15 +466,15 @@ class TextResponseTest(BaseResponseTest):
self.assertEqual(
response.selector.xpath("//title/text()").getall(),
- [u'Some page']
+ ['Some page']
)
self.assertEqual(
response.selector.css("title::text").getall(),
- [u'Some page']
+ ['Some page']
)
self.assertEqual(
response.selector.re("Some (.*)"),
- [u'page']
+ ['page']
)
def test_selector_shortcuts(self):
@@ -595,7 +595,7 @@ class TextResponseTest(BaseResponseTest):
resp1 = self.response_class(
'http://example.com',
encoding='utf8',
- body=u'click me'.encode('utf8')
+ body='click me'.encode('utf8')
)
req = self._assert_followed_url(
resp1.css('a')[0],
@@ -607,7 +607,7 @@ class TextResponseTest(BaseResponseTest):
resp2 = self.response_class(
'http://example.com',
encoding='cp1251',
- body=u'click me'.encode('cp1251')
+ body='click me'.encode('cp1251')
)
req = self._assert_followed_url(
resp2.css('a')[0],
@@ -681,8 +681,8 @@ class TextResponseTest(BaseResponseTest):
def test_body_as_unicode_deprecation_warning(self):
with catch_warnings(record=True) as warnings:
- r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8')
- self.assertEqual(r1.body_as_unicode(), u'Hello')
+ r1 = self.response_class("http://www.example.com", body='Hello', encoding='utf-8')
+ self.assertEqual(r1.body_as_unicode(), 'Hello')
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].category, ScrapyDeprecationWarning)
@@ -787,7 +787,7 @@ class XmlResponseTest(TextResponseTest):
self.assertEqual(
response.selector.xpath("//elem/text()").getall(),
- [u'value']
+ ['value']
)
def test_selector_shortcuts(self):
diff --git a/tests/test_item.py b/tests/test_item.py
index 60468971c..0ce78f8c0 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -20,8 +20,8 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
- i['name'] = u'name'
- self.assertEqual(i['name'], u'name')
+ i['name'] = 'name'
+ self.assertEqual(i['name'], 'name')
def test_init(self):
class TestItem(Item):
@@ -30,17 +30,17 @@ class ItemTest(unittest.TestCase):
i = TestItem()
self.assertRaises(KeyError, i.__getitem__, 'name')
- i2 = TestItem(name=u'john doe')
- self.assertEqual(i2['name'], u'john doe')
+ i2 = TestItem(name='john doe')
+ self.assertEqual(i2['name'], 'john doe')
- i3 = TestItem({'name': u'john doe'})
- self.assertEqual(i3['name'], u'john doe')
+ i3 = TestItem({'name': 'john doe'})
+ self.assertEqual(i3['name'], 'john doe')
i4 = TestItem(i3)
- self.assertEqual(i4['name'], u'john doe')
+ self.assertEqual(i4['name'], 'john doe')
- self.assertRaises(KeyError, TestItem, {'name': u'john doe',
- 'other': u'foo'})
+ self.assertRaises(KeyError, TestItem, {'name': 'john doe',
+ 'other': 'foo'})
def test_invalid_field(self):
class TestItem(Item):
@@ -56,7 +56,7 @@ class ItemTest(unittest.TestCase):
number = Field()
i = TestItem()
- i['name'] = u'John Doe'
+ i['name'] = 'John Doe'
i['number'] = 123
itemrepr = repr(i)
@@ -101,9 +101,9 @@ class ItemTest(unittest.TestCase):
i = TestItem()
self.assertRaises(KeyError, i.get_name)
- i['name'] = u'lala'
- self.assertEqual(i.get_name(), u'lala')
- i.change_name(u'other')
+ i['name'] = 'lala'
+ self.assertEqual(i.get_name(), 'lala')
+ i.change_name('other')
self.assertEqual(i.get_name(), 'other')
def test_metaclass(self):
@@ -113,22 +113,22 @@ class ItemTest(unittest.TestCase):
values = Field()
i = TestItem()
- i['name'] = u'John'
+ i['name'] = 'John'
self.assertEqual(list(i.keys()), ['name'])
self.assertEqual(list(i.values()), ['John'])
- i['keys'] = u'Keys'
- i['values'] = u'Values'
+ i['keys'] = 'Keys'
+ i['values'] = 'Values'
self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name'])
- self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John'])
+ self.assertSortedEqual(list(i.values()), ['Keys', 'Values', 'John'])
def test_metaclass_with_fields_attribute(self):
class TestItem(Item):
fields = {'new': Field(default='X')}
- item = TestItem(new=u'New')
+ item = TestItem(new='New')
self.assertSortedEqual(list(item.keys()), ['new'])
- self.assertSortedEqual(list(item.values()), [u'New'])
+ self.assertSortedEqual(list(item.values()), ['New'])
def test_metaclass_inheritance(self):
class ParentItem(Item):
@@ -238,8 +238,8 @@ class ItemTest(unittest.TestCase):
name = Field()
i = TestItem()
- i['name'] = u'John'
- self.assertEqual(dict(i), {'name': u'John'})
+ i['name'] = 'John'
+ self.assertEqual(dict(i), {'name': 'John'})
def test_copy(self):
class TestItem(Item):
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 8d4538eed..a0bafa5e5 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -31,31 +31,31 @@ class Base:
page4_url = 'http://example.com/page%204.html'
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=u''),
- Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url=page4_url, text=u'href with whitespaces'),
+ Link(url='http://www.google.com/something', text=''),
+ Link(url='http://example.com/innertag.html', text='inner tag'),
+ Link(url=page4_url, text='href with whitespaces'),
])
def test_extract_filter_allow(self):
lx = self.extractor_cls(allow=('sample', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
])
def test_extract_filter_allow_with_duplicates(self):
lx = self.extractor_cls(allow=('sample', ), unique=False)
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
+ Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
])
@@ -63,10 +63,10 @@ class Base:
lx = self.extractor_cls(allow=('sample', ), unique=False,
canonicalize=True)
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
+ Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
])
@@ -74,22 +74,22 @@ class Base:
lx = self.extractor_cls(allow=('sample',), unique=True,
canonicalize=True)
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
])
def test_extract_filter_allow_and_deny(self):
lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
])
def test_extract_filter_allowed_domains(self):
lx = self.extractor_cls(allow_domains=('google.com', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://www.google.com/something', text=u''),
+ Link(url='http://www.google.com/something', text=''),
])
def test_extraction_using_single_values(self):
@@ -97,27 +97,27 @@ class Base:
lx = self.extractor_cls(allow='sample')
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo',
text='sample 3 repetition with fragment')
])
lx = self.extractor_cls(allow='sample', deny='3')
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
])
lx = self.extractor_cls(allow_domains='google.com')
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://www.google.com/something', text=u''),
+ Link(url='http://www.google.com/something', text=''),
])
lx = self.extractor_cls(deny_domains='example.com')
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://www.google.com/something', text=u''),
+ Link(url='http://www.google.com/something', text=''),
])
def test_nofollow(self):
@@ -145,11 +145,11 @@ class Base:
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.org/about.html', text=u'About us'),
- Link(url='http://example.org/follow.html', text=u'Follow this link'),
- Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
- Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
- Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+ Link(url='http://example.org/about.html', text='About us'),
+ Link(url='http://example.org/follow.html', text='Follow this link'),
+ Link(url='http://example.org/nofollow.html', text='Dont follow this one', nofollow=True),
+ Link(url='http://example.org/nofollow2.html', text='Choose to follow or not'),
+ Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
])
def test_matches(self):
@@ -183,8 +183,8 @@ class Base:
def test_restrict_xpaths(self):
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
])
def test_restrict_xpaths_encoding(self):
@@ -202,14 +202,14 @@ class Base:
lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/about.html', text=u'About us\xa3')])
+ [Link(url='http://example.org/about.html', text='About us\xa3')])
def test_restrict_xpaths_with_html_entities(self):
html = b'
text
'
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
self.assertEqual(links,
- [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
+ [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text='text')])
def test_restrict_xpaths_concat_in_handle_data(self):
"""html entities cause SGMLParser to call handle_data hook twice"""
@@ -217,22 +217,22 @@ class Base:
response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
lx = self.extractor_cls(restrict_xpaths="//div")
self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
+ [Link(url='http://example.org/foo', text='>\u4eac<\u4e1c',
fragment='', nofollow=False)])
def test_restrict_css(self):
lx = self.extractor_cls(restrict_css=('#subwrapper a',))
self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample2.html', text=u'sample 2')
+ Link(url='http://example.com/sample2.html', text='sample 2')
])
def test_restrict_css_and_restrict_xpaths_together(self):
lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ),
restrict_css=('#subwrapper + a', ))
self.assertEqual([link for link in lx.extract_links(self.response)], [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
])
def test_area_tag_with_unicode_present(self):
@@ -243,7 +243,7 @@ class Base:
lx.extract_links(response)
lx.extract_links(response)
self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/foo', text=u'',
+ [Link(url='http://example.org/foo', text='',
fragment='', nofollow=False)])
def test_encoded_url(self):
@@ -251,7 +251,7 @@ class Base:
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
- Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+ Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
])
def test_encoded_url_in_restricted_xpath(self):
@@ -259,7 +259,7 @@ class Base:
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
lx = self.extractor_cls(restrict_xpaths="//div")
self.assertEqual(lx.extract_links(response), [
- Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
+ Link(url='http://known.fm/AC%2FDC/?page=2', text='BinB', fragment='', nofollow=False),
])
def test_ignored_extensions(self):
@@ -268,7 +268,7 @@ class Base:
response = HtmlResponse("http://example.org/", body=html)
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.org/page.html', text=u'asd'),
+ Link(url='http://example.org/page.html', text='asd'),
])
# override denied extensions
@@ -308,25 +308,25 @@ class Base:
page4_url = 'http://example.com/page%204.html'
self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=u''),
- Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url=page4_url, text=u'href with whitespaces'),
+ Link(url='http://www.google.com/something', text=''),
+ Link(url='http://example.com/innertag.html', text='inner tag'),
+ Link(url=page4_url, text='href with whitespaces'),
])
lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=())
self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample2.jpg', text=u''),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample2.jpg', text=''),
+ Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=u''),
- Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url=page4_url, text=u'href with whitespaces'),
+ Link(url='http://www.google.com/something', text=''),
+ Link(url='http://example.com/innertag.html', text='inner tag'),
+ Link(url=page4_url, text='href with whitespaces'),
])
lx = self.extractor_cls(attrs=None)
@@ -344,24 +344,24 @@ class Base:
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample1.html', text=u''),
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample1.html', text=''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
])
lx = self.extractor_cls(tags="area")
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample1.html', text=''),
])
lx = self.extractor_cls(tags="a")
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
])
lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=())
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample2.jpg', text=u''),
+ Link(url='http://example.com/sample2.html', text='sample 2'),
+ Link(url='http://example.com/sample2.jpg', text=''),
])
def test_tags_attrs(self):
@@ -375,14 +375,14 @@ class Base:
lx = self.extractor_cls(tags='div', attrs='data-url')
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
- Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+ Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+ Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
])
lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
- Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+ Link(url='http://example.com/get?id=1', text='Item 1', fragment='', nofollow=False),
+ Link(url='http://example.com/get?id=2', text='Item 2', fragment='', nofollow=False)
])
def test_xhtml(self):
@@ -420,13 +420,13 @@ class Base:
self.assertEqual(
lx.extract_links(response),
[
- Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
- Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+ Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+ Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+ Link(url='http://example.com/nofollow.html', text='Dont follow this one',
fragment='', nofollow=True),
- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+ Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
fragment='', nofollow=False),
- Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+ Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
]
)
@@ -436,13 +436,13 @@ class Base:
self.assertEqual(
lx.extract_links(response),
[
- Link(url='http://example.com/about.html', text=u'About us', fragment='', nofollow=False),
- Link(url='http://example.com/follow.html', text=u'Follow this link', fragment='', nofollow=False),
- Link(url='http://example.com/nofollow.html', text=u'Dont follow this one',
+ Link(url='http://example.com/about.html', text='About us', fragment='', nofollow=False),
+ Link(url='http://example.com/follow.html', text='Follow this link', fragment='', nofollow=False),
+ Link(url='http://example.com/nofollow.html', text='Dont follow this one',
fragment='', nofollow=True),
- Link(url='http://example.com/nofollow2.html', text=u'Choose to follow or not',
+ Link(url='http://example.com/nofollow2.html', text='Choose to follow or not',
fragment='', nofollow=False),
- Link(url='http://google.com/something', text=u'External link not to follow', nofollow=True),
+ Link(url='http://google.com/something', text='External link not to follow', nofollow=True),
]
)
@@ -455,8 +455,8 @@ class Base:
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+ Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+ Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
])
def test_ftp_links(self):
@@ -467,7 +467,7 @@ class Base:
response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8')
lx = self.extractor_cls()
self.assertEqual(lx.extract_links(response), [
- Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False),
+ Link(url='ftp://www.external.com/', text='An Item', fragment='', nofollow=False),
])
def test_pickle_extractor(self):
@@ -487,8 +487,8 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
response = HtmlResponse("http://example.org/index.html", body=html)
lx = self.extractor_cls()
self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
+ Link(url='http://example.org/item1.html', text='Item 1', nofollow=False),
+ Link(url='http://example.org/item3.html', text='Item 3', nofollow=False),
])
def test_link_restrict_text(self):
@@ -501,18 +501,18 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
# Simple text inclusion test
lx = self.extractor_cls(restrict_text='dog')
self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+ Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
])
# Unique regex test
lx = self.extractor_cls(restrict_text=r'of.*dog')
self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+ Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
])
# Multiple regex test
lx = self.extractor_cls(restrict_text=[r'of.*dog', r'of.*cat'])
self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text=u'Pic of a cat', nofollow=False),
- Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
+ Link(url='http://example.org/item1.html', text='Pic of a cat', nofollow=False),
+ Link(url='http://example.org/item2.html', text='Pic of a dog', nofollow=False),
])
def test_restrict_xpaths_with_html_entities(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 581183625..2ed6f365f 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -69,23 +69,23 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_add_value_on_unknown_field(self):
il = TestItemLoader()
- self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo'])
+ self.assertRaises(KeyError, il.add_value, 'wrong_field', ['lala', 'lolo'])
def test_load_item_using_default_loader(self):
i = TestItem()
- i['summary'] = u'lala'
+ i['summary'] = 'lala'
il = ItemLoader(item=i)
- il.add_value('name', u'marta')
+ il.add_value('name', 'marta')
item = il.load_item()
assert item is i
- self.assertEqual(item['summary'], [u'lala'])
- self.assertEqual(item['name'], [u'marta'])
+ self.assertEqual(item['summary'], ['lala'])
+ self.assertEqual(item['name'], ['marta'])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
- il.add_value('name', u'marta')
+ il.add_value('name', 'marta')
item = il.load_item()
- self.assertEqual(item['name'], [u'Marta'])
+ self.assertEqual(item['name'], ['Marta'])
class InitializationTestMixin:
@@ -290,137 +290,137 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertRaises(RuntimeError, l.get_css, '#name::text')
def test_init_method_with_selector(self):
- sel = Selector(text=u"marta
")
+ sel = Selector(text="marta
")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
def test_init_method_with_selector_css(self):
- sel = Selector(text=u"marta
")
+ sel = Selector(text="marta
")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
l.add_css('name', 'div::text')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
def test_init_method_with_response(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
def test_init_method_with_response_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css('name', 'div::text')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
l.add_css('url', 'a::attr(href)')
- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
# combining/accumulating CSS selectors and XPath expressions
l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), [u'Marta', u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta', 'Marta'])
l.add_xpath('url', '//img/@src')
- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org', u'/images/logo.png'])
+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org', '/images/logo.png'])
def test_add_xpath_re(self):
l = TestItemLoader(response=self.response)
l.add_xpath('name', '//div/text()', re='ma')
- self.assertEqual(l.get_output_value('name'), [u'Ma'])
+ self.assertEqual(l.get_output_value('name'), ['Ma'])
def test_replace_xpath(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_xpath('name', '//p/text()')
- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])
l.replace_xpath('name', ['//p/text()', '//div/text()'])
- self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
def test_get_xpath(self):
l = TestItemLoader(response=self.response)
- self.assertEqual(l.get_xpath('//p/text()'), [u'paragraph'])
- self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), u'paragraph')
- self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), u'pa')
+ self.assertEqual(l.get_xpath('//p/text()'), ['paragraph'])
+ self.assertEqual(l.get_xpath('//p/text()', TakeFirst()), 'paragraph')
+ self.assertEqual(l.get_xpath('//p/text()', TakeFirst(), re='pa'), 'pa')
- self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), [u'paragraph', 'marta'])
+ self.assertEqual(l.get_xpath(['//p/text()', '//div/text()']), ['paragraph', 'marta'])
def test_replace_xpath_multi_fields(self):
l = TestItemLoader(response=self.response)
l.add_xpath(None, '//div/text()', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_xpath(None, '//p/text()', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])
def test_replace_xpath_re(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_xpath('name', '//div/text()', re='ma')
- self.assertEqual(l.get_output_value('name'), [u'Ma'])
+ self.assertEqual(l.get_output_value('name'), ['Ma'])
def test_add_css_re(self):
l = TestItemLoader(response=self.response)
l.add_css('name', 'div::text', re='ma')
- self.assertEqual(l.get_output_value('name'), [u'Ma'])
+ self.assertEqual(l.get_output_value('name'), ['Ma'])
l.add_css('url', 'a::attr(href)', re='http://(.+)')
- self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
def test_replace_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css('name', 'div::text')
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_css('name', 'p::text')
- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])
l.replace_css('name', ['p::text', 'div::text'])
- self.assertEqual(l.get_output_value('name'), [u'Paragraph', 'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Paragraph', 'Marta'])
l.add_css('url', 'a::attr(href)', re='http://(.+)')
- self.assertEqual(l.get_output_value('url'), [u'www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['www.scrapy.org'])
l.replace_css('url', 'img::attr(src)')
- self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+ self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
def test_get_css(self):
l = TestItemLoader(response=self.response)
- self.assertEqual(l.get_css('p::text'), [u'paragraph'])
- self.assertEqual(l.get_css('p::text', TakeFirst()), u'paragraph')
- self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), u'pa')
+ self.assertEqual(l.get_css('p::text'), ['paragraph'])
+ self.assertEqual(l.get_css('p::text', TakeFirst()), 'paragraph')
+ self.assertEqual(l.get_css('p::text', TakeFirst(), re='pa'), 'pa')
- self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
+ self.assertEqual(l.get_css(['p::text', 'div::text']), ['paragraph', 'marta'])
self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
- [u'http://www.scrapy.org', u'/images/logo.png'])
+ ['http://www.scrapy.org', '/images/logo.png'])
def test_replace_css_multi_fields(self):
l = TestItemLoader(response=self.response)
l.add_css(None, 'div::text', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), [u'Marta'])
+ self.assertEqual(l.get_output_value('name'), ['Marta'])
l.replace_css(None, 'p::text', TakeFirst(), lambda x: {'name': x})
- self.assertEqual(l.get_output_value('name'), [u'Paragraph'])
+ self.assertEqual(l.get_output_value('name'), ['Paragraph'])
l.add_css(None, 'a::attr(href)', TakeFirst(), lambda x: {'url': x})
- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
l.replace_css(None, 'img::attr(src)', TakeFirst(), lambda x: {'url': x})
- self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+ self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
def test_replace_css_re(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_css('url', 'a::attr(href)')
- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
- self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['scrapy.org'])
class SubselectorLoaderTest(unittest.TestCase):
@@ -447,9 +447,9 @@ class SubselectorLoaderTest(unittest.TestCase):
nl.add_css('name_div', '#id')
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
- self.assertEqual(l.get_output_value('name'), [u'marta'])
- self.assertEqual(l.get_output_value('name_div'), [u'marta
'])
- self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+ self.assertEqual(l.get_output_value('name'), ['marta'])
+ self.assertEqual(l.get_output_value('name_div'), ['marta
'])
+ self.assertEqual(l.get_output_value('name_value'), ['marta'])
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -462,9 +462,9 @@ class SubselectorLoaderTest(unittest.TestCase):
nl.add_css('name_div', '#id')
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
- self.assertEqual(l.get_output_value('name'), [u'marta'])
- self.assertEqual(l.get_output_value('name_div'), [u'marta
'])
- self.assertEqual(l.get_output_value('name_value'), [u'marta'])
+ self.assertEqual(l.get_output_value('name'), ['marta'])
+ self.assertEqual(l.get_output_value('name_div'), ['marta
'])
+ self.assertEqual(l.get_output_value('name_value'), ['marta'])
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
@@ -476,11 +476,11 @@ class SubselectorLoaderTest(unittest.TestCase):
nl2 = nl1.nested_xpath('a')
l.add_xpath('url', '//footer/a/@href')
- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
nl1.replace_xpath('url', 'img/@src')
- self.assertEqual(l.get_output_value('url'), [u'/images/logo.png'])
+ self.assertEqual(l.get_output_value('url'), ['/images/logo.png'])
nl2.replace_xpath('url', '@href')
- self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
+ self.assertEqual(l.get_output_value('url'), ['http://www.scrapy.org'])
def test_nested_ordering(self):
l = NestedItemLoader(response=self.response)
@@ -493,10 +493,10 @@ class SubselectorLoaderTest(unittest.TestCase):
l.add_xpath('url', '//footer/a/@href')
self.assertEqual(l.get_output_value('url'), [
- u'/images/logo.png',
- u'http://www.scrapy.org',
- u'homepage',
- u'http://www.scrapy.org',
+ '/images/logo.png',
+ 'http://www.scrapy.org',
+ 'homepage',
+ 'http://www.scrapy.org',
])
def test_nested_load_item(self):
@@ -514,9 +514,9 @@ class SubselectorLoaderTest(unittest.TestCase):
assert item is nl1.item
assert item is nl2.item
- self.assertEqual(item['name'], [u'marta'])
- self.assertEqual(item['url'], [u'http://www.scrapy.org'])
- self.assertEqual(item['image'], [u'/images/logo.png'])
+ self.assertEqual(item['name'], ['marta'])
+ self.assertEqual(item['url'], ['http://www.scrapy.org'])
+ self.assertEqual(item['image'], ['/images/logo.png'])
# Functions as processors
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index d0a59e8cd..eb14de14f 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -51,19 +51,19 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_load_item_using_default_loader(self):
i = TestItem()
- i['summary'] = u'lala'
+ i['summary'] = 'lala'
il = ItemLoader(item=i)
- il.add_value('name', u'marta')
+ il.add_value('name', 'marta')
item = il.load_item()
assert item is i
- self.assertEqual(item['summary'], [u'lala'])
- self.assertEqual(item['name'], [u'marta'])
+ self.assertEqual(item['summary'], ['lala'])
+ self.assertEqual(item['name'], ['marta'])
def test_load_item_using_custom_loader(self):
il = TestItemLoader()
- il.add_value('name', u'marta')
+ il.add_value('name', 'marta')
item = il.load_item()
- self.assertEqual(item['name'], [u'Marta'])
+ self.assertEqual(item['name'], ['Marta'])
def test_load_item_ignore_none_field_values(self):
def validate_sku(value):
@@ -76,23 +76,23 @@ class BasicItemLoaderTest(unittest.TestCase):
price_out = Compose(TakeFirst(), float)
sku_out = Compose(TakeFirst(), validate_sku)
- valid_fragment = u'SKU: 1234'
- invalid_fragment = u'SKU: not available'
+ valid_fragment = 'SKU: 1234'
+ invalid_fragment = 'SKU: not available'
sku_re = 'SKU: (.+)'
il = MyLoader(item={})
# Should not return "sku: None".
il.add_value('sku', [invalid_fragment], re=sku_re)
# Should not ignore empty values.
- il.add_value('name', u'')
- il.add_value('price', [u'0'])
+ il.add_value('name', '')
+ il.add_value('price', ['0'])
self.assertEqual(il.load_item(), {
- 'name': u'',
+ 'name': '',
'price': 0.0,
})
il.replace_value('sku', [valid_fragment], re=sku_re)
- self.assertEqual(il.load_item()['sku'], u'1234')
+ self.assertEqual(il.load_item()['sku'], '1234')
def test_self_referencing_loader(self):
class MyLoader(ItemLoader):
@@ -117,19 +117,19 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_add_value(self):
il = TestItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_collected_values('name'), [u'Marta'])
- self.assertEqual(il.get_output_value('name'), [u'Marta'])
- il.add_value('name', u'pepe')
- self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe'])
- self.assertEqual(il.get_output_value('name'), [u'Marta', u'Pepe'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_collected_values('name'), ['Marta'])
+ self.assertEqual(il.get_output_value('name'), ['Marta'])
+ il.add_value('name', 'pepe')
+ self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe'])
+ self.assertEqual(il.get_output_value('name'), ['Marta', 'Pepe'])
# test add object value
il.add_value('summary', {'key': 1})
self.assertEqual(il.get_collected_values('summary'), [{'key': 1}])
- il.add_value(None, u'Jim', lambda x: {'name': x})
- self.assertEqual(il.get_collected_values('name'), [u'Marta', u'Pepe', u'Jim'])
+ il.add_value(None, 'Jim', lambda x: {'name': x})
+ self.assertEqual(il.get_collected_values('name'), ['Marta', 'Pepe', 'Jim'])
def test_add_zero(self):
il = NameItemLoader()
@@ -138,49 +138,49 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_replace_value(self):
il = TestItemLoader()
- il.replace_value('name', u'marta')
- self.assertEqual(il.get_collected_values('name'), [u'Marta'])
- self.assertEqual(il.get_output_value('name'), [u'Marta'])
- il.replace_value('name', u'pepe')
- self.assertEqual(il.get_collected_values('name'), [u'Pepe'])
- self.assertEqual(il.get_output_value('name'), [u'Pepe'])
+ il.replace_value('name', 'marta')
+ self.assertEqual(il.get_collected_values('name'), ['Marta'])
+ self.assertEqual(il.get_output_value('name'), ['Marta'])
+ il.replace_value('name', 'pepe')
+ self.assertEqual(il.get_collected_values('name'), ['Pepe'])
+ self.assertEqual(il.get_output_value('name'), ['Pepe'])
- il.replace_value(None, u'Jim', lambda x: {'name': x})
- self.assertEqual(il.get_collected_values('name'), [u'Jim'])
+ il.replace_value(None, 'Jim', lambda x: {'name': x})
+ self.assertEqual(il.get_collected_values('name'), ['Jim'])
def test_get_value(self):
il = NameItemLoader()
- self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
- self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
- self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
+ self.assertEqual('FOO', il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
+ self.assertEqual(['foo', 'bar'], il.get_value(['name:foo', 'name:bar'], re='name:(.*)$'))
+ self.assertEqual('foo', il.get_value(['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$'))
- il.add_value('name', [u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')
- self.assertEqual([u'foo'], il.get_collected_values('name'))
- il.replace_value('name', u'name:bar', re=u'name:(.*)$')
- self.assertEqual([u'bar'], il.get_collected_values('name'))
+ il.add_value('name', ['name:foo', 'name:bar'], TakeFirst(), re='name:(.*)$')
+ self.assertEqual(['foo'], il.get_collected_values('name'))
+ il.replace_value('name', 'name:bar', re='name:(.*)$')
+ self.assertEqual(['bar'], il.get_collected_values('name'))
def test_iter_on_input_processor_input(self):
class NameFirstItemLoader(NameItemLoader):
name_in = TakeFirst()
il = NameFirstItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_collected_values('name'), [u'marta'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
- il.add_value('name', [u'marta', u'jose'])
- self.assertEqual(il.get_collected_values('name'), [u'marta'])
+ il.add_value('name', ['marta', 'jose'])
+ self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
- il.replace_value('name', u'marta')
- self.assertEqual(il.get_collected_values('name'), [u'marta'])
+ il.replace_value('name', 'marta')
+ self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
- il.replace_value('name', [u'marta', u'jose'])
- self.assertEqual(il.get_collected_values('name'), [u'marta'])
+ il.replace_value('name', ['marta', 'jose'])
+ self.assertEqual(il.get_collected_values('name'), ['marta'])
il = NameFirstItemLoader()
- il.add_value('name', u'marta')
- il.add_value('name', [u'jose', u'pedro'])
- self.assertEqual(il.get_collected_values('name'), [u'marta', u'jose'])
+ il.add_value('name', 'marta')
+ il.add_value('name', ['jose', 'pedro'])
+ self.assertEqual(il.get_collected_values('name'), ['marta', 'jose'])
def test_map_compose_filter(self):
def filter_world(x):
@@ -195,87 +195,87 @@ class BasicItemLoaderTest(unittest.TestCase):
name_in = MapCompose(lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'Mart'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['Mart'])
item = il.load_item()
- self.assertEqual(item['name'], [u'Mart'])
+ self.assertEqual(item['name'], ['Mart'])
def test_default_input_processor(self):
il = DefaultedItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'mart'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['mart'])
def test_inherited_default_input_processor(self):
class InheritDefaultedItemLoader(DefaultedItemLoader):
pass
il = InheritDefaultedItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'mart'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['mart'])
def test_input_processor_inheritance(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(lambda v: v.lower())
il = ChildItemLoader()
- il.add_value('url', u'HTTP://scrapy.ORG')
- self.assertEqual(il.get_output_value('url'), [u'http://scrapy.org'])
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'Marta'])
+ il.add_value('url', 'HTTP://scrapy.ORG')
+ self.assertEqual(il.get_output_value('url'), ['http://scrapy.org'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['Marta'])
class ChildChildItemLoader(ChildItemLoader):
url_in = MapCompose(lambda v: v.upper())
summary_in = MapCompose(lambda v: v)
il = ChildChildItemLoader()
- il.add_value('url', u'http://scrapy.org')
- self.assertEqual(il.get_output_value('url'), [u'HTTP://SCRAPY.ORG'])
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'Marta'])
+ il.add_value('url', 'http://scrapy.org')
+ self.assertEqual(il.get_output_value('url'), ['HTTP://SCRAPY.ORG'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['Marta'])
def test_empty_map_compose(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose()
il = IdentityDefaultedItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'marta'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['marta'])
def test_identity_input_processor(self):
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = Identity()
il = IdentityDefaultedItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'marta'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['marta'])
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'mARTA'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['mARTA'])
def test_extend_default_input_processors(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
il = ChildDefaultedItemLoader()
- il.add_value('name', u'marta')
- self.assertEqual(il.get_output_value('name'), [u'MART'])
+ il.add_value('name', 'marta')
+ self.assertEqual(il.get_output_value('name'), ['MART'])
def test_output_processor_using_function(self):
il = TestItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
class TakeFirstItemLoader(TestItemLoader):
- name_out = u" ".join
+ name_out = " ".join
il = TakeFirstItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), 'Mar Ta')
def test_output_processor_error(self):
class TestItemLoader(ItemLoader):
@@ -283,9 +283,9 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = MapCompose(float)
il = TestItemLoader()
- il.add_value('name', [u'$10'])
+ il.add_value('name', ['$10'])
try:
- float(u'$10')
+ float('$10')
except Exception as e:
expected_exc_str = str(e)
@@ -303,53 +303,53 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_output_processor_using_classes(self):
il = TestItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
class TakeFirstItemLoader(TestItemLoader):
name_out = Join()
il = TakeFirstItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), u'Mar Ta')
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), 'Mar Ta')
class TakeFirstItemLoader(TestItemLoader):
name_out = Join("
")
il = TakeFirstItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), u'Mar
Ta')
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), 'Mar
Ta')
def test_default_output_processor(self):
il = TestItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
class LalaItemLoader(TestItemLoader):
default_output_processor = Identity()
il = LalaItemLoader()
- il.add_value('name', [u'mar', u'ta'])
- self.assertEqual(il.get_output_value('name'), [u'Mar', u'Ta'])
+ il.add_value('name', ['mar', 'ta'])
+ self.assertEqual(il.get_output_value('name'), ['Mar', 'Ta'])
def test_loader_context_on_declaration(self):
class ChildItemLoader(TestItemLoader):
- url_in = MapCompose(processor_with_args, key=u'val')
+ url_in = MapCompose(processor_with_args, key='val')
il = ChildItemLoader()
- il.add_value('url', u'text')
+ il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['val'])
- il.replace_value('url', u'text2')
+ il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['val'])
def test_loader_context_on_instantiation(self):
class ChildItemLoader(TestItemLoader):
url_in = MapCompose(processor_with_args)
- il = ChildItemLoader(key=u'val')
- il.add_value('url', u'text')
+ il = ChildItemLoader(key='val')
+ il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['val'])
- il.replace_value('url', u'text2')
+ il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['val'])
def test_loader_context_on_assign(self):
@@ -357,10 +357,10 @@ class BasicItemLoaderTest(unittest.TestCase):
url_in = MapCompose(processor_with_args)
il = ChildItemLoader()
- il.context['key'] = u'val'
- il.add_value('url', u'text')
+ il.context['key'] = 'val'
+ il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['val'])
- il.replace_value('url', u'text2')
+ il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['val'])
def test_item_passed_to_input_processor_functions(self):
@@ -372,9 +372,9 @@ class BasicItemLoaderTest(unittest.TestCase):
it = TestItem(name='marta')
il = ChildItemLoader(item=it)
- il.add_value('url', u'text')
+ il.add_value('url', 'text')
self.assertEqual(il.get_output_value('url'), ['marta'])
- il.replace_value('url', u'text2')
+ il.replace_value('url', 'text2')
self.assertEqual(il.get_output_value('url'), ['marta'])
def test_compose_processor(self):
@@ -382,10 +382,10 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = Compose(lambda v: v[0], lambda v: v.title(), lambda v: v[:-1])
il = TestItemLoader()
- il.add_value('name', [u'marta', u'other'])
- self.assertEqual(il.get_output_value('name'), u'Mart')
+ il.add_value('name', ['marta', 'other'])
+ self.assertEqual(il.get_output_value('name'), 'Mart')
item = il.load_item()
- self.assertEqual(item['name'], u'Mart')
+ self.assertEqual(item['name'], 'Mart')
def test_partial_processor(self):
def join(values, sep=None, loader_context=None, ignored=None):
@@ -402,13 +402,13 @@ class BasicItemLoaderTest(unittest.TestCase):
summary_out = Compose(partial(join, ignored='foo'))
il = TestItemLoader()
- il.add_value('name', [u'rabbit', u'hole'])
- il.add_value('url', [u'rabbit', u'hole'])
- il.add_value('summary', [u'rabbit', u'hole'])
+ il.add_value('name', ['rabbit', 'hole'])
+ il.add_value('url', ['rabbit', 'hole'])
+ il.add_value('summary', ['rabbit', 'hole'])
item = il.load_item()
- self.assertEqual(item['name'], u'rabbit+hole')
- self.assertEqual(item['url'], u'rabbit.hole')
- self.assertEqual(item['summary'], u'rabbithole')
+ self.assertEqual(item['name'], 'rabbit+hole')
+ self.assertEqual(item['url'], 'rabbit.hole')
+ self.assertEqual(item['summary'], 'rabbithole')
def test_error_input_processor(self):
class TestItem(Item):
@@ -420,7 +420,7 @@ class BasicItemLoaderTest(unittest.TestCase):
il = TestItemLoader()
self.assertRaises(ValueError, il.add_value, 'name',
- [u'marta', u'other'])
+ ['marta', 'other'])
def test_error_output_processor(self):
class TestItem(Item):
@@ -431,7 +431,7 @@ class BasicItemLoaderTest(unittest.TestCase):
name_out = Compose(Join(), float)
il = TestItemLoader()
- il.add_value('name', u'marta')
+ il.add_value('name', 'marta')
with self.assertRaises(ValueError):
il.load_item()
@@ -444,7 +444,7 @@ class BasicItemLoaderTest(unittest.TestCase):
il = TestItemLoader()
self.assertRaises(ValueError, il.add_value, 'name',
- [u'marta', u'other'], Compose(float))
+ ['marta', 'other'], Compose(float))
class InitializationFromDictTest(unittest.TestCase):
@@ -608,8 +608,8 @@ class ProcessorsTest(unittest.TestCase):
def test_join(self):
proc = Join()
self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
- self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
- self.assertEqual(proc(['hello', 'world']), u'hello world')
+ self.assertEqual(proc(['', 'hello', 'world']), ' hello world')
+ self.assertEqual(proc(['hello', 'world']), 'hello world')
self.assertIsInstance(proc(['hello', 'world']), str)
def test_compose(self):
@@ -626,8 +626,8 @@ class ProcessorsTest(unittest.TestCase):
def filter_world(x):
return None if x == 'world' else x
proc = MapCompose(filter_world, str.upper)
- self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
- [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+ self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
+ ['HELLO', 'THIS', 'IS', 'SCRAPY'])
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
proc = MapCompose(filter_world, str.upper)
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 7064337ad..b771e7d79 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -56,13 +56,13 @@ class LogFormatterTestCase(unittest.TestCase):
def test_dropped(self):
item = {}
- exception = Exception(u"\u2018")
+ exception = Exception("\u2018")
response = Response("http://www.example.com")
logkws = self.formatter.dropped(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
assert all(isinstance(x, str) for x in lines)
- self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
+ self.assertEqual(lines, ["Dropped: \u2018", '{}'])
def test_item_error(self):
# In practice, the complete traceback is shown by passing the
@@ -72,7 +72,7 @@ class LogFormatterTestCase(unittest.TestCase):
response = Response("http://www.example.com")
logkws = self.formatter.item_error(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
- self.assertEqual(logline, u"Error processing {'key': 'value'}")
+ self.assertEqual(logline, "Error processing {'key': 'value'}")
def test_spider_error(self):
# In practice, the complete traceback is shown by passing the
@@ -107,20 +107,20 @@ class LogFormatterTestCase(unittest.TestCase):
def test_scraped(self):
item = CustomItem()
- item['name'] = u'\xa3'
+ item['name'] = '\xa3'
response = Response("http://www.example.com")
logkws = self.formatter.scraped(item, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
assert all(isinstance(x, str) for x in lines)
- self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
+ self.assertEqual(lines, ["Scraped from <200 http://www.example.com>", 'name: \xa3'])
class LogFormatterSubclass(LogFormatter):
def crawled(self, request, response, spider):
kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
CRAWLEDMSG = (
- u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
+ "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
)
log_args = kwargs['args']
log_args['flags'] = str(request.flags)
diff --git a/tests/test_mail.py b/tests/test_mail.py
index 53dbc0686..9b248fbfa 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -73,8 +73,8 @@ class MailSenderTest(unittest.TestCase):
self.catched_msg = dict(**kwargs)
def test_send_utf8(self):
- subject = u'sübjèçt'
- body = u'bödÿ-àéïöñß'
+ subject = 'sübjèçt'
+ body = 'bödÿ-àéïöñß'
mailsender = MailSender(debug=True)
mailsender.send(to=['test@scrapy.org'], subject=subject, body=body,
charset='utf-8', _callback=self._catch_mail_sent)
@@ -90,8 +90,8 @@ class MailSenderTest(unittest.TestCase):
self.assertEqual(msg.get('Content-Type'), 'text/plain; charset="utf-8"')
def test_send_attach_utf8(self):
- subject = u'sübjèçt'
- body = u'bödÿ-àéïöñß'
+ subject = 'sübjèçt'
+ body = 'bödÿ-àéïöñß'
attach = BytesIO()
attach.write(body.encode('utf-8'))
attach.seek(0)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index dd19a69d5..a175f88ca 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -23,11 +23,11 @@ class ResponseTypesTest(unittest.TestCase):
mappings = [
(b'attachment; filename="data.xml"', XmlResponse),
(b'attachment; filename=data.xml', XmlResponse),
- (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
- (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
- (u'attachment;filename=data高.doc'.encode('gbk'), Response),
- (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
- (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+ ('attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+ ('attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+ ('attachment;filename=data高.doc'.encode('gbk'), Response),
+ ('attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+ ('attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
]
for source, cls in mappings:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 24aaaf7ec..9d8c201dd 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -93,7 +93,7 @@ class BaseRobotParserTest:
self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
def test_unicode_url_and_useragent(self):
- robotstxt_robotstxt_body = u"""
+ robotstxt_robotstxt_body = """
User-Agent: *
Disallow: /admin/
Disallow: /static/
@@ -107,11 +107,11 @@ class BaseRobotParserTest:
self.assertTrue(rp.allowed("https://site.local/", "*"))
self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
self.assertFalse(rp.allowed("https://site.local/static/", "*"))
- self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+ self.assertTrue(rp.allowed("https://site.local/admin/", "UnicödeBöt"))
self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
- self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+ self.assertFalse(rp.allowed("https://site.local/wiki/Käyttäjä:", "*"))
self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
- self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+ self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt"))
class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
diff --git a/tests/test_selector.py b/tests/test_selector.py
index 00e663c11..62036ad8c 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -25,19 +25,19 @@ class SelectorTestCase(unittest.TestCase):
)
self.assertEqual(
[x.get() for x in sel.xpath("//input[@name='a']/@name")],
- [u'a']
+ ['a']
)
self.assertEqual(
[x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
- [u'12.0']
+ ['12.0']
)
self.assertEqual(
sel.xpath("concat('xpath', 'rules')").getall(),
- [u'xpathrules']
+ ['xpathrules']
)
self.assertEqual(
[x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
- [u'12']
+ ['12']
)
def test_root_base_url(self):
@@ -52,30 +52,30 @@ class SelectorTestCase(unittest.TestCase):
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'xml')
self.assertEqual(sel.xpath("//div").getall(),
- [u'
Hello
'])
+ ['
Hello
'])
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
self.assertEqual(sel.type, 'html')
self.assertEqual(sel.xpath("//div").getall(),
- [u'
Hello
'])
+ ['
Hello
'])
def test_http_header_encoding_precedence(self):
- # u'\xa3' = pound symbol in unicode
- # u'\xc2\xa3' = pound symbol in utf-8
- # u'\xa3' = pound symbol in latin-1 (iso-8859-1)
+ # '\xa3' = pound symbol in unicode
+ # '\xc2\xa3' = pound symbol in utf-8
+ # '\xa3' = pound symbol in latin-1 (iso-8859-1)
- meta = u''
- head = u'' + meta + u''
- body_content = u'\xa3'
- body = u'' + body_content + u''
- html = u'' + head + body + u''
+ meta = ''
+ head = '' + meta + ''
+ body_content = '\xa3'
+ body = '' + body_content + ''
+ html = '' + head + body + ''
encoding = 'utf-8'
html_utf8 = html.encode(encoding)
headers = {'Content-Type': ['text/html; charset=utf-8']}
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
x = Selector(response)
- self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
+ self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), ['\xa3'])
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence
@@ -92,4 +92,4 @@ class SelectorTestCase(unittest.TestCase):
def test_selector_bad_args(self):
with self.assertRaisesRegex(ValueError, 'received both response and text'):
- Selector(TextResponse(url='http://example.com', body=b''), text=u'')
+ Selector(TextResponse(url='http://example.com', body=b''), text='')
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83c10a3c3..78157a9b9 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -153,13 +153,13 @@ class XMLFeedSpiderTest(SpiderTest):
output = list(spider._parse(response))
self.assertEqual(len(output), 2, iterator)
self.assertEqual(output, [
- {'loc': [u'http://www.example.com/Special-Offers.html'],
- 'updated': [u'2009-08-16'],
- 'custom': [u'fuu'],
- 'other': [u'bar']},
+ {'loc': ['http://www.example.com/Special-Offers.html'],
+ 'updated': ['2009-08-16'],
+ 'custom': ['fuu'],
+ 'other': ['bar']},
{'loc': [],
- 'updated': [u'2009-08-16'],
- 'other': [u'foo'],
+ 'updated': ['2009-08-16'],
+ 'other': ['foo'],
'custom': []},
], iterator)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index d17bb2cbc..298178f08 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -54,7 +54,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_unicode(self):
# example taken from https://github.com/scrapy/scrapy/issues/1665
- body = u"""
+ body = """
<þingflokkar>
<þingflokkur id="26">
@@ -97,15 +97,15 @@ class XmliterTestCase(unittest.TestCase):
XmlResponse(url="http://example.com", body=body, encoding='utf-8'),
):
attrs = []
- for x in self.xmliter(r, u'þingflokkur'):
+ for x in self.xmliter(r, 'þingflokkur'):
attrs.append((x.attrib['id'],
- x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
- x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
+ x.xpath('./skammstafanir/stuttskammstöfun/text()').getall(),
+ x.xpath('./tímabil/fyrstaþing/text()').getall()))
self.assertEqual(attrs,
- [(u'26', [u'-'], [u'80']),
- (u'21', [u'Ab'], [u'76']),
- (u'27', [u'A'], [u'27'])])
+ [('26', ['-'], ['80']),
+ ('21', ['Ab'], ['76']),
+ ('27', ['A'], ['27'])])
def test_xmliter_text(self):
body = (
@@ -114,7 +114,7 @@ class XmliterTestCase(unittest.TestCase):
)
self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
- [[u'one'], [u'two']])
+ [['one'], ['two']])
def test_xmliter_namespaces(self):
body = b"""
@@ -179,7 +179,7 @@ class XmliterTestCase(unittest.TestCase):
response = XmlResponse('http://www.example.com', body=body)
self.assertEqual(
next(self.xmliter(response, 'item')).get(),
- u'- Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6
'
+ '- Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6
'
)
@@ -265,10 +265,10 @@ class UtilsCsvTestCase(unittest.TestCase):
result = [row for row in csv]
self.assertEqual(result,
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+ {'id': '4', 'name': 'empty', 'value': ''}])
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
@@ -281,10 +281,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter='\t')
self.assertEqual([row for row in csv],
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+ {'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_quotechar(self):
body1 = get_testdata('feeds', 'feed-sample6.csv')
@@ -294,19 +294,19 @@ class UtilsCsvTestCase(unittest.TestCase):
csv1 = csviter(response1, quotechar="'")
self.assertEqual([row for row in csv1],
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+ {'id': '4', 'name': 'empty', 'value': ''}])
response2 = TextResponse(url="http://example.com/", body=body2)
csv2 = csviter(response2, delimiter="|", quotechar="'")
self.assertEqual([row for row in csv2],
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+ {'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_wrong_quotechar(self):
body = get_testdata('feeds', 'feed-sample6.csv')
@@ -314,10 +314,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual([row for row in csv],
- [{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
- {u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
- {u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
- {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
+ [{"'id'": "1", "'name'": "'alpha'", "'value'": "'foobar'"},
+ {"'id'": "2", "'name'": "'unicode'", "'value'": "'\xfan\xedc\xf3d\xe9\u203d'"},
+ {"'id'": "'3'", "'name'": "'multi'", "'value'": "'foo"},
+ {"'id'": "4", "'name'": "'empty'", "'value'": ""}])
def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@@ -325,10 +325,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter='\t')
self.assertEqual([row for row in csv],
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+ {'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_headers(self):
sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@@ -338,10 +338,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
self.assertEqual([row for row in csv],
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': 'foo\nbar'},
+ {'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_falserow(self):
body = get_testdata('feeds', 'feed-sample3.csv')
@@ -351,10 +351,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual([row for row in csv],
- [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
- {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
- {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
- {u'id': u'4', u'name': u'empty', u'value': u''}])
+ [{'id': '1', 'name': 'alpha', 'value': 'foobar'},
+ {'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
+ {'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
+ {'id': '4', 'name': 'empty', 'value': ''}])
def test_csviter_exception(self):
body = get_testdata('feeds', 'feed-sample3.csv')
@@ -377,8 +377,8 @@ class UtilsCsvTestCase(unittest.TestCase):
self.assertEqual(
list(csv),
[
- {u'id': u'1', u'name': u'latin1', u'value': u'test'},
- {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+ {'id': '1', 'name': 'latin1', 'value': 'test'},
+ {'id': '2', 'name': 'something', 'value': '\xf1\xe1\xe9\xf3'},
]
)
@@ -387,8 +387,8 @@ class UtilsCsvTestCase(unittest.TestCase):
self.assertEqual(
list(csv),
[
- {u'id': u'1', u'name': u'cp852', u'value': u'test'},
- {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+ {'id': '1', 'name': 'cp852', 'value': 'test'},
+ {'id': '2', 'name': 'something', 'value': '\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
]
)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index ebce3c079..3f93f509e 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -34,13 +34,13 @@ class MutableChainTest(unittest.TestCase):
class ToUnicodeTest(unittest.TestCase):
def test_converting_an_utf8_encoded_string_to_unicode(self):
- self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
+ self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
def test_converting_a_latin_1_encoded_string_to_unicode(self):
- self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
+ self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), 'lel\xf1e')
def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
- self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
+ self.assertEqual(to_unicode('\xf1e\xf1e\xf1e'), '\xf1e\xf1e\xf1e')
def test_converting_a_strange_object_should_raise_TypeError(self):
self.assertRaises(TypeError, to_unicode, 423)
@@ -48,16 +48,16 @@ class ToUnicodeTest(unittest.TestCase):
def test_errors_argument(self):
self.assertEqual(
to_unicode(b'a\xedb', 'utf-8', errors='replace'),
- u'a\ufffdb'
+ 'a\ufffdb'
)
class ToBytesTest(unittest.TestCase):
def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
- self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
+ self.assertEqual(to_bytes('\xa3 49'), b'\xc2\xa3 49')
def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
- self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
+ self.assertEqual(to_bytes('\xa3 49', 'latin-1'), b'\xa3 49')
def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
@@ -67,7 +67,7 @@ class ToBytesTest(unittest.TestCase):
def test_errors_argument(self):
self.assertEqual(
- to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
+ to_bytes('a\ufffdb', 'latin-1', errors='replace'),
b'a?b'
)
@@ -96,7 +96,7 @@ class BinaryIsTextTest(unittest.TestCase):
assert binary_is_text(b"hello")
def test_utf_16_strings_contain_null_bytes(self):
- assert binary_is_text(u"hello".encode('utf-16'))
+ assert binary_is_text("hello".encode('utf-16'))
def test_one_with_encoding(self):
assert binary_is_text(b"Price \xa3
")
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 450e4bdca..de94ec960 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -22,7 +22,7 @@ class RequestSerializationTest(unittest.TestCase):
method="POST",
body=b"some body",
headers={'content-encoding': 'text/html; charset=latin-1'},
- cookies={'currency': u'руб'},
+ cookies={'currency': 'руб'},
encoding='latin-1',
priority=20,
meta={'a': 'b'},
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 5a52dd695..5ff2e41ef 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -19,8 +19,8 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
def test_simple_render(self):
context = dict(project_name='proj', name='spi', classname='TheSpider')
- template = u'from ${project_name}.spiders.${name} import ${classname}'
- rendered = u'from proj.spiders.spi import TheSpider'
+ template = 'from ${project_name}.spiders.${name} import ${classname}'
+ rendered = 'from proj.spiders.spi import TheSpider'
template_path = os.path.join(self.tmp_path, 'templ.py.tmpl')
render_path = os.path.join(self.tmp_path, 'templ.py')