diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 4aeca6903..db2c9db8e 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -25,8 +25,6 @@ class RegexLinkExtractor(SgmlLinkExtractor): clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip() links_text = linkre.findall(response_text) - urlstext = set([(clean_url(url).encode(response_encoding), clean_text(text)) - for url, _, text in links_text]) - - - return [Link(url, text) for url, text in urlstext] + return [Link(clean_url(url).encode(response_encoding), + clean_text(text)) + for url, _, text in links_text] diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index d6b832141..9ced21c80 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,5 +1,7 @@ import unittest, json, cPickle as pickle from cStringIO import StringIO +import lxml.etree +import re from scrapy.item import Item, Field from scrapy.utils.python import str_to_unicode @@ -139,8 +141,13 @@ class CsvItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return CsvItemExporter(self.output, **kwargs) + def assertCsvEqual(self, first, second, msg=None): + csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line)) + for line in csv.splitlines(True)] + return self.assertEqual(csvsplit(first), csvsplit(second), msg) + def _check_output(self): - self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') def test_header(self): output = StringIO() @@ -148,14 +155,14 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() - self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') output = StringIO() ie = CsvItemExporter(output, fields_to_export=['age']) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() - self.assertEqual(output.getvalue(), 'age\r\n22\r\n') + self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n') output = StringIO() ie = CsvItemExporter(output) @@ -163,14 +170,14 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.export_item(self.i) ie.export_item(self.i) ie.finish_exporting() - self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') output = StringIO() ie = CsvItemExporter(output, include_headers_line=False) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() - self.assertEqual(output.getvalue(), '22,John\xc2\xa3\r\n') + self.assertCsvEqual(output.getvalue(), '22,John\xc2\xa3\r\n') def test_join_multivalue(self): class TestItem2(Item): @@ -183,16 +190,29 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.start_exporting() ie.export_item(i) ie.finish_exporting() - self.assertEqual(output.getvalue(), '"Mary,Paul",John\r\n') + self.assertCsvEqual(output.getvalue(), '"Mary,Paul",John\r\n') class XmlItemExporterTest(BaseItemExporterTest): def _get_exporter(self, **kwargs): return XmlItemExporter(self.output, **kwargs) + def assertXmlEquivalent(self, first, second, msg=None): + def xmltuple(elem): + children = list(elem.iterchildren()) + if children: + return [(child.tag, sorted(xmltuple(child))) + for child in children] + else: + return [(elem.tag, [(elem.text, ())])] + def xmlsplit(xmlcontent): + doc = lxml.etree.fromstring(xmlcontent) + return xmltuple(doc) + return self.assertEqual(xmlsplit(first), xmlsplit(second), msg) + def _check_output(self): expected_value = '\n22John\xc2\xa3' - self.assertEqual(self.output.getvalue(), expected_value) + self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): output = StringIO() @@ -202,7 +222,7 @@ class XmlItemExporterTest(BaseItemExporterTest): ie.export_item(item) ie.finish_exporting() expected_value = '\nJohn\xc2\xa3Doe' - self.assertEqual(output.getvalue(), expected_value) + self.assertXmlEquivalent(output.getvalue(), expected_value) def test_nested_item(self): output = StringIO() @@ -224,7 +244,7 @@ class XmlItemExporterTest(BaseItemExporterTest): ''\ 'buz'\ '' - self.assertEqual(output.getvalue(), expected_value) + self.assertXmlEquivalent(output.getvalue(), expected_value) def test_nested_list_item(self): output = StringIO() @@ -243,7 +263,8 @@ class XmlItemExporterTest(BaseItemExporterTest): ''\ 'buz'\ '' - self.assertEqual(output.getvalue(), expected_value) + self.assertXmlEquivalent(output.getvalue(), expected_value) + class JsonLinesItemExporterTest(BaseItemExporterTest): diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 19d122a89..9e201e017 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -312,12 +312,11 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): def test_extraction(self): # Default arguments lx = HtmlParserLinkExtractor() - self.assertEqual([link for link in lx.extract_links(self.response)], [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text='sample 3 repetition'), - Link(url='http://www.google.com/something', text=''), - ]) + self.assertEqual(lx.extract_links(self.response), + [Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://www.google.com/something', text=u''),]) class RegexLinkExtractorTestCase(unittest.TestCase): @@ -329,13 +328,10 @@ class RegexLinkExtractorTestCase(unittest.TestCase): def test_extraction(self): # Default arguments lx = RegexLinkExtractor() - # Note that RegexLinkExtractor returns links in arbitrary order, - # so we need to sort them for comparison - self.assertEqual(sorted(lx.extract_links(self.response), key=lambda x: x.url), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://www.google.com/something', text=u''), - ]) + self.assertEqual(lx.extract_links(self.response), + [Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://www.google.com/something', text=u''),]) if __name__ == "__main__": diff --git a/scrapy/tests/test_djangoitem/__init__.py b/scrapy/tests/test_djangoitem/__init__.py index cff97fc98..d50428d0b 100644 --- a/scrapy/tests/test_djangoitem/__init__.py +++ b/scrapy/tests/test_djangoitem/__init__.py @@ -24,21 +24,24 @@ if 'django' in optional_features: class DjangoItemTest(unittest.TestCase): + def assertSortedEqual(self, first, second, msg=None): + return self.assertEqual(sorted(first), sorted(second), msg) + def setUp(self): if 'django' not in optional_features: raise unittest.SkipTest("Django is not available") def test_base(self): i = BasePersonItem() - self.assertEqual(i.fields.keys(), ['age', 'name']) + self.assertSortedEqual(i.fields.keys(), ['age', 'name']) def test_new_fields(self): i = NewFieldPersonItem() - self.assertEqual(i.fields.keys(), ['age', 'other', 'name']) + self.assertSortedEqual(i.fields.keys(), ['age', 'other', 'name']) def test_override_field(self): i = OverrideFieldPersonItem() - self.assertEqual(i.fields.keys(), ['age', 'name']) + self.assertSortedEqual(i.fields.keys(), ['age', 'name']) def test_custom_primary_key_field(self): """ @@ -46,11 +49,11 @@ class DjangoItemTest(unittest.TestCase): in the field list. """ i = IdentifiedPersonItem() - self.assertEqual(i.fields.keys(), ['age', 'identifier', 'name']) + self.assertSortedEqual(i.fields.keys(), ['age', 'identifier', 'name']) def test_save(self): i = BasePersonItem() - self.assertEqual(i.fields.keys(), ['age', 'name']) + self.assertSortedEqual(i.fields.keys(), ['age', 'name']) i['name'] = 'John' i['age'] = '22' diff --git a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py index acbfb5268..3e5ce6052 100644 --- a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,7 +1,7 @@ import unittest from scrapy.contrib.downloadermiddleware.ajaxcrawl import AjaxCrawlMiddleware -from scrapy.spider import BaseSpider +from scrapy.spider import Spider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler @@ -9,7 +9,7 @@ __doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl'] class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): - self.spider = BaseSpider('foo') + self.spider = Spider('foo') crawler = get_crawler({'AJAXCRAWL_ENABLED': True}) self.mw = AjaxCrawlMiddleware.from_crawler(crawler) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index 20fcc656d..35a86b8ce 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -1,4 +1,5 @@ from unittest import TestCase +import re from scrapy.http import Response, Request from scrapy.spider import Spider @@ -7,6 +8,12 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): + def assertCookieValEqual(self, first, second, msg=None): + cookievaleq = lambda cv: re.split(';\s*', cv) + return self.assertEqual( + sorted(cookievaleq(first)), + sorted(cookievaleq(second)), msg) + def setUp(self): self.spider = Spider('foo') self.mw = CookiesMiddleware() @@ -86,7 +93,8 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') assert self.mw.process_request(req2, self.spider) is None - self.assertEquals(req2.headers.get('Cookie'), "C1=value1; galleta=salada") + + self.assertCookieValEqual(req2.headers.get('Cookie'), "C1=value1; galleta=salada") def test_cookiejar_key(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"}) @@ -99,8 +107,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/', meta=res.meta) assert self.mw.process_request(req2, self.spider) is None - self.assertEquals(req2.headers.get('Cookie'), 'C1=value1; galleta=salada') - + self.assertCookieValEqual(req2.headers.get('Cookie'),'C1=value1; galleta=salada') req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"}) assert self.mw.process_request(req3, self.spider) is None @@ -112,7 +119,7 @@ class CookiesMiddlewareTest(TestCase): req4 = Request('http://scrapytest.org/', meta=res2.meta) assert self.mw.process_request(req4, self.spider) is None - self.assertEquals(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce') + self.assertCookieValEqual(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce') #cookies from hosts with port req5_1 = Request('http://scrapytest.org:1104/') diff --git a/scrapy/tests/test_http_headers.py b/scrapy/tests/test_http_headers.py index 77449bfd6..f68330083 100644 --- a/scrapy/tests/test_http_headers.py +++ b/scrapy/tests/test_http_headers.py @@ -4,6 +4,10 @@ import copy from scrapy.http import Headers class HeadersTest(unittest.TestCase): + + def assertSortedEqual(self, first, second, msg=None): + return self.assertEqual(sorted(first), sorted(second), msg) + def test_basics(self): h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234}) assert h['Content-Type'] @@ -75,13 +79,13 @@ class HeadersTest(unittest.TestCase): idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']} h = Headers(idict) - self.assertEqual(dict(h), {'Content-Type': ['text/html'], 'X-Forwarded-For': ['ip1', 'ip2']}) - self.assertEqual(h.keys(), ['X-Forwarded-For', 'Content-Type']) - self.assertEqual(h.items(), [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])]) - self.assertEqual(list(h.iteritems()), - [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])]) + self.assertDictEqual(dict(h), + {'Content-Type': ['text/html'], 'X-Forwarded-For': ['ip1', 'ip2']}) + self.assertSortedEqual(h.keys(), ['X-Forwarded-For', 'Content-Type']) + self.assertSortedEqual(h.items(), [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])]) + self.assertSortedEqual(h.iteritems(), [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])]) - self.assertEqual(h.values(), ['ip2', 'text/html']) + self.assertSortedEqual(h.values(), ['ip2', 'text/html']) def test_update(self): h = Headers() diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 069b1cd04..81b871451 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -184,6 +184,9 @@ class FormRequestTest(RequestTest): request_class = FormRequest + def assertSortedEqual(self, first, second, msg=None): + return self.assertEqual(sorted(first), sorted(second), msg) + def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) self.assertEqual(r1.body, '') @@ -194,7 +197,8 @@ class FormRequestTest(RequestTest): r2 = self.request_class("http://www.example.com", formdata=data) self.assertEqual(r2.method, 'POST') self.assertEqual(r2.encoding, 'utf-8') - self.assertEqual(r2.body, 'price=%C2%A3+100&one=two') + self.assertSortedEqual(r2.body.split('&'), + 'price=%C2%A3+100&one=two'.split('&')) self.assertEqual(r2.headers['Content-Type'], 'application/x-www-form-urlencoded') def test_custom_encoding(self): @@ -207,7 +211,8 @@ class FormRequestTest(RequestTest): # using multiples values for a single key data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertEqual(r3.body, 'colours=red&colours=blue&colours=green&price=%C2%A3+100') + self.assertSortedEqual(r3.body.split('&'), + 'colours=red&colours=blue&colours=green&price=%C2%A3+100'.split('&')) def test_from_response_post(self): response = _buildresponse( diff --git a/scrapy/tests/test_item.py b/scrapy/tests/test_item.py index 764c94d6e..2a24fddca 100644 --- a/scrapy/tests/test_item.py +++ b/scrapy/tests/test_item.py @@ -5,6 +5,9 @@ from scrapy.item import Item, Field class ItemTest(unittest.TestCase): + def assertSortedEqual(self, first, second, msg=None): + return self.assertEqual(sorted(first), sorted(second), msg) + def test_simple(self): class TestItem(Item): name = Field() @@ -108,8 +111,8 @@ class ItemTest(unittest.TestCase): i['keys'] = u'Keys' i['values'] = u'Values' - self.assertEqual(i.keys(), ['keys', 'values', 'name']) - self.assertEqual(i.values(), [u'Keys', u'Values', u'John']) + self.assertSortedEqual(i.keys(), ['keys', 'values', 'name']) + self.assertSortedEqual(i.values(), [u'Keys', u'Values', u'John']) def test_metaclass_inheritance(self): class BaseItem(Item): diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py index 9de09f343..4e01f121b 100644 --- a/scrapy/tests/test_webclient.py +++ b/scrapy/tests/test_webclient.py @@ -169,7 +169,9 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.makeConnection(transport) - self.assertEqual(transport.value(), testvalue) + self.assertEqual( + set(transport.value().splitlines()), + set(testvalue.splitlines())) return testvalue def test_non_standard_line_endings(self):