RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #570 from redapple/travistests 

[MRG] Fix tests for Travis-CI build
This commit is contained in:
Daniel Graña 2014-01-31 09:16:54 -08:00
parent 05158803df cc157f182b
commit 1e553ec0cd
10 changed files with 89 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/contrib/linkextractors/regex.py
View File

@ -25,8 +25,6 @@ class RegexLinkExtractor(SgmlLinkExtractor):
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()
links_text = linkre.findall(response_text)
urlstext = set([(clean_url(url).encode(response_encoding), clean_text(text))
for url, _, text in links_text])
return [Link(url, text) for url, text in urlstext]
return [Link(clean_url(url).encode(response_encoding),
clean_text(text))
for url, _, text in links_text]

41
scrapy/tests/test_contrib_exporter.py
View File

@ -1,5 +1,7 @@
import unittest, json, cPickle as pickle
from cStringIO import StringIO
import lxml.etree
import re
from scrapy.item import Item, Field
from scrapy.utils.python import str_to_unicode
@ -139,8 +141,13 @@ class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
for line in csv.splitlines(True)]
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
def _check_output(self):
self.assertEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
def test_header(self):
output = StringIO()
@ -148,14 +155,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
ie.start_exporting()
ie.export_item(self.i)
ie.finish_exporting()
self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
output = StringIO()
ie = CsvItemExporter(output, fields_to_export=['age'])
ie.start_exporting()
ie.export_item(self.i)
ie.finish_exporting()
self.assertEqual(output.getvalue(), 'age\r\n22\r\n')
self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n')
output = StringIO()
ie = CsvItemExporter(output)
@ -163,14 +170,14 @@ class CsvItemExporterTest(BaseItemExporterTest):
ie.export_item(self.i)
ie.export_item(self.i)
ie.finish_exporting()
self.assertEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
output = StringIO()
ie = CsvItemExporter(output, include_headers_line=False)
ie.start_exporting()
ie.export_item(self.i)
ie.finish_exporting()
self.assertEqual(output.getvalue(), '22,John\xc2\xa3\r\n')
self.assertCsvEqual(output.getvalue(), '22,John\xc2\xa3\r\n')
def test_join_multivalue(self):
class TestItem2(Item):
@ -183,16 +190,29 @@ class CsvItemExporterTest(BaseItemExporterTest):
ie.start_exporting()
ie.export_item(i)
ie.finish_exporting()
self.assertEqual(output.getvalue(), '"Mary,Paul",John\r\n')
self.assertCsvEqual(output.getvalue(), '"Mary,Paul",John\r\n')
class XmlItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
return XmlItemExporter(self.output, **kwargs)
def assertXmlEquivalent(self, first, second, msg=None):
def xmltuple(elem):
children = list(elem.iterchildren())
if children:
return [(child.tag, sorted(xmltuple(child)))
for child in children]
else:
return [(elem.tag, [(elem.text, ())])]
def xmlsplit(xmlcontent):
doc = lxml.etree.fromstring(xmlcontent)
return xmltuple(doc)
return self.assertEqual(xmlsplit(first), xmlsplit(second), msg)
def _check_output(self):
expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
self.assertEqual(self.output.getvalue(), expected_value)
self.assertXmlEquivalent(self.output.getvalue(), expected_value)
def test_multivalued_fields(self):
output = StringIO()
@ -202,7 +222,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
ie.export_item(item)
ie.finish_exporting()
expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
self.assertEqual(output.getvalue(), expected_value)
self.assertXmlEquivalent(output.getvalue(), expected_value)
def test_nested_item(self):
output = StringIO()
@ -224,7 +244,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
'</age>'\
'<name>buz</name>'\
'</item></items>'
self.assertEqual(output.getvalue(), expected_value)
self.assertXmlEquivalent(output.getvalue(), expected_value)
def test_nested_list_item(self):
output = StringIO()
@ -243,7 +263,8 @@ class XmlItemExporterTest(BaseItemExporterTest):
'</age>'\
'<name>buz</name>'\
'</item></items>'
self.assertEqual(output.getvalue(), expected_value)
self.assertXmlEquivalent(output.getvalue(), expected_value)
class JsonLinesItemExporterTest(BaseItemExporterTest):

22
scrapy/tests/test_contrib_linkextractors.py
View File

@ -312,12 +312,11 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
def test_extraction(self):
# Default arguments
lx = HtmlParserLinkExtractor()
self.assertEqual([link for link in lx.extract_links(self.response)], [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://www.google.com/something', text=''),
])
self.assertEqual(lx.extract_links(self.response),
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),])
class RegexLinkExtractorTestCase(unittest.TestCase):
@ -329,13 +328,10 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
def test_extraction(self):
# Default arguments
lx = RegexLinkExtractor()
# Note that RegexLinkExtractor returns links in arbitrary order,
# so we need to sort them for comparison
self.assertEqual(sorted(lx.extract_links(self.response), key=lambda x: x.url), [
Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),
])
self.assertEqual(lx.extract_links(self.response),
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),])
if __name__ == "__main__":

13
scrapy/tests/test_djangoitem/__init__.py
View File

@ -24,21 +24,24 @@ if 'django' in optional_features:
class DjangoItemTest(unittest.TestCase):
def assertSortedEqual(self, first, second, msg=None):
return self.assertEqual(sorted(first), sorted(second), msg)
def setUp(self):
if 'django' not in optional_features:
raise unittest.SkipTest("Django is not available")
def test_base(self):
i = BasePersonItem()
self.assertEqual(i.fields.keys(), ['age', 'name'])
self.assertSortedEqual(i.fields.keys(), ['age', 'name'])
def test_new_fields(self):
i = NewFieldPersonItem()
self.assertEqual(i.fields.keys(), ['age', 'other', 'name'])
self.assertSortedEqual(i.fields.keys(), ['age', 'other', 'name'])
def test_override_field(self):
i = OverrideFieldPersonItem()
self.assertEqual(i.fields.keys(), ['age', 'name'])
self.assertSortedEqual(i.fields.keys(), ['age', 'name'])
def test_custom_primary_key_field(self):
"""
@ -46,11 +49,11 @@ class DjangoItemTest(unittest.TestCase):
in the field list.
"""
i = IdentifiedPersonItem()
self.assertEqual(i.fields.keys(), ['age', 'identifier', 'name'])
self.assertSortedEqual(i.fields.keys(), ['age', 'identifier', 'name'])
def test_save(self):
i = BasePersonItem()
self.assertEqual(i.fields.keys(), ['age', 'name'])
self.assertSortedEqual(i.fields.keys(), ['age', 'name'])
i['name'] = 'John'
i['age'] = '22'

4
scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py
View File

@ -1,7 +1,7 @@
import unittest
from scrapy.contrib.downloadermiddleware.ajaxcrawl import AjaxCrawlMiddleware
from scrapy.spider import BaseSpider
from scrapy.spider import Spider
from scrapy.http import Request, HtmlResponse, Response
from scrapy.utils.test import get_crawler
@ -9,7 +9,7 @@ __doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl']
class AjaxCrawlMiddlewareTest(unittest.TestCase):
def setUp(self):
self.spider = BaseSpider('foo')
self.spider = Spider('foo')
crawler = get_crawler({'AJAXCRAWL_ENABLED': True})
self.mw = AjaxCrawlMiddleware.from_crawler(crawler)

15
scrapy/tests/test_downloadermiddleware_cookies.py
View File

@ -1,4 +1,5 @@
from unittest import TestCase
import re
from scrapy.http import Response, Request
from scrapy.spider import Spider
@ -7,6 +8,12 @@ from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware
class CookiesMiddlewareTest(TestCase):
def assertCookieValEqual(self, first, second, msg=None):
cookievaleq = lambda cv: re.split(';\s*', cv)
return self.assertEqual(
sorted(cookievaleq(first)),
sorted(cookievaleq(second)), msg)
def setUp(self):
self.spider = Spider('foo')
self.mw = CookiesMiddleware()
@ -86,7 +93,8 @@ class CookiesMiddlewareTest(TestCase):
req2 = Request('http://scrapytest.org/sub1/')
assert self.mw.process_request(req2, self.spider) is None
self.assertEquals(req2.headers.get('Cookie'), "C1=value1; galleta=salada")
self.assertCookieValEqual(req2.headers.get('Cookie'), "C1=value1; galleta=salada")
def test_cookiejar_key(self):
req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
@ -99,8 +107,7 @@ class CookiesMiddlewareTest(TestCase):
req2 = Request('http://scrapytest.org/', meta=res.meta)
assert self.mw.process_request(req2, self.spider) is None
self.assertEquals(req2.headers.get('Cookie'), 'C1=value1; galleta=salada')
self.assertCookieValEqual(req2.headers.get('Cookie'),'C1=value1; galleta=salada')
req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
assert self.mw.process_request(req3, self.spider) is None
@ -112,7 +119,7 @@ class CookiesMiddlewareTest(TestCase):
req4 = Request('http://scrapytest.org/', meta=res2.meta)
assert self.mw.process_request(req4, self.spider) is None
self.assertEquals(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce')
self.assertCookieValEqual(req4.headers.get('Cookie'), 'C2=value2; galleta=dulce')
#cookies from hosts with port
req5_1 = Request('http://scrapytest.org:1104/')

16
scrapy/tests/test_http_headers.py
View File

@ -4,6 +4,10 @@ import copy
from scrapy.http import Headers
class HeadersTest(unittest.TestCase):
def assertSortedEqual(self, first, second, msg=None):
return self.assertEqual(sorted(first), sorted(second), msg)
def test_basics(self):
h = Headers({'Content-Type': 'text/html', 'Content-Length': 1234})
assert h['Content-Type']
@ -75,13 +79,13 @@ class HeadersTest(unittest.TestCase):
idict = {'Content-Type': 'text/html', 'X-Forwarded-For': ['ip1', 'ip2']}
h = Headers(idict)
self.assertEqual(dict(h), {'Content-Type': ['text/html'], 'X-Forwarded-For': ['ip1', 'ip2']})
self.assertEqual(h.keys(), ['X-Forwarded-For', 'Content-Type'])
self.assertEqual(h.items(), [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])])
self.assertEqual(list(h.iteritems()),
[('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])])
self.assertDictEqual(dict(h),
{'Content-Type': ['text/html'], 'X-Forwarded-For': ['ip1', 'ip2']})
self.assertSortedEqual(h.keys(), ['X-Forwarded-For', 'Content-Type'])
self.assertSortedEqual(h.items(), [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])])
self.assertSortedEqual(h.iteritems(), [('X-Forwarded-For', ['ip1', 'ip2']), ('Content-Type', ['text/html'])])
self.assertEqual(h.values(), ['ip2', 'text/html'])
self.assertSortedEqual(h.values(), ['ip2', 'text/html'])
def test_update(self):
h = Headers()

9
scrapy/tests/test_http_request.py
View File

@ -184,6 +184,9 @@ class FormRequestTest(RequestTest):
request_class = FormRequest
def assertSortedEqual(self, first, second, msg=None):
return self.assertEqual(sorted(first), sorted(second), msg)
def test_empty_formdata(self):
r1 = self.request_class("http://www.example.com", formdata={})
self.assertEqual(r1.body, '')
@ -194,7 +197,8 @@ class FormRequestTest(RequestTest):
r2 = self.request_class("http://www.example.com", formdata=data)
self.assertEqual(r2.method, 'POST')
self.assertEqual(r2.encoding, 'utf-8')
self.assertEqual(r2.body, 'price=%C2%A3+100&one=two')
self.assertSortedEqual(r2.body.split('&'),
'price=%C2%A3+100&one=two'.split('&'))
self.assertEqual(r2.headers['Content-Type'], 'application/x-www-form-urlencoded')
def test_custom_encoding(self):
@ -207,7 +211,8 @@ class FormRequestTest(RequestTest):
# using multiples values for a single key
data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertEqual(r3.body, 'colours=red&colours=blue&colours=green&price=%C2%A3+100')
self.assertSortedEqual(r3.body.split('&'),
'colours=red&colours=blue&colours=green&price=%C2%A3+100'.split('&'))
def test_from_response_post(self):
response = _buildresponse(

7
scrapy/tests/test_item.py
View File

@ -5,6 +5,9 @@ from scrapy.item import Item, Field
class ItemTest(unittest.TestCase):
def assertSortedEqual(self, first, second, msg=None):
return self.assertEqual(sorted(first), sorted(second), msg)
def test_simple(self):
class TestItem(Item):
name = Field()
@ -108,8 +111,8 @@ class ItemTest(unittest.TestCase):
i['keys'] = u'Keys'
i['values'] = u'Values'
self.assertEqual(i.keys(), ['keys', 'values', 'name'])
self.assertEqual(i.values(), [u'Keys', u'Values', u'John'])
self.assertSortedEqual(i.keys(), ['keys', 'values', 'name'])
self.assertSortedEqual(i.values(), [u'Keys', u'Values', u'John'])
def test_metaclass_inheritance(self):
class BaseItem(Item):

4
scrapy/tests/test_webclient.py
View File

@ -169,7 +169,9 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
protocol = client.ScrapyHTTPPageGetter()
protocol.factory = factory
protocol.makeConnection(transport)
self.assertEqual(transport.value(), testvalue)
self.assertEqual(
set(transport.value().splitlines()),
set(testvalue.splitlines()))
return testvalue
def test_non_standard_line_endings(self):