RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automated merge with ssh://hg.scrapy.org:2222/scrapy-0.12

This commit is contained in:
Pablo Hoffman 2011-02-15 07:25:12 -02:00
parent 16d9a33951 9b07b0ab0a
commit 1fb55bdaf0
11 changed files with 123 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/scrapyd.rst
View File

@ -86,7 +86,7 @@ in your Ubuntu servers.
So, if you plan to deploy Scrapyd on a Ubuntu server, just add the Ubuntu
repositories as described in :ref:`topics-ubuntu` and then run::
aptitude install scrapyd-0.12
aptitude install scrapyd-0.13
This will install Scrapyd in your Ubuntu server creating a ``scrapy`` user
which Scrapyd will run as. It will also create some directories and files that

2
docs/topics/ubuntu.rst
View File

@ -13,7 +13,7 @@ latest bug fixes.
To use the packages, just add the following line to your
``/etc/apt/sources.list``, and then run ``aptitude update`` and ``aptitude
install scrapy-0.12``::
install scrapy-0.13``::
deb http://archive.scrapy.org/ubuntu DISTRO main

4
scrapy/__init__.py
View File

@ -2,8 +2,8 @@
Scrapy - a screen scraping framework written in Python
"""
version_info = (0, 12, 0)
__version__ = "0.12.0"
version_info = (0, 13, 0)
__version__ = "0.13.0"
import sys, os, warnings

4
scrapy/contrib/ibl/htmlpage.py
View File

@ -80,8 +80,8 @@ class HtmlTag(HtmlDataFragment):
def __repr__(self):
return str(self)
_ATTR = "((?:[^=/>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?"
_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>"
_ATTR = "((?:[^=/<>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?"
_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s*" + _ATTR + ")+\s*|\s*)(\/?)>?"
_DOCTYPE = r"<!DOCTYPE.*?>"
_SCRIPT = "(<script.*?>)(.*?)(</script.*?>)"
_COMMENT = "(<!--.*?-->)"

11
scrapy/contrib_exp/iterators.py
View File

@ -2,14 +2,19 @@ from scrapy.http import Response
from scrapy.selector import XmlXPathSelector
def xmliter_lxml(obj, nodename):
def xmliter_lxml(obj, nodename, namespace=None):
from lxml import etree
reader = _StreamReader(obj)
iterable = etree.iterparse(reader, tag=nodename, encoding=reader.encoding)
tag = '{%s}%s' % (namespace, nodename) if namespace else nodename
iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
selxpath = '//' + ('x:%s' % nodename if namespace else nodename)
for _, node in iterable:
nodetext = etree.tostring(node)
node.clear()
yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0]
xs = XmlXPathSelector(text=nodetext)
if namespace:
xs.register_namespace('x', namespace)
yield xs.select(selxpath)[0]
class _StreamReader(object):

3
scrapy/link.py
View File

@ -18,6 +18,9 @@ class Link(object):
def __eq__(self, other):
return self.url == other.url and self.text == other.text
def __hash__(self):
return hash(self.url) ^ hash(self.text)
def __repr__(self):
return '<Link url=%r text=%r >' % (self.url, self.text)

2
scrapy/tests/test_contrib_ibl/test_extraction.py
View File

@ -516,7 +516,7 @@ ANNOTATED_PAGE19 = u"""
<div>
<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;name&quot;}}">Product name</p>
<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;content&quot;: &quot;price&quot;}}">60.00</p>
<img data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}"src="image.jpg" />
<img data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;annotations&quot;: {&quot;src&quot;: &quot;image_urls&quot;}}" src="image.jpg" />
<p data-scrapy-annotate="{&quot;variant&quot;: 0, &quot;required&quot;: [&quot;description&quot;], &quot;annotations&quot;: {&quot;content&quot;: &quot;description&quot;}}">description</p>
</div>
</body></html>

16
scrapy/tests/test_contrib_ibl/test_htmlpage.py
View File

@ -137,3 +137,19 @@ class TestParseHtml(TestCase):
parsed = list(parse_html("<IMG SRC='http://images.play.com/banners/SAM550a.jpg' align='left' / hspace=5>"))
self.assertEqual(parsed[0].attributes, {'src': 'http://images.play.com/banners/SAM550a.jpg', \
'align': 'left', 'hspace': '5', '/': None})
def test_no_ending_body(self):
"""Test case when no ending body nor html elements are present"""
parsed = [_decode_element(d) for d in PARSED7]
self._test_sample(PAGE7, parsed)
def test_malformed(self):
"""Test parsing of some malformed cases"""
parsed = [_decode_element(d) for d in PARSED8]
self._test_sample(PAGE8, parsed)
def test_malformed2(self):
"""Test case when attributes are not separated by space (still recognizable because of quotes)"""
parsed = [_decode_element(d) for d in PARSED9]
self._test_sample(PAGE9, parsed)

29
scrapy/tests/test_contrib_ibl/test_htmlpage_data.py
View File

@ -246,3 +246,32 @@ PARSED7 = [
{'end': 99, 'start': 85},
]
PAGE8 = u"""<a href="/overview.asp?id=277"><img border="0" src="/img/5200814311.jpg" title=\'Vinyl Cornice\'</a></td><table width=\'5\'>"""
PARSED8 = [
{'attributes' : {u'href' : u"/overview.asp?id=277"}, 'end': 31, 'start': 0, 'tag': u'a', 'tag_type': 1},
{'attributes' : {u'src' : u"/img/5200814311.jpg", u'border' : u"0", u'title': u'Vinyl Cornice'}, 'end': 94, 'start': 31, 'tag': u'img', 'tag_type': 1},
{'attributes' : {}, 'end': 98, 'start': 94, 'tag': u'a', 'tag_type': 2},
{'attributes' : {}, 'end': 103, 'start': 98, 'tag': u'td', 'tag_type': 2},
{'attributes' : {u'width': u'5'}, 'end': 120, 'start': 103, 'tag': u'table', 'tag_type': 1}
]
PAGE9 = u"""\
<html>\
<body>\
<img width='230' height='150'src='/images/9589.jpg' >\
<a href="/product/9589">Click here</a>\
</body>\
</html>\
"""
PARSED9 = [
{'attributes' : {}, 'end': 6, 'start': 0, 'tag': 'html', 'tag_type': 1},
{'attributes' : {}, 'end': 12, 'start': 6, 'tag': 'body', 'tag_type': 1},
{'attributes' : {'width': '230', 'height': '150', 'src': '/images/9589.jpg'}, 'end': 65, 'start': 12, 'tag': 'img', 'tag_type': 1},
{'attributes' : {'href': '/product/9589'}, 'end': 89, 'start': 65, 'tag': 'a', 'tag_type': 1},
{'end': 99, 'start': 89},
{'attributes' : {}, 'end': 103, 'start': 99, 'tag': 'a', 'tag_type': 2},
{'attributes' : {}, 'end': 110, 'start': 103, 'tag': 'body', 'tag_type': 2},
{'attributes' : {}, 'end': 117, 'start': 110, 'tag': 'html', 'tag_type': 2},
]

28
scrapy/tests/test_link.py Normal file
View File

@ -0,0 +1,28 @@
import unittest
from scrapy.link import Link
class LinkTest(unittest.TestCase):
def test_eq_and_hash(self):
l1 = Link("http://www.example.com")
l2 = Link("http://www.example.com/other")
l3 = Link("http://www.example.com")
self.assertEqual(l1, l1)
self.assertEqual(hash(l1), hash(l1))
self.assertNotEqual(l1, l2)
self.assertNotEqual(hash(l1), hash(l2))
self.assertEqual(l1, l3)
self.assertEqual(hash(l1), hash(l3))
l4 = Link("http://www.example.com", text="test")
l5 = Link("http://www.example.com", text="test2")
l6 = Link("http://www.example.com", text="test")
self.assertEqual(l4, l4)
self.assertEqual(hash(l4), hash(l4))
self.assertNotEqual(l4, l5)
self.assertNotEqual(hash(l4), hash(l5))
self.assertEqual(l4, l6)
self.assertEqual(hash(l4), hash(l6))

35
scrapy/tests/test_utils_iterators.py
View File

@ -29,12 +29,12 @@ class XmliterTestCase(unittest.TestCase):
for x in self.xmliter(response, 'product'):
attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract()))
self.assertEqual(attrs,
self.assertEqual(attrs,
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
def test_xmliter_text(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')],
[[u'one'], [u'two']])
@ -74,7 +74,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_exception(self):
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
iter = self.xmliter(body, 'product')
iter.next()
iter.next()
@ -97,6 +97,35 @@ class LxmlXmliterTestCase(XmliterTestCase):
except ImportError:
skip = "lxml not available"
def test_xmliter_iterate_namespace(self):
body = """\
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns="http://base.google.com/ns/1.0">
<channel>
<title>My Dummy Company</title>
<link>http://www.mydummycompany.com</link>
<description>This is a dummy company. We do nothing.</description>
<item>
<title>Item 1</title>
<description>This is item 1</description>
<link>http://www.mydummycompany.com/items/1</link>
<image_link>http://www.mydummycompany.com/images/item1.jpg</image_link>
<image_link>http://www.mydummycompany.com/images/item2.jpg</image_link>
</item>
</channel>
</rss>
"""
response = XmlResponse(url='http://mydummycompany.com', body=body)
no_namespace_iter = self.xmliter(response, 'image_link')
self.assertEqual(len(list(no_namespace_iter)), 0)
namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
node = namespace_iter.next()
self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
node = namespace_iter.next()
self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')