From cd42bd7d0c8c8d9d5a4a3f38e442b5020a68116b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 2 Jan 2011 17:21:31 -0200 Subject: [PATCH 1/5] Bumped version to 0.13 --- scrapy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 3d8a7a4ae..c73dae498 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 12, 0) -__version__ = "0.12.0" +version_info = (0, 13, 0) +__version__ = "0.13.0" import sys, os, warnings From d7f193cbeac9dba4eda7bb463b6590900bbb2998 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 2 Jan 2011 17:29:43 -0200 Subject: [PATCH 2/5] bumped version to 0.13 in documentation --- docs/topics/scrapyd.rst | 2 +- docs/topics/ubuntu.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index 6bfe8d3f7..143e983a3 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -86,7 +86,7 @@ in your Ubuntu servers. So, if you plan to deploy Scrapyd on a Ubuntu server, just add the Ubuntu repositories as described in :ref:`topics-ubuntu` and then run:: - aptitude install scrapyd-0.12 + aptitude install scrapyd-0.13 This will install Scrapyd in your Ubuntu server creating a ``scrapy`` user which Scrapyd will run as. It will also create some directories and files that diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 13bdd4b7b..6cd164f7b 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -13,7 +13,7 @@ latest bug fixes. To use the packages, just add the following line to your ``/etc/apt/sources.list``, and then run ``aptitude update`` and ``aptitude -install scrapy-0.12``:: +install scrapy-0.13``:: deb http://archive.scrapy.org/ubuntu DISTRO main From 579463aff252c30e97ccb2045e84f4f4d69690da Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 4 Jan 2011 13:57:32 -0200 Subject: [PATCH 3/5] make scrapy*-0.13 packages conflict with scrapy*-0.12 packages --- debian/control | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index b4b159470..cc0d49040 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,7 @@ Homepage: http://scrapy.org/ Package: scrapy-SUFFIX Architecture: all Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl -Conflicts: python-scrapy, scrapy, scrapy-0.11 +Conflicts: python-scrapy, scrapy, scrapy-0.11, scrapy-0.12 Provides: python-scrapy, scrapy Description: Python web crawling and scraping framework Scrapy is a fast high-level screen scraping and web crawling framework, @@ -20,7 +20,7 @@ Description: Python web crawling and scraping framework Package: scrapyd-SUFFIX Architecture: all Depends: scrapy, python-setuptools -Conflicts: scrapyd, scrapyd-0.11 +Conflicts: scrapyd, scrapyd-0.11, scrapyd-0.12 Provides: scrapyd Description: Scrapy Service The Scrapy service allows you to deploy your Scrapy projects by building From 0ba9999cca6526bd1aea84eb5ce808fc840639b7 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Wed, 5 Jan 2011 11:02:05 -0200 Subject: [PATCH 4/5] Handle badformed tags with no trailing > --- scrapy/contrib/ibl/htmlpage.py | 2 +- scrapy/tests/test_contrib_ibl/test_extraction.py | 2 +- scrapy/tests/test_contrib_ibl/test_htmlpage.py | 10 ++++++++++ scrapy/tests/test_contrib_ibl/test_htmlpage_data.py | 9 +++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/ibl/htmlpage.py b/scrapy/contrib/ibl/htmlpage.py index 023030bdc..30c21c043 100644 --- a/scrapy/contrib/ibl/htmlpage.py +++ b/scrapy/contrib/ibl/htmlpage.py @@ -81,7 +81,7 @@ class HtmlTag(HtmlDataFragment): return str(self) _ATTR = "((?:[^=/>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?" -_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>" +_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>?" _DOCTYPE = r"" _SCRIPT = "()(.*?)()" _COMMENT = "()" diff --git a/scrapy/tests/test_contrib_ibl/test_extraction.py b/scrapy/tests/test_contrib_ibl/test_extraction.py index f79c9dc78..e9ce86f7e 100644 --- a/scrapy/tests/test_contrib_ibl/test_extraction.py +++ b/scrapy/tests/test_contrib_ibl/test_extraction.py @@ -516,7 +516,7 @@ ANNOTATED_PAGE19 = u"""

Product name

60.00

- +

description

diff --git a/scrapy/tests/test_contrib_ibl/test_htmlpage.py b/scrapy/tests/test_contrib_ibl/test_htmlpage.py index cdba56853..b3bc91fa5 100644 --- a/scrapy/tests/test_contrib_ibl/test_htmlpage.py +++ b/scrapy/tests/test_contrib_ibl/test_htmlpage.py @@ -137,3 +137,13 @@ class TestParseHtml(TestCase): parsed = list(parse_html("")) self.assertEqual(parsed[0].attributes, {'src': 'http://images.play.com/banners/SAM550a.jpg', \ 'align': 'left', 'hspace': '5', '/': None}) + + def test_no_ending_body(self): + """Test case when no ending body nor html elements are present""" + parsed = [_decode_element(d) for d in PARSED7] + self._test_sample(PAGE7, parsed) + + def test_malformed(self): + """Test parsing of some malformed cases""" + parsed = [_decode_element(d) for d in PARSED8] + self._test_sample(PAGE8, parsed) diff --git a/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py b/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py index 62cd6b526..39771a666 100644 --- a/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py +++ b/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py @@ -246,3 +246,12 @@ PARSED7 = [ {'end': 99, 'start': 85}, ] +PAGE8 = u"""""" + +PARSED8 = [ + {'attributes' : {u'href' : u"/overview.asp?id=277"}, 'end': 31, 'start': 0, 'tag': u'a', 'tag_type': 1}, + {'attributes' : {u'src' : u"/img/5200814311.jpg", u'border' : u"0", u'title': u'Vinyl Cornice'}, 'end': 94, 'start': 31, 'tag': u'img', 'tag_type': 1}, + {'attributes' : {}, 'end': 98, 'start': 94, 'tag': u'a', 'tag_type': 2}, + {'attributes' : {}, 'end': 103, 'start': 98, 'tag': u'td', 'tag_type': 2}, + {'attributes' : {u'width': u'5'}, 'end': 120, 'start': 103, 'tag': u'table', 'tag_type': 1} +] From ebf5ad933e718198cfa557272693fe0aa57a775f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 5 Jan 2011 11:59:19 -0200 Subject: [PATCH 5/5] fixed compatibility with python 2.5 and removed unused code --- scrapy/utils/memory.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/memory.py b/scrapy/utils/memory.py index bed6821f9..3340a0a6e 100644 --- a/scrapy/utils/memory.py +++ b/scrapy/utils/memory.py @@ -1,3 +1,5 @@ +from __future__ import with_statement + import os import sys import struct @@ -64,7 +66,4 @@ def _vmvalue_solaris(vmkey, pid): vm_in_kB = parts[vmkey_index[vmkey]] - def kB_to_Bytes(kB): - return kB * 1024 - return vm_in_kB * 1024