From 97e4003a560aadd8e6015524cc330f9d2320e707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 4 Apr 2012 16:17:18 -0300 Subject: [PATCH 01/11] do not fail handling unicode xpaths in libxml2 backed selectors --- scrapy/selector/libxml2sel.py | 1 + scrapy/tests/test_selector.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py index 3784902e0..9342c9639 100644 --- a/scrapy/selector/libxml2sel.py +++ b/scrapy/selector/libxml2sel.py @@ -37,6 +37,7 @@ class XPathSelector(object_ref): def select(self, xpath): if hasattr(self.xmlNode, 'xpathEval'): self.doc.xpathContext.setContextNode(self.xmlNode) + xpath = unicode_to_str(xpath, 'utf-8') try: xpath_result = self.doc.xpathContext.xpathEval(xpath) except libxml2.xpathError: diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 3b2bbc982..73fe30216 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -43,6 +43,12 @@ class XPathSelectorTestCase(unittest.TestCase): self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) + def test_selector_unicode_query(self): + body = u"

" + response = TextResponse(url="http://example.com", body=body, encoding='utf8') + xpath = self.hxs_cls(response) + self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1']) + @libxml2debug def test_selector_same_type(self): """Test XPathSelector returning the same type in x() method""" From af0e1c40f5082910b22000df4b0068003611948f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 10 Apr 2012 13:36:52 -0300 Subject: [PATCH 02/11] Avoid logging useless error messages about ignored requests in robots.txt --- scrapy/core/scraper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9e46b6b59..ef6e17110 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -183,8 +183,9 @@ class Scraper(object): errors that got propagated thru here) """ if spider_failure is download_failure: - log.msg("Error downloading %s: %s" % \ - (request, spider_failure.getErrorMessage()), log.ERROR, spider=spider) + errmsg = spider_failure.getErrorMessage() + if errmsg: + log.msg("Error downloading %s: %s" % (request, errmsg), log.ERROR, spider=spider) return return spider_failure From 6e8edbd72ef2214422c0267b908aaa1a4a5125d6 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 10 Apr 2012 15:52:14 -0300 Subject: [PATCH 03/11] switched default selectors backend to lxml --- debian/control | 4 ++-- docs/intro/install.rst | 4 ++-- scrapy/selector/__init__.py | 8 ++++---- setup.py | 6 +----- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/debian/control b/debian/control index 68376f76f..7cd4013c6 100644 --- a/debian/control +++ b/debian/control @@ -2,13 +2,13 @@ Source: scrapy-SUFFIX Section: python Priority: optional Maintainer: Insophia Team -Build-Depends: debhelper (>= 7.0.50), python (>=2.6), python-twisted, python-w3lib +Build-Depends: debhelper (>= 7.0.50), python (>=2.6), python-twisted, python-w3lib, python-lxml Standards-Version: 3.8.4 Homepage: http://scrapy.org/ Package: scrapy-SUFFIX Architecture: all -Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl, python-w3lib (>= 1.1-r23) +Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl, python-w3lib (>= 1.1-r23) Recommends: python-setuptools Conflicts: python-scrapy, scrapy, scrapy-0.11 Provides: python-scrapy, scrapy diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4f7aa52be..4f6e30fec 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -111,7 +111,7 @@ Debian or Ubuntu (9.04 or older) If you're running Debian Linux, run the following command as root:: - apt-get install python-twisted python-libxml2 python-pyopenssl python-simplejson + apt-get install python-twisted python-lxmlxml python-pyopenssl python-simplejson Then:: @@ -124,7 +124,7 @@ Arch Linux If you are running Arch Linux, run the following command as root:: - pacman -S twisted libxml2 pyopenssl python-simplejson + pacman -S twisted python-lxml pyopenssl python-simplejson Then:: diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 392108ea7..e9af791ac 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -18,13 +18,13 @@ elif settings['SELECTORS_BACKEND'] == 'dummy': from scrapy.selector.dummysel import * else: try: - import libxml2 + import lxml except ImportError: try: - import lxml + import libxml2 except ImportError: from scrapy.selector.dummysel import * else: - from scrapy.selector.lxmlsel import * + from scrapy.selector.libxml2sel import * else: - from scrapy.selector.libxml2sel import * + from scrapy.selector.lxmlsel import * diff --git a/setup.py b/setup.py index 7bcd127a4..a085fa450 100644 --- a/setup.py +++ b/setup.py @@ -120,12 +120,8 @@ try: except ImportError: from distutils.core import setup else: - setup_args['install_requires'] = ['Twisted>=8.0', 'w3lib>=1.1', 'pyOpenSSL'] + setup_args['install_requires'] = ['Twisted>=8.0', 'w3lib>=1.1', 'lxml', 'pyOpenSSL'] if sys.version_info < (2, 6): setup_args['install_requires'] += ['simplejson'] - try: - import libxml2 - except ImportError: - setup_args['install_requires'] += ['lxml'] setup(**setup_args) From 4f28ffcb2cc476e5c6e02ffddbef963b4a5a4e5b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 10 Apr 2012 16:01:36 -0300 Subject: [PATCH 04/11] removed no longer needed dependency on simplejson --- docs/intro/install.rst | 7 ++----- scrapy/utils/serialize.py | 2 +- setup.py | 2 -- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4f6e30fec..4699ead90 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -20,8 +20,6 @@ Requirements * `lxml`_ or `libxml2`_ (if using `libxml2`_, version 2.6.28 or above is highly recommended) -* `simplejson`_ (not required if using Python 2.6 or above) - * `pyopenssl `_ (for HTTPS support. Optional, but highly recommended) @@ -111,7 +109,7 @@ Debian or Ubuntu (9.04 or older) If you're running Debian Linux, run the following command as root:: - apt-get install python-twisted python-lxmlxml python-pyopenssl python-simplejson + apt-get install python-twisted python-lxmlxml python-pyopenssl Then:: @@ -124,7 +122,7 @@ Arch Linux If you are running Arch Linux, run the following command as root:: - pacman -S twisted python-lxml pyopenssl python-simplejson + pacman -S twisted python-lxml pyopenssl Then:: @@ -178,7 +176,6 @@ There are two ways to install Scrapy in Windows: .. _lxml: http://codespeak.net/lxml/ .. _libxml2: http://xmlsoft.org .. _pywin32: http://sourceforge.net/projects/pywin32/ -.. _simplejson: http://pypi.python.org/pypi/simplejson/ .. _Zope.Interface: http://pypi.python.org/pypi/zope.interface#download .. _this Twisted bug: http://twistedmatrix.com/trac/ticket/3707 .. _pip: http://pypi.python.org/pypi/pip diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 1c0e603c7..41ee88860 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -12,7 +12,7 @@ from scrapy.http import Request, Response class SpiderReferencer(object): """Class to serialize (and deserialize) objects (typically dicts) containing references to running spiders (ie. Spider objects). This is - required because simplejson fails to serialize dicts containing + required because json library fails to serialize dicts containing non-primitive types as keys, even when you override ScrapyJSONEncoder.default() with a custom encoding mechanism. """ diff --git a/setup.py b/setup.py index a085fa450..7e5349b3e 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,5 @@ except ImportError: from distutils.core import setup else: setup_args['install_requires'] = ['Twisted>=8.0', 'w3lib>=1.1', 'lxml', 'pyOpenSSL'] - if sys.version_info < (2, 6): - setup_args['install_requires'] += ['simplejson'] setup(**setup_args) From 02833e3265eece72c7a0ea992adea53824ad883f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 11 Apr 2012 10:44:31 -0300 Subject: [PATCH 05/11] fix typo in module description. closes #112 --- scrapy/utils/signal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index d05ab5c70..5496b7041 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,4 +1,4 @@ -"""Helper functinos for working with signals""" +"""Helper functions for working with signals""" from twisted.internet.defer import maybeDeferred, DeferredList, Deferred from twisted.python.failure import Failure From f1802289cd0a2755656b539bd42d5fdf53718f5a Mon Sep 17 00:00:00 2001 From: stav Date: Wed, 11 Apr 2012 12:05:39 -0500 Subject: [PATCH 06/11] small doc typo change to get the fork rolling --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 69078df84..7d8501fd8 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -64,7 +64,7 @@ single Python class that defines one or more of the following methods: This method is called for each response that goes through the spider middleware and into the spider, for processing. - :meth:`process_spider_input` should return ``None`` or raise and + :meth:`process_spider_input` should return ``None`` or raise an exception. If it returns ``None``, Scrapy will continue processing this response, From 7cca916ed5e6ac5b896e77824f5b1baca71c9235 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 11 Apr 2012 15:53:23 -0300 Subject: [PATCH 07/11] added release notes to official documentation, including all release notes since Scrapy 0.7 --- docs/_ext/scrapydocs.py | 16 +- docs/index.rst | 4 + docs/news.rst | 339 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 docs/news.rst diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 54622f1ad..995db5095 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -23,9 +23,23 @@ def setup(app): indextemplate = "pair: %s; reqmeta", ) app.add_role('source', source_role) + app.add_role('commit', commit_role) + app.add_role('rev', rev_role) def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - url = 'https://github.com/scrapy/scrapy/blob/master/' + text + ref = 'https://github.com/scrapy/scrapy/blob/master/' + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] + +def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + ref = 'https://github.com/scrapy/scrapy/commit/' + text + set_classes(options) + node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) + return [node], [] + +def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + ref = 'http://hg.scrapy.org/scrapy/changeset/' + text + set_classes(options) + node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options) + return [node], [] diff --git a/docs/index.rst b/docs/index.rst index 1236d5329..80b5d513d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -224,10 +224,14 @@ All the rest .. toctree:: :hidden: + news contributing versioning experimental/index +:doc:`news` + See what has changed in recent Scrapy versions. + :doc:`contributing` Learn how to contribute to the Scrapy project. diff --git a/docs/news.rst b/docs/news.rst new file mode 100644 index 000000000..42289fa3d --- /dev/null +++ b/docs/news.rst @@ -0,0 +1,339 @@ +Release notes +============= + +0.16 (not yet released) +----------------------- + +Scrapy changes: + +- SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`) +- StackTraceDump extension: also dump trackref live references (:commit:`fe2ce93`) +- added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider +- decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that mdule +- dropped support for Python 2.5. See http://blog.scrapy.org/scrapy-dropping-support-for-python-25 +- dropped support for Twisted 2.5 +- added :setting:`REFERER_ENABLED` setting, to control referer middleware +- changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)`` + +Scrapyd changes: + +- New Scrapy API methods (see documentation for details) + - ``listjobs.json`` to see pending/running/finished jobs + - ``cancel.json`` to cancel pending and running jobs +- Items are now stored on disk using feed exports, and accessible through the Scrapyd web interface +- Support making Scrapyd listen into a specific IP address (see ``bind_address`` option) + +0.14.2 +------ + +- bumped version to 0.14.2 (:commit:`68c1c98`) +- move buffer pointing to start of file before computing checksum. refs #92 (:commit:`6a5bef2`) +- Compute image checksum before persisting images. closes #92 (:commit:`9817df1`) +- remove leaking references in cached failures (:commit:`673a120`) +- fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) +- Merge branch '0.14' of github.com:scrapy/scrapy into 0.14 (:commit:`1627320`) +- fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) +- ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) +- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Speed-up libxml2 XPathSelector (:commit:`2fbd662`) +- updated versioning doc according to recent changes (:commit:`0a070f5`) +- scrapyd: fixed documentation link (:commit:`2b4e4c3`) +- extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`) + +0.14.1 +------ + +- extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`) +- bumped version to 0.14.1 (:commit:`6cb9e1c`) +- fixed reference to tutorial directory (:commit:`4b86bd6`) +- doc: removed duplicated callback argument from Request.replace() (:commit:`1aeccdd`) +- fixed formatting of scrapyd doc (:commit:`8bf19e6`) +- Dump stacks for all running threads and fix engine status dumped by StackTraceDump extension (:commit:`14a8e6e`) +- added comment about why we disable ssl on boto images upload (:commit:`5223575`) +- SSL handshaking hangs when doing too many parallel connections to S3 (:commit:`63d583d`) +- change tutorial to follow changes on dmoz site (:commit:`bcb3198`) +- Avoid _disconnectedDeferred AttributeError exception in Twisted>=11.1.0 (:commit:`98f3f87`) +- allow spider to set autothrottle max concurrency (:commit:`175a4b5`) + +0.14 +---- + +New features and settings +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Support for `AJAX crawleable urls`_ +- New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`) +- added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``) +- Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`) +- New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`) +- Add boto 2.0 support for S3 downloader handler (:rev:`2763`) +- Added `marshal`_ to formats supported by feed exports (:rev:`2744`) +- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`) +- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) + - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - check the documentation for more details +- Added builtin caching DNS resolver (:rev:`2728`) +- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) +- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`) +- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`) +- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`) +- Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`) +- Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`) +- Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`) +- Improved encoding detection by adding support for HTML5 meta charset declaration (:rev:`2690`) +- Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`) +- Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`) +- Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`) +- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`. +- Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`) +- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`) +- Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines. +- Added :signal:`spider_error` signal (:rev:`2628`) +- Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`) +- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there. +- Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`) +- Added new DBM HTTP cache storage backend (:rev:`2576`) +- Added ``listjobs.json`` API to Scrapyd (:rev:`2571`) +- ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`) +- Added namespace support to ``xmliter_lxml`` (:rev:`2552`) +- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`) +- Several improvements to Scrapyd and Link extractors + +Code rearranged and removed +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Merged item passed and item scraped concepts, as they have often proved confusing in the past. This means: (:rev:`2630`) + - original item_scraped signal was removed + - original item_passed signal was renamed to item_scraped + - old log lines ``Scraped Item...`` were removed + - old log lines ``Passed Item...`` were renamed to ``Scraped Item...`` lines and downgraded to ``DEBUG`` level +- Reduced Scrapy codebase by striping part of Scrapy code into two new libraries: + - `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`) + - `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`) +- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`) +- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot +- Removed support for default field values in Scrapy items (:rev:`2616`) +- Removed experimental crawlspider v2 (:rev:`2632`) +- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`) +- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`) +- Removed deprecated Execution Queue (:rev:`2704`) +- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`) +- removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`) +- Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`) +- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept. + +0.12 +---- + +The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. + +New features and improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273) +- Added verbose option to ``scrapy version`` command, useful for bug reports (#298) +- HTTP cache now stored by default in the project data dir (#279) +- Added project data storage directory (#276, #277) +- Documented file structure of Scrapy projects (see command-line tool doc) +- New lxml backend for XPath selectors (#147) +- Per-spider settings (#245) +- Support exit codes to signal errors in Scrapy commands (#248) +- Added ``-c`` argument to ``scrapy shell`` command +- Made ``libxml2`` optional (#260) +- New ``deploy`` command (#261) +- Added :setting:`CLOSESPIDER_PAGECOUNT` setting (#253) +- Added :setting:`CLOSESPIDER_ERRORCOUNT` setting (#254) + +Scrapyd changes +~~~~~~~~~~~~~~~ + +- Scrapyd now uses one process per spider +- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default) +- A minimal web ui was added, available at http://localhost:6800 by default +- There is now a `scrapy server` command to start a Scrapyd server of the current project + +Changes to settings +~~~~~~~~~~~~~~~~~~~ + +- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware +- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire". + +Deprecated/obsoleted functionality +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Deprecated ``runserver`` command in favor of ``server`` command which starts a Scrapyd server. See also: Scrapyd changes +- Deprecated ``queue`` command in favor of using Scrapyd ``schedule.json`` API. See also: Scrapyd changes +- Removed the !LxmlItemLoader (experimental contrib which never graduated to main contrib) + +0.10 +---- + +The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. + +New features and improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- New Scrapy service called ``scrapyd`` for deploying Scrapy crawlers in production (#218) (documentation available) +- Simplified Images pipeline usage which doesn't require subclassing your own images pipeline now (#217) +- Scrapy shell now shows the Scrapy log by default (#206) +- Refactored execution queue in a common base code and pluggable backends called "spider queues" (#220) +- New persistent spider queue (based on SQLite) (#198), available by default, which allows to start Scrapy in server mode and then schedule spiders to run. +- Added documentation for Scrapy command-line tool and all its available sub-commands. (documentation available) +- Feed exporters with pluggable backends (#197) (documentation available) +- Deferred signals (#193) +- Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195) +- Support for overriding default request headers per spider (#181) +- Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186) +- Splitted Debian package into two packages - the library and the service (#187) +- Scrapy log refactoring (#188) +- New extension for keeping persistent spider contexts among different runs (#203) +- Added `dont_redirect` request.meta key for avoiding redirects (#233) +- Added `dont_retry` request.meta key for avoiding retries (#234) + +Command-line tool changes +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199) + - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project + - Added `scrapy.bat` script for running more conveniently from Windows +- Added bash completion to command-line tool (#210) +- Renamed command `start` to `runserver` (#209) + +API changes +~~~~~~~~~~~ + +- ``url`` and ``body`` attributes of Request objects are now read-only (#230) +- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) +- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) - see this snippet for a replacement: http://snippets.scrapy.org/snippets/12/ +- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) +- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself. +- Changes to Scrapy Manager (now called "Crawler"): + - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` + - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler`` +- Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager`` +- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton. +- moved Stats Collector classes: (#204) + - ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector`` + - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` +- default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201) +- changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)`` + - backwards compatibility kept (with deprecation warning) +- moved ``scrapy.core.signals`` module to ``scrapy.signals`` + - backwards compatibility kept (with deprecation warning) +- moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions`` + - backwards compatibility kept (with deprecation warning) +- added ``handles_request()`` class method to ``BaseSpider`` +- dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead) +- dropped ``component`` argument of ``scrapy.log.msg()`` function +- dropped ``scrapy.log.log_level`` attribute +- Added ``from_settings()`` class methods to Spider Manager, and Item Pipeline Manager + +Changes to settings +~~~~~~~~~~~~~~~~~~~ + +- Added ``HTTPCACHE_IGNORE_SCHEMES`` setting to ignore certain schemes on !HttpCacheMiddleware (#225) +- Added ``SPIDER_QUEUE_CLASS`` setting which defines the spider queue to use (#220) +- Added ``KEEP_ALIVE`` setting (#220) +- Removed ``SERVICE_QUEUE`` setting (#220) +- Removed ``COMMANDS_SETTINGS_MODULE`` setting (#201) +- Renamed ``REQUEST_HANDLERS`` to ``DOWNLOAD_HANDLERS`` and make download handlers classes (instead of functions) + +0.9 +--- + +The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. + +New features and improvements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Added SMTP-AUTH support to scrapy.mail +- New settings added: ``MAIL_USER``, ``MAIL_PASS`` (:rev:`2065` | #149) +- Added new scrapy-ctl view command - To view URL in the browser, as seen by Scrapy (:rev:`2039`) +- Added web service for controlling Scrapy process (this also deprecates the web console. (:rev:`2053` | #167) +- Support for running Scrapy as a service, for production systems (:rev:`1988`, :rev:`2054`, :rev:`2055`, :rev:`2056`, :rev:`2057` | #168) +- Added wrapper induction library (documentation only available in source code for now). (:rev:`2011`) +- Simplified and improved response encoding support (:rev:`1961`, :rev:`1969`) +- Added ``LOG_ENCODING`` setting (:rev:`1956`, documentation available) +- Added ``RANDOMIZE_DOWNLOAD_DELAY`` setting (enabled by default) (:rev:`1923`, doc available) +- ``MailSender`` is no longer IO-blocking (:rev:`1955` | #146) +- Linkextractors and new Crawlspider now handle relative base tag urls (:rev:`1960` | #148) +- Several improvements to Item Loaders and processors (:rev:`2022`, :rev:`2023`, :rev:`2024`, :rev:`2025`, :rev:`2026`, :rev:`2027`, :rev:`2028`, :rev:`2029`, :rev:`2030`) +- Added support for adding variables to telnet console (:rev:`2047` | #165) +- Support for requests without callbacks (:rev:`2050` | #166) + +API changes +~~~~~~~~~~~ + +- Change ``Spider.domain_name`` to ``Spider.name`` (SEP-012, :rev:`1975`) +- ``Response.encoding`` is now the detected encoding (:rev:`1961`) +- ``HttpErrorMiddleware`` now returns None or raises an exception (:rev:`2006` | #157) +- ``scrapy.command`` modules relocation (:rev:`2035`, :rev:`2036`, :rev:`2037`) +- Added ``ExecutionQueue`` for feeding spiders to scrape (:rev:`2034`) +- Removed ``ExecutionEngine`` singleton (:rev:`2039`) +- Ported ``S3ImagesStore`` (images pipeline) to use boto and threads (:rev:`2033`) +- Moved module: ``scrapy.management.telnet`` to ``scrapy.telnet`` (:rev:`2047`) + +Changes to default settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Changed default ``SCHEDULER_ORDER`` to ``DFO`` (:rev:`1939`) + +0.8 +--- + +The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available. + +New features +~~~~~~~~~~~~ + +- Added DEFAULT_RESPONSE_ENCODING setting (:rev:`1809`) +- Added ``dont_click`` argument to ``FormRequest.from_response()`` method (:rev:`1813`, :rev:`1816`) +- Added ``clickdata`` argument to ``FormRequest.from_response()`` method (:rev:`1802`, :rev:`1803`) +- Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`) +- Offiste spider middleware now logs messages when filtering out requests (:rev:`1841`) + +Backwards-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`) +- Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`) +- Removed deprecated ``scrapy.xpath`` module - use ``scrapy.selector`` instead. (:rev:`1836`) +- Removed deprecated ``core.signals.domain_open`` signal - use ``core.signals.domain_opened`` instead (:rev:`1822`) +- ``log.msg()`` now receives a ``spider`` argument (:rev:`1822`) + - Old domain argument has been deprecated and will be removed in 0.9. For spiders, you should always use the ``spider`` argument and pass spider references. If you really want to pass a string, use the ``component`` argument instead. +- Changed core signals ``domain_opened``, ``domain_closed``, ``domain_idle`` +- Changed Item pipeline to use spiders instead of domains + - The ``domain`` argument of ``process_item()`` item pipeline method was changed to ``spider``, the new signature is: ``process_item(spider, item)`` (:rev:`1827` | #105) + - To quickly port your code (to work with Scrapy 0.8) just use ``spider.domain_name`` where you previously used ``domain``. +- Changed Stats API to use spiders instead of domains (:rev:`1849` | #113) + - ``StatsCollector`` was changed to receive spider references (instead of domains) in its methods (``set_value``, ``inc_value``, etc). + - added ``StatsCollector.iter_spider_stats()`` method + - removed ``StatsCollector.list_domains()`` method + - Also, Stats signals were renamed and now pass around spider references (instead of domains). Here's a summary of the changes: + - To quickly port your code (to work with Scrapy 0.8) just use ``spider.domain_name`` where you previously used ``domain``. ``spider_stats`` contains exactly the same data as ``domain_stats``. +- ``CloseDomain`` extension moved to ``scrapy.contrib.closespider.CloseSpider`` (:rev:`1833`) + - Its settings were also renamed: + - ``CLOSEDOMAIN_TIMEOUT`` to ``CLOSESPIDER_TIMEOUT`` + - ``CLOSEDOMAIN_ITEMCOUNT`` to ``CLOSESPIDER_ITEMCOUNT`` +- Removed deprecated ``SCRAPYSETTINGS_MODULE`` environment variable - use ``SCRAPY_SETTINGS_MODULE`` instead (:rev:`1840`) +- Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`) +- Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`) +- Refactored HTTP Cache middleware +- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` ) +- Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120) +- Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121) +- Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`) + +0.7 +--- + +First release of Scrapy. + + +.. _AJAX crawleable urls: http://code.google.com/web/ajaxcrawling/docs/getting-started.html +.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding +.. _w3lib: http://https://github.com/scrapy/w3lib +.. _scrapely: https://github.com/scrapy/scrapely +.. _marshal: http://docs.python.org/library/marshal.html +.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py From b9efa5ee73867a67891dc84124221ffc0f076305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 13 Apr 2012 00:21:48 -0300 Subject: [PATCH 08/11] more fixes to lxml selector incompatibilities * Do not fail parsing empty bodies * Do not fail parsing bodies with null bytes * Recode to utf8 using response.body_as_unicode() to avoid decoding bugs * Return empty results with unevaluable nodes like text or attribute nodes * Return u'1' and u'0' for boolean xpaths --- scrapy/selector/lxmlsel.py | 32 +++++++++++----------- scrapy/tests/test_selector.py | 51 ++++++++++++++++++++++++++++++++--- 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index cc32a81f4..301858890 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -35,27 +35,29 @@ class XPathSelector(object_ref): @property def root(self): if self._root is None: - parser = self._parser(encoding=self.response.encoding, recover=True) - self._root = etree.fromstring(self.response.body, parser=parser, \ - base_url=self.response.url) + url = self.response.url + body = self.response.body_as_unicode().strip().encode('utf8') or '' + parser = self._parser(recover=True, encoding='utf8') + self._root = etree.fromstring(body, parser=parser, base_url=url) + assert self._root is not None, 'BUG lxml selector with None root' return self._root - @property - def xpathev(self): - if self._xpathev is None: - self._xpathev = etree.XPathEvaluator(self.root, namespaces=self.namespaces) - return self._xpathev - def select(self, xpath): try: - result = self.xpathev(xpath) + xpathev = self.root.xpath + except AttributeError: + return XPathSelectorList([]) + + try: + result = xpathev(xpath, namespaces=self.namespaces) except etree.XPathError: raise ValueError("Invalid XPath: %s" % xpath) - if hasattr(result, '__iter__'): - result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \ - for x in result] - else: - result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)] + + if type(result) is not list: + result = [result] + + result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) + for x in result] return XPathSelectorList(result) def re(self, regex): diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 73fe30216..20672098f 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -238,9 +238,55 @@ class XPathSelectorTestCase(unittest.TestCase): @libxml2debug def test_empty_bodies(self): + # shouldn't raise errors r1 = TextResponse('http://www.example.com', body='') - self.hxs_cls(r1) # shouldn't raise error - self.xxs_cls(r1) # shouldn't raise error + self.hxs_cls(r1).select('//text()').extract() + self.xxs_cls(r1).select('//text()').extract() + + @libxml2debug + def test_null_bytes(self): + # shouldn't raise errors + r1 = TextResponse('http://www.example.com', \ + body='pre\x00post', \ + encoding='utf-8') + self.hxs_cls(r1).select('//text()').extract() + self.xxs_cls(r1).select('//text()').extract() + + @libxml2debug + def test_badly_encoded_body(self): + # \xe9 alone isn't valid utf8 sequence + r1 = TextResponse('http://www.example.com', \ + body='

an Jos\xe9 de

', \ + encoding='utf-8') + self.hxs_cls(r1).select('//text()').extract() + self.xxs_cls(r1).select('//text()').extract() + + @libxml2debug + def test_select_on_unevaluable_nodes(self): + r = self.hxs_cls(text=u'some text') + # Text node + x1 = r.select('//text()') + self.assertEquals(x1.extract(), [u'some text']) + self.assertEquals(x1.select('.//b').extract(), []) + # Tag attribute + x1 = r.select('//span/@class') + self.assertEquals(x1.extract(), [u'big']) + self.assertEquals(x1.select('.//text()').extract(), []) + + @libxml2debug + def test_select_on_text_nodes(self): + # FIXME: can't get this to work with lxml backend + r = self.hxs_cls(text=u'
Options:opt1
Otheropt2
') + x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") + self.assertEquals(x1.extract(), [u'opt1']) + + x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") + self.assertEquals(x1.extract(), [u'Options:']) + + x1 = r.select("//div/descendant::text()") + x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]") + self.assertEquals(x2.extract(), [u'Options:']) + test_select_on_text_nodes.skip = True @libxml2debug def test_weakref_slots(self): @@ -250,4 +296,3 @@ class XPathSelectorTestCase(unittest.TestCase): weakref.ref(x) assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ x.__class__.__name__ - From 3dbe211d29c84aaceceb80fc9572ed597aac94d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 13 Apr 2012 09:29:56 -0300 Subject: [PATCH 09/11] lxml boolean results fix. oops #116 --- scrapy/selector/lxmlsel.py | 7 ++++++- scrapy/tests/test_selector.py | 8 ++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 301858890..bb0a4e1ec 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -68,7 +68,12 @@ class XPathSelector(object_ref): return etree.tostring(self.root, method=self._tostring_method, \ encoding=unicode) except (AttributeError, TypeError): - return unicode(self.root) + if self.root is True: + return u'1' + elif self.root is False: + return u'0' + else: + return unicode(self.root) def register_namespace(self, prefix, uri): if self.namespaces is None: diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 20672098f..e4197b084 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -63,12 +63,8 @@ class XPathSelectorTestCase(unittest.TestCase): body = "

" response = TextResponse(url="http://example.com", body=body) xs = self.hxs_cls(response) - true = xs.select("//input[@name='a']/@name='a'").extract()[0] - false = xs.select("//input[@name='a']/@name='n'").extract()[0] - - # the actual result depends on the backend used - assert true in [u'1', u'True'], true - assert false in [u'0', u'False'], false + self.assertEquals(xs.select("//input[@name='a']/@name='a'").extract(), [u'1']) + self.assertEquals(xs.select("//input[@name='a']/@name='n'").extract(), [u'0']) @libxml2debug def test_selector_xml_html(self): From ac4f6cc17cffed99d6ba218d2323a7383c1a80cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 13 Apr 2012 13:54:48 -0300 Subject: [PATCH 10/11] unify libxml2 document and factories --- scrapy/selector/document.py | 40 -------------- scrapy/selector/factories.py | 45 --------------- scrapy/selector/libxml2document.py | 80 +++++++++++++++++++++++++++ scrapy/selector/libxml2sel.py | 3 +- scrapy/tests/test_selector_libxml2.py | 3 +- 5 files changed, 83 insertions(+), 88 deletions(-) delete mode 100644 scrapy/selector/document.py delete mode 100644 scrapy/selector/factories.py create mode 100644 scrapy/selector/libxml2document.py diff --git a/scrapy/selector/document.py b/scrapy/selector/document.py deleted file mode 100644 index ba75053ea..000000000 --- a/scrapy/selector/document.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -This module contains a simple class (Libxml2Document) which provides cache and -garbage collection to libxml2 documents (xmlDoc). -""" - -import weakref - -from scrapy.utils.trackref import object_ref -from .factories import xmlDoc_from_html - -class Libxml2Document(object_ref): - - cache = weakref.WeakKeyDictionary() - __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] - - def __new__(cls, response, factory=xmlDoc_from_html): - cache = cls.cache.setdefault(response, {}) - if factory not in cache: - obj = object_ref.__new__(cls) - obj.xmlDoc = factory(response) - obj.xpathContext = obj.xmlDoc.xpathNewContext() - cache[factory] = obj - return cache[factory] - - def __del__(self): - # we must call both cleanup functions, so we try/except all exceptions - # to make sure one doesn't prevent the other from being called - # this call sometimes raises a "NoneType is not callable" TypeError - # so the try/except block silences them - try: - self.xmlDoc.freeDoc() - except: - pass - try: - self.xpathContext.xpathFreeContext() - except: - pass - - def __str__(self): - return "" % self.xmlDoc.name diff --git a/scrapy/selector/factories.py b/scrapy/selector/factories.py deleted file mode 100644 index 44dc4f94f..000000000 --- a/scrapy/selector/factories.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This module provides functions for generating libxml2 documents (xmlDoc). - -Constructors must receive a Response object and return a xmlDoc object. -""" - -import libxml2 - -xml_parser_options = libxml2.XML_PARSE_RECOVER + \ - libxml2.XML_PARSE_NOERROR + \ - libxml2.XML_PARSE_NOWARNING - -html_parser_options = libxml2.HTML_PARSE_RECOVER + \ - libxml2.HTML_PARSE_NOERROR + \ - libxml2.HTML_PARSE_NOWARNING - -utf8_encodings = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) - -def body_as_utf8(response): - if response.encoding in utf8_encodings: - return response.body - else: - return response.body_as_unicode().encode('utf-8') - -def xmlDoc_from_html(response): - """Return libxml2 doc for HTMLs""" - utf8body = body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ - html_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', html_parser_options) - return lxdoc - -def xmlDoc_from_xml(response): - """Return libxml2 doc for XMLs""" - utf8body = body_as_utf8(response) or ' ' - try: - lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ - xml_parser_options) - except TypeError: # libxml2 doesn't parse text with null bytes - lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ - 'utf-8', xml_parser_options) - return lxdoc diff --git a/scrapy/selector/libxml2document.py b/scrapy/selector/libxml2document.py new file mode 100644 index 000000000..a1b903772 --- /dev/null +++ b/scrapy/selector/libxml2document.py @@ -0,0 +1,80 @@ +""" +This module contains a simple class (Libxml2Document) which provides cache and +garbage collection to libxml2 documents (xmlDoc). +""" + +import weakref +import libxml2 +from scrapy.utils.trackref import object_ref + +xml_parser_options = libxml2.XML_PARSE_RECOVER + \ + libxml2.XML_PARSE_NOERROR + \ + libxml2.XML_PARSE_NOWARNING + +html_parser_options = libxml2.HTML_PARSE_RECOVER + \ + libxml2.HTML_PARSE_NOERROR + \ + libxml2.HTML_PARSE_NOWARNING + + +_UTF8_ENCODINGS = set(('utf-8', 'UTF-8', 'utf8', 'UTF8')) +def _body_as_utf8(response): + if response.encoding in _UTF8_ENCODINGS: + return response.body + else: + return response.body_as_unicode().encode('utf-8') + + +def xmlDoc_from_html(response): + """Return libxml2 doc for HTMLs""" + utf8body = _body_as_utf8(response) or ' ' + try: + lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \ + html_parser_options) + except TypeError: # libxml2 doesn't parse text with null bytes + lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \ + 'utf-8', html_parser_options) + return lxdoc + + +def xmlDoc_from_xml(response): + """Return libxml2 doc for XMLs""" + utf8body = _body_as_utf8(response) or ' ' + try: + lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \ + xml_parser_options) + except TypeError: # libxml2 doesn't parse text with null bytes + lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \ + 'utf-8', xml_parser_options) + return lxdoc + + +class Libxml2Document(object_ref): + + cache = weakref.WeakKeyDictionary() + __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] + + def __new__(cls, response, factory=xmlDoc_from_html): + cache = cls.cache.setdefault(response, {}) + if factory not in cache: + obj = object_ref.__new__(cls) + obj.xmlDoc = factory(response) + obj.xpathContext = obj.xmlDoc.xpathNewContext() + cache[factory] = obj + return cache[factory] + + def __del__(self): + # we must call both cleanup functions, so we try/except all exceptions + # to make sure one doesn't prevent the other from being called + # this call sometimes raises a "NoneType is not callable" TypeError + # so the try/except block silences them + try: + self.xmlDoc.freeDoc() + except: + pass + try: + self.xpathContext.xpathFreeContext() + except: + pass + + def __str__(self): + return "" % self.xmlDoc.name diff --git a/scrapy/selector/libxml2sel.py b/scrapy/selector/libxml2sel.py index 9342c9639..7bd3b8a38 100644 --- a/scrapy/selector/libxml2sel.py +++ b/scrapy/selector/libxml2sel.py @@ -9,8 +9,7 @@ from scrapy.utils.python import unicode_to_str from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated -from .factories import xmlDoc_from_html, xmlDoc_from_xml -from .document import Libxml2Document +from .libxml2document import Libxml2Document, xmlDoc_from_html, xmlDoc_from_xml from .list import XPathSelectorList __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ diff --git a/scrapy/tests/test_selector_libxml2.py b/scrapy/tests/test_selector_libxml2.py index 917ddaf96..e447a6297 100644 --- a/scrapy/tests/test_selector_libxml2.py +++ b/scrapy/tests/test_selector_libxml2.py @@ -7,10 +7,11 @@ import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \ XPathSelector -from scrapy.selector.document import Libxml2Document +from scrapy.selector.libxml2document import Libxml2Document from scrapy.utils.test import libxml2debug from scrapy.tests import test_selector + class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase): xs_cls = XPathSelector From 4c7d29b7f7fd6071449e925ce4f6df6e5b36c808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 13 Apr 2012 14:43:30 -0300 Subject: [PATCH 11/11] Cache response's element trees using LxmlDocument similar to Libxml2Document --- scrapy/selector/lxmldocument.py | 31 ++++++++++++++++ scrapy/selector/lxmlsel.py | 49 +++++++++++-------------- scrapy/tests/test_selector_lxml.py | 59 ++++++++++++++++-------------- 3 files changed, 84 insertions(+), 55 deletions(-) create mode 100644 scrapy/selector/lxmldocument.py diff --git a/scrapy/selector/lxmldocument.py b/scrapy/selector/lxmldocument.py new file mode 100644 index 000000000..065ce222b --- /dev/null +++ b/scrapy/selector/lxmldocument.py @@ -0,0 +1,31 @@ +""" +This module contains a simple class (LxmlDocument) which provides cache and +garbage collection to lxml element tree documents. +""" + +import weakref +from lxml import etree +from scrapy.utils.trackref import object_ref + + +def _factory(response, parser_cls): + url = response.url + body = response.body_as_unicode().strip().encode('utf8') or '' + parser = parser_cls(recover=True, encoding='utf8') + return etree.fromstring(body, parser=parser, base_url=url) + + +class LxmlDocument(object_ref): + + cache = weakref.WeakKeyDictionary() + __slots__ = ['xmlDoc', 'xpathContext', '__weakref__'] + + def __new__(cls, response, parser=etree.HTMLParser): + cache = cls.cache.setdefault(response, {}) + if parser not in cache: + obj = object_ref.__new__(cls) + cache[parser] = _factory(response, parser) + return cache[parser] + + def __str__(self): + return "" % self.root.tag diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index bb0a4e1ec..af6839474 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -9,42 +9,35 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.python import unicode_to_str from scrapy.utils.decorator import deprecated from scrapy.http import TextResponse +from .lxmldocument import LxmlDocument from .list import XPathSelectorList + __all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \ - 'XPathSelectorList'] + 'XPathSelectorList'] + class XPathSelector(object_ref): - __slots__ = ['response', 'text', 'expr', 'namespaces', '_root', '_xpathev', \ - '__weakref__'] + __slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__'] _parser = etree.HTMLParser _tostring_method = 'html' - def __init__(self, response=None, text=None, root=None, expr=None, namespaces=None): - if text: - self.response = TextResponse(url='about:blank', \ + def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None): + if text is not None: + response = TextResponse(url='about:blank', \ body=unicode_to_str(text, 'utf-8'), encoding='utf-8') - else: - self.response = response - self._root = root - self._xpathev = None - self.namespaces = namespaces - self.expr = expr + if response is not None: + _root = LxmlDocument(response, self._parser) - @property - def root(self): - if self._root is None: - url = self.response.url - body = self.response.body_as_unicode().strip().encode('utf8') or '' - parser = self._parser(recover=True, encoding='utf8') - self._root = etree.fromstring(body, parser=parser, base_url=url) - assert self._root is not None, 'BUG lxml selector with None root' - return self._root + self.namespaces = namespaces + self.response = response + self._root = _root + self._expr = _expr def select(self, xpath): try: - xpathev = self.root.xpath + xpathev = self._root.xpath except AttributeError: return XPathSelectorList([]) @@ -56,7 +49,7 @@ class XPathSelector(object_ref): if type(result) is not list: result = [result] - result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) + result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces) for x in result] return XPathSelectorList(result) @@ -65,15 +58,15 @@ class XPathSelector(object_ref): def extract(self): try: - return etree.tostring(self.root, method=self._tostring_method, \ + return etree.tostring(self._root, method=self._tostring_method, \ encoding=unicode) except (AttributeError, TypeError): - if self.root is True: + if self._root is True: return u'1' - elif self.root is False: + elif self._root is False: return u'0' else: - return unicode(self.root) + return unicode(self._root) def register_namespace(self, prefix, uri): if self.namespaces is None: @@ -85,7 +78,7 @@ class XPathSelector(object_ref): def __str__(self): data = repr(self.extract()[:40]) - return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data) + return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data) __repr__ = __str__ diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index 8050a2758..c8eca9679 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -2,35 +2,40 @@ Selectors tests, specific for lxml backend """ -from scrapy.http import TextResponse, XmlResponse -has_lxml = True -try: - from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \ - XPathSelector -except ImportError: - has_lxml = False -from scrapy.utils.test import libxml2debug +import unittest from scrapy.tests import test_selector +from scrapy.http import TextResponse, HtmlResponse +from scrapy.selector.lxmldocument import LxmlDocument +from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector + class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): - if has_lxml: - xs_cls = XPathSelector - hxs_cls = HtmlXPathSelector - xxs_cls = XmlXPathSelector - else: - skip = "lxml not available" + xs_cls = XPathSelector + hxs_cls = HtmlXPathSelector + xxs_cls = XmlXPathSelector - # XXX: this test was disabled because lxml behaves inconsistently when - # handling null bytes between different 2.2.x versions, but it may be due - # to differences in libxml2 too. it's also unclear what should be the - # proper behaviour (pablo - 26 oct 2010) - #@libxml2debug - #def test_null_bytes(self): - # hxs = HtmlXPathSelector(text='la\x00la') - # self.assertEqual(hxs.extract(), - # u'la') - # - # xxs = XmlXPathSelector(text='la\x00la') - # self.assertEqual(xxs.extract(), - # u'la') + +class Libxml2DocumentTest(unittest.TestCase): + + def test_caching(self): + r1 = HtmlResponse('http://www.example.com', body='') + r2 = r1.copy() + + doc1 = LxmlDocument(r1) + doc2 = LxmlDocument(r1) + doc3 = LxmlDocument(r2) + + # make sure it's cached + assert doc1 is doc2 + assert doc1 is not doc3 + + # don't leave documents in memory to avoid wrong libxml2 leaks reports + del doc1, doc2, doc3 + + def test_null_char(self): + # make sure bodies with null char ('\x00') don't raise a TypeError exception + self.body_content = 'test problematic \x00 body' + response = TextResponse('http://example.com/catalog/product/blabla-123', + headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content) + LxmlDocument(response)