mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into lxml-formrequest
This commit is contained in:
Daniel Graña
commit
9e10abcc43
|
|
@ -2,13 +2,13 @@ Source: scrapy-SUFFIX
|
|||
Section: python
|
||||
Priority: optional
|
||||
Maintainer: Insophia Team <info@insophia.com>
|
||||
Build-Depends: debhelper (>= 7.0.50), python (>=2.6), python-twisted, python-w3lib
|
||||
Build-Depends: debhelper (>= 7.0.50), python (>=2.6), python-twisted, python-w3lib, python-lxml
|
||||
Standards-Version: 3.8.4
|
||||
Homepage: http://scrapy.org/
|
||||
|
||||
Package: scrapy-SUFFIX
|
||||
Architecture: all
|
||||
Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl, python-w3lib (>= 1.1-r23)
|
||||
Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl, python-w3lib (>= 1.1-r23)
|
||||
Recommends: python-setuptools
|
||||
Conflicts: python-scrapy, scrapy, scrapy-0.11
|
||||
Provides: python-scrapy, scrapy
|
||||
|
|
|
|||
|
|
@ -23,9 +23,23 @@ def setup(app):
|
|||
indextemplate = "pair: %s; reqmeta",
|
||||
)
|
||||
app.add_role('source', source_role)
|
||||
app.add_role('commit', commit_role)
|
||||
app.add_role('rev', rev_role)
|
||||
|
||||
def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
url = 'https://github.com/scrapy/scrapy/blob/master/' + text
|
||||
ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'https://github.com/scrapy/scrapy/commit/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
||||
def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
|
||||
set_classes(options)
|
||||
node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)
|
||||
return [node], []
|
||||
|
|
|
|||
|
|
@ -224,10 +224,14 @@ All the rest
|
|||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
news
|
||||
contributing
|
||||
versioning
|
||||
experimental/index
|
||||
|
||||
:doc:`news`
|
||||
See what has changed in recent Scrapy versions.
|
||||
|
||||
:doc:`contributing`
|
||||
Learn how to contribute to the Scrapy project.
|
||||
|
||||
|
|
|
|||
|
|
@ -20,8 +20,6 @@ Requirements
|
|||
|
||||
* `lxml`_ or `libxml2`_ (if using `libxml2`_, version 2.6.28 or above is highly recommended)
|
||||
|
||||
* `simplejson`_ (not required if using Python 2.6 or above)
|
||||
|
||||
* `pyopenssl <http://pyopenssl.sourceforge.net>`_ (for HTTPS support. Optional,
|
||||
but highly recommended)
|
||||
|
||||
|
|
@ -111,7 +109,7 @@ Debian or Ubuntu (9.04 or older)
|
|||
|
||||
If you're running Debian Linux, run the following command as root::
|
||||
|
||||
apt-get install python-twisted python-libxml2 python-pyopenssl python-simplejson
|
||||
apt-get install python-twisted python-lxmlxml python-pyopenssl
|
||||
|
||||
Then::
|
||||
|
||||
|
|
@ -124,7 +122,7 @@ Arch Linux
|
|||
|
||||
If you are running Arch Linux, run the following command as root::
|
||||
|
||||
pacman -S twisted libxml2 pyopenssl python-simplejson
|
||||
pacman -S twisted python-lxml pyopenssl
|
||||
|
||||
Then::
|
||||
|
||||
|
|
@ -178,7 +176,6 @@ There are two ways to install Scrapy in Windows:
|
|||
.. _lxml: http://codespeak.net/lxml/
|
||||
.. _libxml2: http://xmlsoft.org
|
||||
.. _pywin32: http://sourceforge.net/projects/pywin32/
|
||||
.. _simplejson: http://pypi.python.org/pypi/simplejson/
|
||||
.. _Zope.Interface: http://pypi.python.org/pypi/zope.interface#download
|
||||
.. _this Twisted bug: http://twistedmatrix.com/trac/ticket/3707
|
||||
.. _pip: http://pypi.python.org/pypi/pip
|
||||
|
|
|
|||
|
|
@ -0,0 +1,339 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
0.16 (not yet released)
|
||||
-----------------------
|
||||
|
||||
Scrapy changes:
|
||||
|
||||
- SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`)
|
||||
- StackTraceDump extension: also dump trackref live references (:commit:`fe2ce93`)
|
||||
- added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider
|
||||
- decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that mdule
|
||||
- dropped support for Python 2.5. See http://blog.scrapy.org/scrapy-dropping-support-for-python-25
|
||||
- dropped support for Twisted 2.5
|
||||
- added :setting:`REFERER_ENABLED` setting, to control referer middleware
|
||||
- changed default user agent to: ``Scrapy/VERSION (+http://scrapy.org)``
|
||||
|
||||
Scrapyd changes:
|
||||
|
||||
- New Scrapy API methods (see documentation for details)
|
||||
- ``listjobs.json`` to see pending/running/finished jobs
|
||||
- ``cancel.json`` to cancel pending and running jobs
|
||||
- Items are now stored on disk using feed exports, and accessible through the Scrapyd web interface
|
||||
- Support making Scrapyd listen into a specific IP address (see ``bind_address`` option)
|
||||
|
||||
0.14.2
|
||||
------
|
||||
|
||||
- bumped version to 0.14.2 (:commit:`68c1c98`)
|
||||
- move buffer pointing to start of file before computing checksum. refs #92 (:commit:`6a5bef2`)
|
||||
- Compute image checksum before persisting images. closes #92 (:commit:`9817df1`)
|
||||
- remove leaking references in cached failures (:commit:`673a120`)
|
||||
- fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`)
|
||||
- Merge branch '0.14' of github.com:scrapy/scrapy into 0.14 (:commit:`1627320`)
|
||||
- fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`)
|
||||
- ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`)
|
||||
- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`)
|
||||
- Speed-up libxml2 XPathSelector (:commit:`2fbd662`)
|
||||
- updated versioning doc according to recent changes (:commit:`0a070f5`)
|
||||
- scrapyd: fixed documentation link (:commit:`2b4e4c3`)
|
||||
- extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`)
|
||||
|
||||
0.14.1
|
||||
------
|
||||
|
||||
- extras/makedeb.py: no longer obtaining version from git (:commit:`caffe0e`)
|
||||
- bumped version to 0.14.1 (:commit:`6cb9e1c`)
|
||||
- fixed reference to tutorial directory (:commit:`4b86bd6`)
|
||||
- doc: removed duplicated callback argument from Request.replace() (:commit:`1aeccdd`)
|
||||
- fixed formatting of scrapyd doc (:commit:`8bf19e6`)
|
||||
- Dump stacks for all running threads and fix engine status dumped by StackTraceDump extension (:commit:`14a8e6e`)
|
||||
- added comment about why we disable ssl on boto images upload (:commit:`5223575`)
|
||||
- SSL handshaking hangs when doing too many parallel connections to S3 (:commit:`63d583d`)
|
||||
- change tutorial to follow changes on dmoz site (:commit:`bcb3198`)
|
||||
- Avoid _disconnectedDeferred AttributeError exception in Twisted>=11.1.0 (:commit:`98f3f87`)
|
||||
- allow spider to set autothrottle max concurrency (:commit:`175a4b5`)
|
||||
|
||||
0.14
|
||||
----
|
||||
|
||||
New features and settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Support for `AJAX crawleable urls`_
|
||||
- New persistent scheduler that stores requests on disk, allowing to suspend and resume crawls (:rev:`2737`)
|
||||
- added ``-o`` option to ``scrapy crawl``, a shortcut for dumping scraped items into a file (or standard output using ``-``)
|
||||
- Added support for passing custom settings to Scrapyd ``schedule.json`` api (:rev:`2779`, :rev:`2783`)
|
||||
- New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`)
|
||||
- Add boto 2.0 support for S3 downloader handler (:rev:`2763`)
|
||||
- Added `marshal`_ to formats supported by feed exports (:rev:`2744`)
|
||||
- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`)
|
||||
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
||||
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||
- check the documentation for more details
|
||||
- Added builtin caching DNS resolver (:rev:`2728`)
|
||||
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
||||
- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`)
|
||||
- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`)
|
||||
- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
|
||||
- Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`)
|
||||
- Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`)
|
||||
- Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`)
|
||||
- Improved encoding detection by adding support for HTML5 meta charset declaration (:rev:`2690`)
|
||||
- Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`)
|
||||
- Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`)
|
||||
- Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`)
|
||||
- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`.
|
||||
- Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`)
|
||||
- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`)
|
||||
- Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines.
|
||||
- Added :signal:`spider_error` signal (:rev:`2628`)
|
||||
- Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`)
|
||||
- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
|
||||
- Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`)
|
||||
- Added new DBM HTTP cache storage backend (:rev:`2576`)
|
||||
- Added ``listjobs.json`` API to Scrapyd (:rev:`2571`)
|
||||
- ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`)
|
||||
- Added namespace support to ``xmliter_lxml`` (:rev:`2552`)
|
||||
- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`)
|
||||
- Several improvements to Scrapyd and Link extractors
|
||||
|
||||
Code rearranged and removed
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Merged item passed and item scraped concepts, as they have often proved confusing in the past. This means: (:rev:`2630`)
|
||||
- original item_scraped signal was removed
|
||||
- original item_passed signal was renamed to item_scraped
|
||||
- old log lines ``Scraped Item...`` were removed
|
||||
- old log lines ``Passed Item...`` were renamed to ``Scraped Item...`` lines and downgraded to ``DEBUG`` level
|
||||
- Reduced Scrapy codebase by striping part of Scrapy code into two new libraries:
|
||||
- `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`)
|
||||
- `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`)
|
||||
- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`)
|
||||
- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot
|
||||
- Removed support for default field values in Scrapy items (:rev:`2616`)
|
||||
- Removed experimental crawlspider v2 (:rev:`2632`)
|
||||
- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`)
|
||||
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
|
||||
- Removed deprecated Execution Queue (:rev:`2704`)
|
||||
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
|
||||
- removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`)
|
||||
- Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`)
|
||||
- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept.
|
||||
|
||||
0.12
|
||||
----
|
||||
|
||||
The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
|
||||
|
||||
New features and improvements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273)
|
||||
- Added verbose option to ``scrapy version`` command, useful for bug reports (#298)
|
||||
- HTTP cache now stored by default in the project data dir (#279)
|
||||
- Added project data storage directory (#276, #277)
|
||||
- Documented file structure of Scrapy projects (see command-line tool doc)
|
||||
- New lxml backend for XPath selectors (#147)
|
||||
- Per-spider settings (#245)
|
||||
- Support exit codes to signal errors in Scrapy commands (#248)
|
||||
- Added ``-c`` argument to ``scrapy shell`` command
|
||||
- Made ``libxml2`` optional (#260)
|
||||
- New ``deploy`` command (#261)
|
||||
- Added :setting:`CLOSESPIDER_PAGECOUNT` setting (#253)
|
||||
- Added :setting:`CLOSESPIDER_ERRORCOUNT` setting (#254)
|
||||
|
||||
Scrapyd changes
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
- Scrapyd now uses one process per spider
|
||||
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
|
||||
- A minimal web ui was added, available at http://localhost:6800 by default
|
||||
- There is now a `scrapy server` command to start a Scrapyd server of the current project
|
||||
|
||||
Changes to settings
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware
|
||||
- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire".
|
||||
|
||||
Deprecated/obsoleted functionality
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Deprecated ``runserver`` command in favor of ``server`` command which starts a Scrapyd server. See also: Scrapyd changes
|
||||
- Deprecated ``queue`` command in favor of using Scrapyd ``schedule.json`` API. See also: Scrapyd changes
|
||||
- Removed the !LxmlItemLoader (experimental contrib which never graduated to main contrib)
|
||||
|
||||
0.10
|
||||
----
|
||||
|
||||
The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
|
||||
|
||||
New features and improvements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- New Scrapy service called ``scrapyd`` for deploying Scrapy crawlers in production (#218) (documentation available)
|
||||
- Simplified Images pipeline usage which doesn't require subclassing your own images pipeline now (#217)
|
||||
- Scrapy shell now shows the Scrapy log by default (#206)
|
||||
- Refactored execution queue in a common base code and pluggable backends called "spider queues" (#220)
|
||||
- New persistent spider queue (based on SQLite) (#198), available by default, which allows to start Scrapy in server mode and then schedule spiders to run.
|
||||
- Added documentation for Scrapy command-line tool and all its available sub-commands. (documentation available)
|
||||
- Feed exporters with pluggable backends (#197) (documentation available)
|
||||
- Deferred signals (#193)
|
||||
- Added two new methods to item pipeline open_spider(), close_spider() with deferred support (#195)
|
||||
- Support for overriding default request headers per spider (#181)
|
||||
- Replaced default Spider Manager with one with similar functionality but not depending on Twisted Plugins (#186)
|
||||
- Splitted Debian package into two packages - the library and the service (#187)
|
||||
- Scrapy log refactoring (#188)
|
||||
- New extension for keeping persistent spider contexts among different runs (#203)
|
||||
- Added `dont_redirect` request.meta key for avoiding redirects (#233)
|
||||
- Added `dont_retry` request.meta key for avoiding retries (#234)
|
||||
|
||||
Command-line tool changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199)
|
||||
- there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project
|
||||
- Added `scrapy.bat` script for running more conveniently from Windows
|
||||
- Added bash completion to command-line tool (#210)
|
||||
- Renamed command `start` to `runserver` (#209)
|
||||
|
||||
API changes
|
||||
~~~~~~~~~~~
|
||||
|
||||
- ``url`` and ``body`` attributes of Request objects are now read-only (#230)
|
||||
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
|
||||
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) - see this snippet for a replacement: http://snippets.scrapy.org/snippets/12/
|
||||
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
|
||||
- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
|
||||
- Changes to Scrapy Manager (now called "Crawler"):
|
||||
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
|
||||
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
|
||||
- Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager``
|
||||
- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton.
|
||||
- moved Stats Collector classes: (#204)
|
||||
- ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector``
|
||||
- ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector``
|
||||
- default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201)
|
||||
- changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)``
|
||||
- backwards compatibility kept (with deprecation warning)
|
||||
- moved ``scrapy.core.signals`` module to ``scrapy.signals``
|
||||
- backwards compatibility kept (with deprecation warning)
|
||||
- moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions``
|
||||
- backwards compatibility kept (with deprecation warning)
|
||||
- added ``handles_request()`` class method to ``BaseSpider``
|
||||
- dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead)
|
||||
- dropped ``component`` argument of ``scrapy.log.msg()`` function
|
||||
- dropped ``scrapy.log.log_level`` attribute
|
||||
- Added ``from_settings()`` class methods to Spider Manager, and Item Pipeline Manager
|
||||
|
||||
Changes to settings
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added ``HTTPCACHE_IGNORE_SCHEMES`` setting to ignore certain schemes on !HttpCacheMiddleware (#225)
|
||||
- Added ``SPIDER_QUEUE_CLASS`` setting which defines the spider queue to use (#220)
|
||||
- Added ``KEEP_ALIVE`` setting (#220)
|
||||
- Removed ``SERVICE_QUEUE`` setting (#220)
|
||||
- Removed ``COMMANDS_SETTINGS_MODULE`` setting (#201)
|
||||
- Renamed ``REQUEST_HANDLERS`` to ``DOWNLOAD_HANDLERS`` and make download handlers classes (instead of functions)
|
||||
|
||||
0.9
|
||||
---
|
||||
|
||||
The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
|
||||
|
||||
New features and improvements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Added SMTP-AUTH support to scrapy.mail
|
||||
- New settings added: ``MAIL_USER``, ``MAIL_PASS`` (:rev:`2065` | #149)
|
||||
- Added new scrapy-ctl view command - To view URL in the browser, as seen by Scrapy (:rev:`2039`)
|
||||
- Added web service for controlling Scrapy process (this also deprecates the web console. (:rev:`2053` | #167)
|
||||
- Support for running Scrapy as a service, for production systems (:rev:`1988`, :rev:`2054`, :rev:`2055`, :rev:`2056`, :rev:`2057` | #168)
|
||||
- Added wrapper induction library (documentation only available in source code for now). (:rev:`2011`)
|
||||
- Simplified and improved response encoding support (:rev:`1961`, :rev:`1969`)
|
||||
- Added ``LOG_ENCODING`` setting (:rev:`1956`, documentation available)
|
||||
- Added ``RANDOMIZE_DOWNLOAD_DELAY`` setting (enabled by default) (:rev:`1923`, doc available)
|
||||
- ``MailSender`` is no longer IO-blocking (:rev:`1955` | #146)
|
||||
- Linkextractors and new Crawlspider now handle relative base tag urls (:rev:`1960` | #148)
|
||||
- Several improvements to Item Loaders and processors (:rev:`2022`, :rev:`2023`, :rev:`2024`, :rev:`2025`, :rev:`2026`, :rev:`2027`, :rev:`2028`, :rev:`2029`, :rev:`2030`)
|
||||
- Added support for adding variables to telnet console (:rev:`2047` | #165)
|
||||
- Support for requests without callbacks (:rev:`2050` | #166)
|
||||
|
||||
API changes
|
||||
~~~~~~~~~~~
|
||||
|
||||
- Change ``Spider.domain_name`` to ``Spider.name`` (SEP-012, :rev:`1975`)
|
||||
- ``Response.encoding`` is now the detected encoding (:rev:`1961`)
|
||||
- ``HttpErrorMiddleware`` now returns None or raises an exception (:rev:`2006` | #157)
|
||||
- ``scrapy.command`` modules relocation (:rev:`2035`, :rev:`2036`, :rev:`2037`)
|
||||
- Added ``ExecutionQueue`` for feeding spiders to scrape (:rev:`2034`)
|
||||
- Removed ``ExecutionEngine`` singleton (:rev:`2039`)
|
||||
- Ported ``S3ImagesStore`` (images pipeline) to use boto and threads (:rev:`2033`)
|
||||
- Moved module: ``scrapy.management.telnet`` to ``scrapy.telnet`` (:rev:`2047`)
|
||||
|
||||
Changes to default settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Changed default ``SCHEDULER_ORDER`` to ``DFO`` (:rev:`1939`)
|
||||
|
||||
0.8
|
||||
---
|
||||
|
||||
The numbers like #NNN reference tickets in the old issue tracker (Trac) which is no longer available.
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Added DEFAULT_RESPONSE_ENCODING setting (:rev:`1809`)
|
||||
- Added ``dont_click`` argument to ``FormRequest.from_response()`` method (:rev:`1813`, :rev:`1816`)
|
||||
- Added ``clickdata`` argument to ``FormRequest.from_response()`` method (:rev:`1802`, :rev:`1803`)
|
||||
- Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`)
|
||||
- Offiste spider middleware now logs messages when filtering out requests (:rev:`1841`)
|
||||
|
||||
Backwards-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
- Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`)
|
||||
- Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`)
|
||||
- Removed deprecated ``scrapy.xpath`` module - use ``scrapy.selector`` instead. (:rev:`1836`)
|
||||
- Removed deprecated ``core.signals.domain_open`` signal - use ``core.signals.domain_opened`` instead (:rev:`1822`)
|
||||
- ``log.msg()`` now receives a ``spider`` argument (:rev:`1822`)
|
||||
- Old domain argument has been deprecated and will be removed in 0.9. For spiders, you should always use the ``spider`` argument and pass spider references. If you really want to pass a string, use the ``component`` argument instead.
|
||||
- Changed core signals ``domain_opened``, ``domain_closed``, ``domain_idle``
|
||||
- Changed Item pipeline to use spiders instead of domains
|
||||
- The ``domain`` argument of ``process_item()`` item pipeline method was changed to ``spider``, the new signature is: ``process_item(spider, item)`` (:rev:`1827` | #105)
|
||||
- To quickly port your code (to work with Scrapy 0.8) just use ``spider.domain_name`` where you previously used ``domain``.
|
||||
- Changed Stats API to use spiders instead of domains (:rev:`1849` | #113)
|
||||
- ``StatsCollector`` was changed to receive spider references (instead of domains) in its methods (``set_value``, ``inc_value``, etc).
|
||||
- added ``StatsCollector.iter_spider_stats()`` method
|
||||
- removed ``StatsCollector.list_domains()`` method
|
||||
- Also, Stats signals were renamed and now pass around spider references (instead of domains). Here's a summary of the changes:
|
||||
- To quickly port your code (to work with Scrapy 0.8) just use ``spider.domain_name`` where you previously used ``domain``. ``spider_stats`` contains exactly the same data as ``domain_stats``.
|
||||
- ``CloseDomain`` extension moved to ``scrapy.contrib.closespider.CloseSpider`` (:rev:`1833`)
|
||||
- Its settings were also renamed:
|
||||
- ``CLOSEDOMAIN_TIMEOUT`` to ``CLOSESPIDER_TIMEOUT``
|
||||
- ``CLOSEDOMAIN_ITEMCOUNT`` to ``CLOSESPIDER_ITEMCOUNT``
|
||||
- Removed deprecated ``SCRAPYSETTINGS_MODULE`` environment variable - use ``SCRAPY_SETTINGS_MODULE`` instead (:rev:`1840`)
|
||||
- Renamed setting: ``REQUESTS_PER_DOMAIN`` to ``CONCURRENT_REQUESTS_PER_SPIDER`` (:rev:`1830`, :rev:`1844`)
|
||||
- Renamed setting: ``CONCURRENT_DOMAINS`` to ``CONCURRENT_SPIDERS`` (:rev:`1830`)
|
||||
- Refactored HTTP Cache middleware
|
||||
- HTTP Cache middleware has been heavilty refactored, retaining the same functionality except for the domain sectorization which was removed. (:rev:`1843` )
|
||||
- Renamed exception: ``DontCloseDomain`` to ``DontCloseSpider`` (:rev:`1859` | #120)
|
||||
- Renamed extension: ``DelayedCloseDomain`` to ``SpiderCloseDelay`` (:rev:`1861` | #121)
|
||||
- Removed obsolete ``scrapy.utils.markup.remove_escape_chars`` function - use ``scrapy.utils.markup.replace_escape_chars`` instead (:rev:`1865`)
|
||||
|
||||
0.7
|
||||
---
|
||||
|
||||
First release of Scrapy.
|
||||
|
||||
|
||||
.. _AJAX crawleable urls: http://code.google.com/web/ajaxcrawling/docs/getting-started.html
|
||||
.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
.. _w3lib: http://https://github.com/scrapy/w3lib
|
||||
.. _scrapely: https://github.com/scrapy/scrapely
|
||||
.. _marshal: http://docs.python.org/library/marshal.html
|
||||
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
|
||||
|
|
@ -64,7 +64,7 @@ single Python class that defines one or more of the following methods:
|
|||
This method is called for each response that goes through the spider
|
||||
middleware and into the spider, for processing.
|
||||
|
||||
:meth:`process_spider_input` should return ``None`` or raise and
|
||||
:meth:`process_spider_input` should return ``None`` or raise an
|
||||
exception.
|
||||
|
||||
If it returns ``None``, Scrapy will continue processing this response,
|
||||
|
|
|
|||
|
|
@ -183,8 +183,9 @@ class Scraper(object):
|
|||
errors that got propagated thru here)
|
||||
"""
|
||||
if spider_failure is download_failure:
|
||||
log.msg("Error downloading %s: %s" % \
|
||||
(request, spider_failure.getErrorMessage()), log.ERROR, spider=spider)
|
||||
errmsg = spider_failure.getErrorMessage()
|
||||
if errmsg:
|
||||
log.msg("Error downloading %s: %s" % (request, errmsg), log.ERROR, spider=spider)
|
||||
return
|
||||
return spider_failure
|
||||
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ elif settings['SELECTORS_BACKEND'] == 'dummy':
|
|||
from scrapy.selector.dummysel import *
|
||||
else:
|
||||
try:
|
||||
import libxml2
|
||||
import lxml
|
||||
except ImportError:
|
||||
try:
|
||||
import lxml
|
||||
import libxml2
|
||||
except ImportError:
|
||||
from scrapy.selector.dummysel import *
|
||||
else:
|
||||
from scrapy.selector.lxmlsel import *
|
||||
from scrapy.selector.libxml2sel import *
|
||||
else:
|
||||
from scrapy.selector.libxml2sel import *
|
||||
from scrapy.selector.lxmlsel import *
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
"""
|
||||
This module contains a simple class (Libxml2Document) which provides cache and
|
||||
garbage collection to libxml2 documents (xmlDoc).
|
||||
"""
|
||||
|
||||
import weakref
|
||||
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from .factories import xmlDoc_from_html
|
||||
|
||||
class Libxml2Document(object_ref):
|
||||
|
||||
cache = weakref.WeakKeyDictionary()
|
||||
__slots__ = ['xmlDoc', 'xpathContext', '__weakref__']
|
||||
|
||||
def __new__(cls, response, factory=xmlDoc_from_html):
|
||||
cache = cls.cache.setdefault(response, {})
|
||||
if factory not in cache:
|
||||
obj = object_ref.__new__(cls)
|
||||
obj.xmlDoc = factory(response)
|
||||
obj.xpathContext = obj.xmlDoc.xpathNewContext()
|
||||
cache[factory] = obj
|
||||
return cache[factory]
|
||||
|
||||
def __del__(self):
|
||||
# we must call both cleanup functions, so we try/except all exceptions
|
||||
# to make sure one doesn't prevent the other from being called
|
||||
# this call sometimes raises a "NoneType is not callable" TypeError
|
||||
# so the try/except block silences them
|
||||
try:
|
||||
self.xmlDoc.freeDoc()
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
self.xpathContext.xpathFreeContext()
|
||||
except:
|
||||
pass
|
||||
|
||||
def __str__(self):
|
||||
return "<Libxml2Document %s>" % self.xmlDoc.name
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
"""
|
||||
This module provides functions for generating libxml2 documents (xmlDoc).
|
||||
|
||||
Constructors must receive a Response object and return a xmlDoc object.
|
||||
"""
|
||||
|
||||
import libxml2
|
||||
|
||||
xml_parser_options = libxml2.XML_PARSE_RECOVER + \
|
||||
libxml2.XML_PARSE_NOERROR + \
|
||||
libxml2.XML_PARSE_NOWARNING
|
||||
|
||||
html_parser_options = libxml2.HTML_PARSE_RECOVER + \
|
||||
libxml2.HTML_PARSE_NOERROR + \
|
||||
libxml2.HTML_PARSE_NOWARNING
|
||||
|
||||
utf8_encodings = set(('utf-8', 'UTF-8', 'utf8', 'UTF8'))
|
||||
|
||||
def body_as_utf8(response):
|
||||
if response.encoding in utf8_encodings:
|
||||
return response.body
|
||||
else:
|
||||
return response.body_as_unicode().encode('utf-8')
|
||||
|
||||
def xmlDoc_from_html(response):
|
||||
"""Return libxml2 doc for HTMLs"""
|
||||
utf8body = body_as_utf8(response) or ' '
|
||||
try:
|
||||
lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \
|
||||
html_parser_options)
|
||||
except TypeError: # libxml2 doesn't parse text with null bytes
|
||||
lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \
|
||||
'utf-8', html_parser_options)
|
||||
return lxdoc
|
||||
|
||||
def xmlDoc_from_xml(response):
|
||||
"""Return libxml2 doc for XMLs"""
|
||||
utf8body = body_as_utf8(response) or ' '
|
||||
try:
|
||||
lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \
|
||||
xml_parser_options)
|
||||
except TypeError: # libxml2 doesn't parse text with null bytes
|
||||
lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \
|
||||
'utf-8', xml_parser_options)
|
||||
return lxdoc
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
"""
|
||||
This module contains a simple class (Libxml2Document) which provides cache and
|
||||
garbage collection to libxml2 documents (xmlDoc).
|
||||
"""
|
||||
|
||||
import weakref
|
||||
import libxml2
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
xml_parser_options = libxml2.XML_PARSE_RECOVER + \
|
||||
libxml2.XML_PARSE_NOERROR + \
|
||||
libxml2.XML_PARSE_NOWARNING
|
||||
|
||||
html_parser_options = libxml2.HTML_PARSE_RECOVER + \
|
||||
libxml2.HTML_PARSE_NOERROR + \
|
||||
libxml2.HTML_PARSE_NOWARNING
|
||||
|
||||
|
||||
_UTF8_ENCODINGS = set(('utf-8', 'UTF-8', 'utf8', 'UTF8'))
|
||||
def _body_as_utf8(response):
|
||||
if response.encoding in _UTF8_ENCODINGS:
|
||||
return response.body
|
||||
else:
|
||||
return response.body_as_unicode().encode('utf-8')
|
||||
|
||||
|
||||
def xmlDoc_from_html(response):
|
||||
"""Return libxml2 doc for HTMLs"""
|
||||
utf8body = _body_as_utf8(response) or ' '
|
||||
try:
|
||||
lxdoc = libxml2.htmlReadDoc(utf8body, response.url, 'utf-8', \
|
||||
html_parser_options)
|
||||
except TypeError: # libxml2 doesn't parse text with null bytes
|
||||
lxdoc = libxml2.htmlReadDoc(utf8body.replace("\x00", ""), response.url, \
|
||||
'utf-8', html_parser_options)
|
||||
return lxdoc
|
||||
|
||||
|
||||
def xmlDoc_from_xml(response):
|
||||
"""Return libxml2 doc for XMLs"""
|
||||
utf8body = _body_as_utf8(response) or ' '
|
||||
try:
|
||||
lxdoc = libxml2.readDoc(utf8body, response.url, 'utf-8', \
|
||||
xml_parser_options)
|
||||
except TypeError: # libxml2 doesn't parse text with null bytes
|
||||
lxdoc = libxml2.readDoc(utf8body.replace("\x00", ""), response.url, \
|
||||
'utf-8', xml_parser_options)
|
||||
return lxdoc
|
||||
|
||||
|
||||
class Libxml2Document(object_ref):
|
||||
|
||||
cache = weakref.WeakKeyDictionary()
|
||||
__slots__ = ['xmlDoc', 'xpathContext', '__weakref__']
|
||||
|
||||
def __new__(cls, response, factory=xmlDoc_from_html):
|
||||
cache = cls.cache.setdefault(response, {})
|
||||
if factory not in cache:
|
||||
obj = object_ref.__new__(cls)
|
||||
obj.xmlDoc = factory(response)
|
||||
obj.xpathContext = obj.xmlDoc.xpathNewContext()
|
||||
cache[factory] = obj
|
||||
return cache[factory]
|
||||
|
||||
def __del__(self):
|
||||
# we must call both cleanup functions, so we try/except all exceptions
|
||||
# to make sure one doesn't prevent the other from being called
|
||||
# this call sometimes raises a "NoneType is not callable" TypeError
|
||||
# so the try/except block silences them
|
||||
try:
|
||||
self.xmlDoc.freeDoc()
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
self.xpathContext.xpathFreeContext()
|
||||
except:
|
||||
pass
|
||||
|
||||
def __str__(self):
|
||||
return "<Libxml2Document %s>" % self.xmlDoc.name
|
||||
|
|
@ -9,8 +9,7 @@ from scrapy.utils.python import unicode_to_str
|
|||
from scrapy.utils.misc import extract_regex
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.utils.decorator import deprecated
|
||||
from .factories import xmlDoc_from_html, xmlDoc_from_xml
|
||||
from .document import Libxml2Document
|
||||
from .libxml2document import Libxml2Document, xmlDoc_from_html, xmlDoc_from_xml
|
||||
from .list import XPathSelectorList
|
||||
|
||||
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
|
||||
|
|
@ -37,6 +36,7 @@ class XPathSelector(object_ref):
|
|||
def select(self, xpath):
|
||||
if hasattr(self.xmlNode, 'xpathEval'):
|
||||
self.doc.xpathContext.setContextNode(self.xmlNode)
|
||||
xpath = unicode_to_str(xpath, 'utf-8')
|
||||
try:
|
||||
xpath_result = self.doc.xpathContext.xpathEval(xpath)
|
||||
except libxml2.xpathError:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
"""
|
||||
This module contains a simple class (LxmlDocument) which provides cache and
|
||||
garbage collection to lxml element tree documents.
|
||||
"""
|
||||
|
||||
import weakref
|
||||
from lxml import etree
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
|
||||
def _factory(response, parser_cls):
|
||||
url = response.url
|
||||
body = response.body_as_unicode().strip().encode('utf8') or '<html/>'
|
||||
parser = parser_cls(recover=True, encoding='utf8')
|
||||
return etree.fromstring(body, parser=parser, base_url=url)
|
||||
|
||||
|
||||
class LxmlDocument(object_ref):
|
||||
|
||||
cache = weakref.WeakKeyDictionary()
|
||||
__slots__ = ['xmlDoc', 'xpathContext', '__weakref__']
|
||||
|
||||
def __new__(cls, response, parser=etree.HTMLParser):
|
||||
cache = cls.cache.setdefault(response, {})
|
||||
if parser not in cache:
|
||||
obj = object_ref.__new__(cls)
|
||||
cache[parser] = _factory(response, parser)
|
||||
return cache[parser]
|
||||
|
||||
def __str__(self):
|
||||
return "<LxmlDocument %s>" % self.root.tag
|
||||
|
|
@ -9,53 +9,48 @@ from scrapy.utils.trackref import object_ref
|
|||
from scrapy.utils.python import unicode_to_str
|
||||
from scrapy.utils.decorator import deprecated
|
||||
from scrapy.http import TextResponse
|
||||
from .lxmldocument import LxmlDocument
|
||||
from .list import XPathSelectorList
|
||||
|
||||
|
||||
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', \
|
||||
'XPathSelectorList']
|
||||
'XPathSelectorList']
|
||||
|
||||
|
||||
class XPathSelector(object_ref):
|
||||
|
||||
__slots__ = ['response', 'text', 'expr', 'namespaces', '_root', '_xpathev', \
|
||||
'__weakref__']
|
||||
__slots__ = ['response', 'text', 'namespaces', '_expr', '_root', '__weakref__']
|
||||
_parser = etree.HTMLParser
|
||||
_tostring_method = 'html'
|
||||
|
||||
def __init__(self, response=None, text=None, root=None, expr=None, namespaces=None):
|
||||
if text:
|
||||
self.response = TextResponse(url='about:blank', \
|
||||
def __init__(self, response=None, text=None, namespaces=None, _root=None, _expr=None):
|
||||
if text is not None:
|
||||
response = TextResponse(url='about:blank', \
|
||||
body=unicode_to_str(text, 'utf-8'), encoding='utf-8')
|
||||
else:
|
||||
self.response = response
|
||||
self._root = root
|
||||
self._xpathev = None
|
||||
if response is not None:
|
||||
_root = LxmlDocument(response, self._parser)
|
||||
|
||||
self.namespaces = namespaces
|
||||
self.expr = expr
|
||||
|
||||
@property
|
||||
def root(self):
|
||||
if self._root is None:
|
||||
parser = self._parser(encoding=self.response.encoding, recover=True)
|
||||
self._root = etree.fromstring(self.response.body, parser=parser, \
|
||||
base_url=self.response.url)
|
||||
return self._root
|
||||
|
||||
@property
|
||||
def xpathev(self):
|
||||
if self._xpathev is None:
|
||||
self._xpathev = etree.XPathEvaluator(self.root, namespaces=self.namespaces)
|
||||
return self._xpathev
|
||||
self.response = response
|
||||
self._root = _root
|
||||
self._expr = _expr
|
||||
|
||||
def select(self, xpath):
|
||||
try:
|
||||
result = self.xpathev(xpath)
|
||||
xpathev = self._root.xpath
|
||||
except AttributeError:
|
||||
return XPathSelectorList([])
|
||||
|
||||
try:
|
||||
result = xpathev(xpath, namespaces=self.namespaces)
|
||||
except etree.XPathError:
|
||||
raise ValueError("Invalid XPath: %s" % xpath)
|
||||
if hasattr(result, '__iter__'):
|
||||
result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \
|
||||
for x in result]
|
||||
else:
|
||||
result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)]
|
||||
|
||||
if type(result) is not list:
|
||||
result = [result]
|
||||
|
||||
result = [self.__class__(_root=x, _expr=xpath, namespaces=self.namespaces)
|
||||
for x in result]
|
||||
return XPathSelectorList(result)
|
||||
|
||||
def re(self, regex):
|
||||
|
|
@ -63,10 +58,15 @@ class XPathSelector(object_ref):
|
|||
|
||||
def extract(self):
|
||||
try:
|
||||
return etree.tostring(self.root, method=self._tostring_method, \
|
||||
return etree.tostring(self._root, method=self._tostring_method, \
|
||||
encoding=unicode)
|
||||
except (AttributeError, TypeError):
|
||||
return unicode(self.root)
|
||||
if self._root is True:
|
||||
return u'1'
|
||||
elif self._root is False:
|
||||
return u'0'
|
||||
else:
|
||||
return unicode(self._root)
|
||||
|
||||
def register_namespace(self, prefix, uri):
|
||||
if self.namespaces is None:
|
||||
|
|
@ -78,7 +78,7 @@ class XPathSelector(object_ref):
|
|||
|
||||
def __str__(self):
|
||||
data = repr(self.extract()[:40])
|
||||
return "<%s xpath=%r data=%s>" % (type(self).__name__, self.expr, data)
|
||||
return "<%s xpath=%r data=%s>" % (type(self).__name__, self._expr, data)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
|
|
|
|||
|
|
@ -43,6 +43,12 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12'])
|
||||
|
||||
def test_selector_unicode_query(self):
|
||||
body = u"<p><input name='\xa9' value='1'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body, encoding='utf8')
|
||||
xpath = self.hxs_cls(response)
|
||||
self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1'])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_same_type(self):
|
||||
"""Test XPathSelector returning the same type in x() method"""
|
||||
|
|
@ -57,12 +63,8 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>"
|
||||
response = TextResponse(url="http://example.com", body=body)
|
||||
xs = self.hxs_cls(response)
|
||||
true = xs.select("//input[@name='a']/@name='a'").extract()[0]
|
||||
false = xs.select("//input[@name='a']/@name='n'").extract()[0]
|
||||
|
||||
# the actual result depends on the backend used
|
||||
assert true in [u'1', u'True'], true
|
||||
assert false in [u'0', u'False'], false
|
||||
self.assertEquals(xs.select("//input[@name='a']/@name='a'").extract(), [u'1'])
|
||||
self.assertEquals(xs.select("//input[@name='a']/@name='n'").extract(), [u'0'])
|
||||
|
||||
@libxml2debug
|
||||
def test_selector_xml_html(self):
|
||||
|
|
@ -232,9 +234,55 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
|
||||
@libxml2debug
|
||||
def test_empty_bodies(self):
|
||||
# shouldn't raise errors
|
||||
r1 = TextResponse('http://www.example.com', body='')
|
||||
self.hxs_cls(r1) # shouldn't raise error
|
||||
self.xxs_cls(r1) # shouldn't raise error
|
||||
self.hxs_cls(r1).select('//text()').extract()
|
||||
self.xxs_cls(r1).select('//text()').extract()
|
||||
|
||||
@libxml2debug
|
||||
def test_null_bytes(self):
|
||||
# shouldn't raise errors
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body='<root>pre\x00post</root>', \
|
||||
encoding='utf-8')
|
||||
self.hxs_cls(r1).select('//text()').extract()
|
||||
self.xxs_cls(r1).select('//text()').extract()
|
||||
|
||||
@libxml2debug
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body='<html><p>an Jos\xe9 de</p><html>', \
|
||||
encoding='utf-8')
|
||||
self.hxs_cls(r1).select('//text()').extract()
|
||||
self.xxs_cls(r1).select('//text()').extract()
|
||||
|
||||
@libxml2debug
|
||||
def test_select_on_unevaluable_nodes(self):
|
||||
r = self.hxs_cls(text=u'<span class="big">some text</span>')
|
||||
# Text node
|
||||
x1 = r.select('//text()')
|
||||
self.assertEquals(x1.extract(), [u'some text'])
|
||||
self.assertEquals(x1.select('.//b').extract(), [])
|
||||
# Tag attribute
|
||||
x1 = r.select('//span/@class')
|
||||
self.assertEquals(x1.extract(), [u'big'])
|
||||
self.assertEquals(x1.select('.//text()').extract(), [])
|
||||
|
||||
@libxml2debug
|
||||
def test_select_on_text_nodes(self):
|
||||
# FIXME: can't get this to work with lxml backend
|
||||
r = self.hxs_cls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
|
||||
x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
|
||||
self.assertEquals(x1.extract(), [u'opt1'])
|
||||
|
||||
x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
|
||||
self.assertEquals(x1.extract(), [u'<b>Options:</b>'])
|
||||
|
||||
x1 = r.select("//div/descendant::text()")
|
||||
x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]")
|
||||
self.assertEquals(x2.extract(), [u'<b>Options:</b>'])
|
||||
test_select_on_text_nodes.skip = True
|
||||
|
||||
@libxml2debug
|
||||
def test_weakref_slots(self):
|
||||
|
|
@ -244,4 +292,3 @@ class XPathSelectorTestCase(unittest.TestCase):
|
|||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,11 @@ import unittest
|
|||
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
|
||||
from scrapy.selector.libxml2sel import XmlXPathSelector, HtmlXPathSelector, \
|
||||
XPathSelector
|
||||
from scrapy.selector.document import Libxml2Document
|
||||
from scrapy.selector.libxml2document import Libxml2Document
|
||||
from scrapy.utils.test import libxml2debug
|
||||
from scrapy.tests import test_selector
|
||||
|
||||
|
||||
class Libxml2XPathSelectorTestCase(test_selector.XPathSelectorTestCase):
|
||||
|
||||
xs_cls = XPathSelector
|
||||
|
|
|
|||
|
|
@ -2,35 +2,40 @@
|
|||
Selectors tests, specific for lxml backend
|
||||
"""
|
||||
|
||||
from scrapy.http import TextResponse, XmlResponse
|
||||
has_lxml = True
|
||||
try:
|
||||
from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, \
|
||||
XPathSelector
|
||||
except ImportError:
|
||||
has_lxml = False
|
||||
from scrapy.utils.test import libxml2debug
|
||||
import unittest
|
||||
from scrapy.tests import test_selector
|
||||
from scrapy.http import TextResponse, HtmlResponse
|
||||
from scrapy.selector.lxmldocument import LxmlDocument
|
||||
from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
|
||||
|
||||
|
||||
class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase):
|
||||
|
||||
if has_lxml:
|
||||
xs_cls = XPathSelector
|
||||
hxs_cls = HtmlXPathSelector
|
||||
xxs_cls = XmlXPathSelector
|
||||
else:
|
||||
skip = "lxml not available"
|
||||
xs_cls = XPathSelector
|
||||
hxs_cls = HtmlXPathSelector
|
||||
xxs_cls = XmlXPathSelector
|
||||
|
||||
# XXX: this test was disabled because lxml behaves inconsistently when
|
||||
# handling null bytes between different 2.2.x versions, but it may be due
|
||||
# to differences in libxml2 too. it's also unclear what should be the
|
||||
# proper behaviour (pablo - 26 oct 2010)
|
||||
#@libxml2debug
|
||||
#def test_null_bytes(self):
|
||||
# hxs = HtmlXPathSelector(text='<root>la\x00la</root>')
|
||||
# self.assertEqual(hxs.extract(),
|
||||
# u'<html><body><root>la</root></body></html>')
|
||||
#
|
||||
# xxs = XmlXPathSelector(text='<root>la\x00la</root>')
|
||||
# self.assertEqual(xxs.extract(),
|
||||
# u'<root>la</root>')
|
||||
|
||||
class Libxml2DocumentTest(unittest.TestCase):
|
||||
|
||||
def test_caching(self):
|
||||
r1 = HtmlResponse('http://www.example.com', body='<html><head></head><body></body></html>')
|
||||
r2 = r1.copy()
|
||||
|
||||
doc1 = LxmlDocument(r1)
|
||||
doc2 = LxmlDocument(r1)
|
||||
doc3 = LxmlDocument(r2)
|
||||
|
||||
# make sure it's cached
|
||||
assert doc1 is doc2
|
||||
assert doc1 is not doc3
|
||||
|
||||
# don't leave documents in memory to avoid wrong libxml2 leaks reports
|
||||
del doc1, doc2, doc3
|
||||
|
||||
def test_null_char(self):
|
||||
# make sure bodies with null char ('\x00') don't raise a TypeError exception
|
||||
self.body_content = 'test problematic \x00 body'
|
||||
response = TextResponse('http://example.com/catalog/product/blabla-123',
|
||||
headers={'Content-Type': 'text/plain; charset=utf-8'}, body=self.body_content)
|
||||
LxmlDocument(response)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from scrapy.http import Request, Response
|
|||
class SpiderReferencer(object):
|
||||
"""Class to serialize (and deserialize) objects (typically dicts)
|
||||
containing references to running spiders (ie. Spider objects). This is
|
||||
required because simplejson fails to serialize dicts containing
|
||||
required because json library fails to serialize dicts containing
|
||||
non-primitive types as keys, even when you override
|
||||
ScrapyJSONEncoder.default() with a custom encoding mechanism.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Helper functinos for working with signals"""
|
||||
"""Helper functions for working with signals"""
|
||||
|
||||
from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
|
|
|||
8
setup.py
8
setup.py
|
|
@ -120,12 +120,6 @@ try:
|
|||
except ImportError:
|
||||
from distutils.core import setup
|
||||
else:
|
||||
setup_args['install_requires'] = ['Twisted>=8.0', 'w3lib>=1.1', 'pyOpenSSL']
|
||||
if sys.version_info < (2, 6):
|
||||
setup_args['install_requires'] += ['simplejson']
|
||||
try:
|
||||
import libxml2
|
||||
except ImportError:
|
||||
setup_args['install_requires'] += ['lxml']
|
||||
setup_args['install_requires'] = ['Twisted>=8.0', 'w3lib>=1.1', 'lxml', 'pyOpenSSL']
|
||||
|
||||
setup(**setup_args)
|
||||
|
|
|
|||
Loading…
Reference in New Issue