diff --git a/Makefile b/Makefile deleted file mode 100644 index 78fb5bfec..000000000 --- a/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -VERSION ?= $(shell python -c "import scrapy; print '.'.join(map(str, scrapy.version_info[:2]))") -DEBREV ?= 1 -HGREV ?= $(shell hg log -l1 --template '{rev}') -FULL_VERSION ?= $(VERSION)~r$(HGREV) -BUILDBASE ?= $(shell pwd)/build/deb -BUILDDIR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig -BUILDTAR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig.tar.gz - -help: - @echo 'Available targets:' - @echo ' deb-binary - build debian binary package' - @echo ' deb-source - build debian source package' - @echo ' deb-all - build source and binary debian packages' - @echo ' tarball - build source tarball' - @echo ' sign - sign release' - -deb-binary: deb-prepare - cd $(BUILDDIR); debuild -i -us -uc -b - -deb-source: deb-prepare - cd $(BUILDDIR); debuild -i -us -uc -S - -deb-all: deb-prepare - cd $(BUILDDIR); debuild -i -us -uc - -deb-prepare: - @if [ -d $(BUILDBASE) ]; then \ - rm -rf $(BUILDBASE); \ - fi; - mkdir -p $(BUILDBASE) - hg archive -t tgz -X .hgtags $(BUILDTAR) - tar zxf $(BUILDTAR) -C $(BUILDBASE) - rm -f $(BUILDDIR)/Makefile # to avoid confusing dh_auto_build - cp -r debian $(BUILDDIR) - cd $(BUILDDIR); debchange -m -D unstable --force-distribution -v $(FULL_VERSION)-$(DEBREV) "Automatic build" - -tarball: - hg purge --all - python setup.py sdist - -sign: - md5sum dist/Scrapy-* > dist/MD5SUMS - sha1sum dist/Scrapy-* > dist/SHA1SUMS - gpg -ba dist/MD5SUMS - gpg -ba dist/SHA1SUMS diff --git a/debian/changelog b/debian/changelog index b1f6b3d94..a01ddd7bb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -scrapy (0.9~r2000-1) unstable; urgency=low +scrapy (0.10) unstable; urgency=low * Initial release. diff --git a/debian/conf/environment b/debian/conf/environment deleted file mode 100644 index bea783073..000000000 --- a/debian/conf/environment +++ /dev/null @@ -1,3 +0,0 @@ -# Environment variables to use in Scrapy service - -#export PYTHONPATH=/var/lib/scrapy/myproject diff --git a/debian/control b/debian/control index 54cf6a236..a6a1788c0 100644 --- a/debian/control +++ b/debian/control @@ -2,17 +2,22 @@ Source: scrapy Section: python Priority: optional Maintainer: Insophia Team -Build-Depends: debhelper (>= 7.0.50), python-twisted -Build-Depends-Indep: python-support, python, python-sphinx, libjs-jquery +Build-Depends: debhelper (>= 7.0.50), python (>=2.5), python-twisted Standards-Version: 3.8.4 Homepage: http://scrapy.org/ Package: scrapy Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, python-libxml2, python-twisted, python-openssl -Provides: ${python:Provides} -Description: Python web scraping and crawling framework +Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl +Description: Python web crawling and scraping framework Scrapy is a fast high-level screen scraping and web crawling framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. + +Package: scrapy-service +Architecture: all +Depends: scrapy +Description: Scrapy Service + This package provides support for running Scrapy as a system service, + controlled through an upstart script. diff --git a/debian/copyright b/debian/copyright index 452544013..71d3765e6 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,4 +1,4 @@ -This package was debianized by Insophia . +This package was debianized by the Insophia . It was downloaded from http://scrapy.org diff --git a/debian/pycompat b/debian/pycompat deleted file mode 100644 index 0cfbf0888..000000000 --- a/debian/pycompat +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/debian/pyversions b/debian/pyversions index 3ec13be1a..b3dc41ebc 100644 --- a/debian/pyversions +++ b/debian/pyversions @@ -1 +1 @@ -2.5-2.6 +2.5- diff --git a/debian/rules b/debian/rules index 0e30f3a98..13d85dc9c 100755 --- a/debian/rules +++ b/debian/rules @@ -1,13 +1,12 @@ #!/usr/bin/make -f # -*- makefile -*- -export DH_ALWAYS_EXCLUDE=license.txt:_sources/:.buildinfo - %: dh $@ override_dh_auto_install: dh_auto_install - mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl - mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-ws.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ws - mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-sqs.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-sqs + mkdir -p $(CURDIR)/debian/scrapy/usr/bin + mv $(CURDIR)/debian/tmp/usr/bin/scrapy-ctl.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl + mv $(CURDIR)/debian/tmp/usr/bin/scrapy-ws.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ws + mv $(CURDIR)/debian/tmp/usr/bin/scrapy-sqs.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-sqs diff --git a/debian/scrapy-service.default b/debian/scrapy-service.default new file mode 100644 index 000000000..59852048d --- /dev/null +++ b/debian/scrapy-service.default @@ -0,0 +1,4 @@ +# Defaults for Scrapy service + +export PYTHONPATH=/etc/scrapy +export SCRAPY_SETTINGS_MODULE=service_conf diff --git a/debian/scrapy.dirs b/debian/scrapy-service.dirs similarity index 54% rename from debian/scrapy.dirs rename to debian/scrapy-service.dirs index b3c079630..a9eaa09f0 100644 --- a/debian/scrapy.dirs +++ b/debian/scrapy-service.dirs @@ -1,4 +1,2 @@ -usr/bin -usr/share/scrapy var/lib/scrapy var/log/scrapy diff --git a/debian/scrapy-service.install b/debian/scrapy-service.install new file mode 100644 index 000000000..7ee823edc --- /dev/null +++ b/debian/scrapy-service.install @@ -0,0 +1,2 @@ +debian/service_conf.py etc/scrapy +extras/scrapy.tac usr/share/scrapy diff --git a/debian/scrapy-service.lintian-overrides b/debian/scrapy-service.lintian-overrides new file mode 100644 index 000000000..48868e9a7 --- /dev/null +++ b/debian/scrapy-service.lintian-overrides @@ -0,0 +1,2 @@ +new-package-should-close-itp-bug +script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapy-service diff --git a/debian/scrapy-service.postinst b/debian/scrapy-service.postinst new file mode 100644 index 000000000..9dc708fff --- /dev/null +++ b/debian/scrapy-service.postinst @@ -0,0 +1,34 @@ +#!/bin/sh + +set -e + +case "$1" in + configure) + # Create user to run the service as + if [ -z "`id -u scrapy 2> /dev/null`" ]; then + adduser --system --home /var/lib/scrapy --gecos "scrapy" \ + --no-create-home --disabled-password \ + --quiet scrapy || true + fi + if [ ! -d /var/run/scrapy ]; then + mkdir /var/run/scrapy + chown scrapy:nogroup /var/run/scrapy + fi + + chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# + +exit 0 + + diff --git a/debian/scrapy.postrm b/debian/scrapy-service.postrm similarity index 74% rename from debian/scrapy.postrm rename to debian/scrapy-service.postrm index b6a3bf880..b16d0c2b6 100644 --- a/debian/scrapy.postrm +++ b/debian/scrapy-service.postrm @@ -1,4 +1,5 @@ #!/bin/sh + set -e if [ purge = "$1" ]; then @@ -10,9 +11,6 @@ if [ purge = "$1" ]; then rm -rf /var/run/scrapy fi -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - #DEBHELPER# exit 0 diff --git a/debian/scrapy-service.prerm b/debian/scrapy-service.prerm new file mode 100644 index 000000000..c0e87b571 --- /dev/null +++ b/debian/scrapy-service.prerm @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +case "$1" in + remove|upgrade|deconfigure) + rm -f /etc/scrapy/service_conf.pyc + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/scrapy-service.upstart b/debian/scrapy-service.upstart new file mode 100644 index 000000000..8e7d33c6a --- /dev/null +++ b/debian/scrapy-service.upstart @@ -0,0 +1,12 @@ +# Scrapy service + +start on runlevel [2345] +stop on runlevel [06] + +script + [ -r /etc/default/scrapy-service ] && . /etc/default/scrapy-service + exec twistd -ny /usr/share/scrapy/scrapy.tac \ + -u scrapy -g nogroup \ + -l /var/log/scrapy/service.log \ + --pidfile /var/run/scrapy/scrapy.pid +end script diff --git a/debian/scrapy.install b/debian/scrapy.install index 7bbeab3c5..856338403 100644 --- a/debian/scrapy.install +++ b/debian/scrapy.install @@ -1,3 +1 @@ -debian/conf/service_conf.py etc/scrapy -debian/conf/environment etc/scrapy -extras/scrapy.tac usr/share/scrapy +debian/tmp/usr diff --git a/debian/scrapy.lintian-overrides b/debian/scrapy.lintian-overrides new file mode 100644 index 000000000..e22020de1 --- /dev/null +++ b/debian/scrapy.lintian-overrides @@ -0,0 +1,4 @@ +new-package-should-close-itp-bug +binary-without-manpage usr/bin/scrapy-sqs +binary-without-manpage usr/bin/scrapy-ws +extra-license-file usr/share/pyshared/scrapy/xlib/pydispatch/license.txt diff --git a/debian/scrapy.postinst b/debian/scrapy.postinst deleted file mode 100644 index cbc63b20c..000000000 --- a/debian/scrapy.postinst +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# postinst script for scrapy -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see http://www.debian.org/doc/debian-policy/ or -# the debian-policy package - -case "$1" in - configure) - # Fix for old packages not managing pyc files - for d in /usr/lib/python*/site-packages/scrapy ; do - if [ -d "$d" ] ; then - echo "Removing old python byte-compiled files in $d" - rm -rf "$d" - fi - done - - # Create user to run the service as - if [ -z "`id -u scrapy 2> /dev/null`" ]; then - adduser --system --home /var/lib/scrapy --gecos "scrapy" \ - --no-create-home --disabled-password \ - --quiet scrapy || true - fi - if [ ! -d /var/run/scrapy ]; then - mkdir /var/run/scrapy - chown scrapy:nogroup /var/run/scrapy - fi - - chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 - - diff --git a/debian/scrapy.upstart b/debian/scrapy.upstart deleted file mode 100644 index c4cf6f0f4..000000000 --- a/debian/scrapy.upstart +++ /dev/null @@ -1,17 +0,0 @@ -# Scrapy service - -description "Scrapy web crawler" -author "Pablo Hoffman " - -start on runlevel [2345] -stop on runlevel [!2345] - -script - [ -r /etc/scrapy/environment ] && . /etc/scrapy/environment - export PYTHONPATH=/etc/scrapy:$PYTHONPATH - export SCRAPY_SETTINGS_MODULE=service_conf - exec twistd -ny /usr/share/scrapy/scrapy.tac \ - -u scrapy -g nogroup \ - -l /var/log/scrapy/service.log \ - --pidfile /var/run/scrapy/scrapy.pid -end script diff --git a/debian/conf/service_conf.py b/debian/service_conf.py similarity index 100% rename from debian/conf/service_conf.py rename to debian/service_conf.py diff --git a/debian/watch b/debian/watch deleted file mode 100644 index a62198e8b..000000000 --- a/debian/watch +++ /dev/null @@ -1,2 +0,0 @@ -version=3 -http://scrapy.org/releases/(\d\.\d)/Scrapy-([\d\.]+)\.tar\.gz diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index dd79756f1..bd091c445 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -138,8 +138,6 @@ This is the code for our first Spider, save it in a file named def parse(self, response): filename = response.url.split("/")[-2] open(filename, 'wb').write(response.body) - - SPIDER = DmozSpider() Crawling -------- @@ -370,8 +368,6 @@ Let's add this code to our spider:: link = site.select('a/@href').extract() desc = site.select('text()').extract() print title, link, desc - - SPIDER = DmozSpider() Now try crawling the dmoz.org domain again and you'll see sites being printed in your output, run:: @@ -418,8 +414,6 @@ scraped so far, the code for our Spider should be like this:: item['desc'] = site.select('text()').extract() items.append(item) return items - - SPIDER = DmozSpider() Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dfd30d3b2..77f327b35 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -177,7 +177,9 @@ DefaultHeadersMiddleware .. class:: DefaultHeadersMiddleware This middleware sets all default requests headers specified in the - :setting:`DEFAULT_REQUEST_HEADERS` setting. + :setting:`DEFAULT_REQUEST_HEADERS` setting plus those found in spider + ``default_request_headers`` attribute. Spider headers has precedence over + global headers. HttpAuthMiddleware ------------------ @@ -276,6 +278,17 @@ Number of seconds to use for HTTP cache expiration. Requests that were cached before this time will be re-downloaded. If zero, cached requests will always expire. A negative number means requests will never expire. +.. setting:: HTTPCACHE_IGNORE_HTTP_CODES + +HTTPCACHE_IGNORE_HTTP_CODES +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.10 + +Default: ``[]`` + +Don't cache response with these HTTP codes. + .. setting:: HTTPCACHE_IGNORE_MISSING HTTPCACHE_IGNORE_MISSING diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 649b2667e..065304280 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -93,8 +93,6 @@ This is how the spider would look so far:: # write the category page data extraction code here pass - SPIDER = GoogleDirectorySpider() - Extracting the data =================== diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 5e1a0e4bb..1d9b04727 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -53,10 +53,6 @@ scrapy.log module .. module:: scrapy.log :synopsis: Logging facility -.. attribute:: log_level - - The current log level being used - .. attribute:: started A boolean which is ``True`` is logging has been started or ``False`` otherwise. @@ -81,7 +77,7 @@ scrapy.log module setting will be used. :type logstdout: boolean -.. function:: msg(message, level=INFO, component=BOT_NAME, spider=None) +.. function:: msg(message, level=INFO, spider=None) Log a message @@ -91,24 +87,11 @@ scrapy.log module :param level: the log level for this message. See :ref:`topics-logging-levels`. - :param component: the component to use for logging, it defaults to - :setting:`BOT_NAME` - :type component: str - :param spider: the spider to use for logging this message. This parameter should always be used when logging things related to a particular spider. :type spider: :class:`~scrapy.spider.BaseSpider` object -.. function:: exc(message, level=ERROR, component=BOT_NAME, spider=None) - - Log an exception. Similar to ``msg()`` but it also appends a stack trace - report using `traceback.format_exc`. - - .. _traceback.format_exc: http://docs.python.org/library/traceback.html#traceback.format_exc - - It accepts the same parameters as the :func:`msg` function. - .. data:: CRITICAL Log level for critical errors diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 0504a4e70..fafd76788 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -177,8 +177,6 @@ Let's see an example:: def parse(self, response): self.log('A response from %s just arrived!' % response.url) - SPIDER = MySpider() - Another example returning multiples Requests and Items from a single callback:: from scrapy.selector import HtmlXPathSelector @@ -203,8 +201,6 @@ Another example returning multiples Requests and Items from a single callback:: for url in hxs.select('//a/@href').extract(): yield Request(url, callback=self.parse) - SPIDER = MySpider() - .. module:: scrapy.contrib.spiders :synopsis: Collection of generic spiders @@ -230,7 +226,7 @@ CrawlSpider Crawling rules ~~~~~~~~~~~~~~ -.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None) +.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None) ``link_extractor`` is a :ref:`Link Extractor ` object which defines how links will be extracted from each crawled page. @@ -253,6 +249,10 @@ Crawling rules of links extracted from each response using the specified ``link_extractor``. This is mainly used for filtering purposes. + ``process_request`` is a callable, or a string (in which case a method from + the spider object with that name will be used) which will be called with + every request extracted by this rule, and must return a request or None (to + filter out the request). CrawlSpider example ------------------- @@ -288,8 +288,6 @@ Let's now take a look at an example CrawlSpider with rules:: item['description'] = hxs.select('//td[@id="item_description"]/text()').extract() return item - SPIDER = MySpider() - This spider would start crawling example.com's home page, collecting category links, and item links, parsing the latter with the ``parse_item`` method. For @@ -405,8 +403,6 @@ These spiders are pretty easy to use, let's have at one example:: item['description'] = node.select('description').extract() return item - SPIDER = MySpider() - Basically what we did up there was creating a spider that downloads a feed from the given ``start_urls``, and then iterates through each of its ``item`` tags, prints them out, and stores some random data in an :class:`~scrapy.item.Item`. @@ -462,6 +458,3 @@ Let's see an example similar to the previous one, but using a item['name'] = row['name'] item['description'] = row['description'] return item - - SPIDER = MySpider() - diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py index 2ed7c52a6..53abcbd21 100644 --- a/examples/experimental/googledir/googledir/spiders/google_directory.py +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -36,6 +36,3 @@ class GoogleDirectorySpider(CrawlSpider): # Here we populate the item and yield it yield l.load_item() - -SPIDER = GoogleDirectorySpider() - diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py index 8c2ebcd01..fed14ab07 100644 --- a/examples/experimental/imdb/imdb/spiders/imdb_site.py +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -137,5 +137,3 @@ class ImdbSiteSpider(CrawlSpider): def _urljoin(self, response, url): """Helper to convert relative urls to absolute""" return urljoin_rfc(response.url, url, response.encoding) - -SPIDER = ImdbSiteSpider() diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index b214be283..ceb370e3e 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -38,5 +38,3 @@ class GoogleDirectorySpider(CrawlSpider): # Here we populate the item and yield it yield l.load_item() - -SPIDER = GoogleDirectorySpider() diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 2db13a5dd..fd1628ef4 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 9, 0, '') -__version__ = "0.9" +version_info = (0, 10, 0, 'dev') +__version__ = "0.10-dev" import sys, os, warnings diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index c1b46edf0..411e52371 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -84,6 +84,9 @@ class Command(ScrapyCommand): return else: spider = spiders.create_for_request(request) + if spider is None: + log.msg('Unable to find spider for URL: %s' % args[0], log.ERROR) + return scrapymanager.configure() scrapymanager.queue.append_request(request, spider) @@ -112,7 +115,7 @@ class Command(ScrapyCommand): for rule in spider.rules: if rule.link_extractor.matches(response.url) \ and rule.callback: - + items, links = self.run_callback(spider, response, rule.callback, args, opts) diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 809a62b98..7f018603b 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -123,6 +123,7 @@ HTTPCACHE_DIR = '' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 +HTTPCACHE_IGNORE_HTTP_CODES = [] ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' @@ -190,7 +191,7 @@ SCHEDULER_ORDER = 'DFO' SERVICE_QUEUE = 'scrapy.core.queue.KeepAliveExecutionQueue' -SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager' +SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.SpiderManager' SPIDER_MIDDLEWARES = {} @@ -233,10 +234,6 @@ USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = 6023 # if None, uses a dynamic port -WEBCONSOLE_ENABLED = True -WEBCONSOLE_PORT = 6080 -WEBCONSOLE_LOGFILE = None - WEBSERVICE_ENABLED = True WEBSERVICE_LOGFILE = None WEBSERVICE_PORT = 6080 diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py index cebfe247b..1fab70858 100644 --- a/scrapy/contrib/downloadermiddleware/defaultheaders.py +++ b/scrapy/contrib/downloadermiddleware/defaultheaders.py @@ -3,15 +3,27 @@ DefaultHeaders downloader middleware See documentation in docs/topics/downloader-middleware.rst """ - from scrapy.conf import settings +from scrapy.xlib.pydispatch import dispatcher +from scrapy.core import signals + class DefaultHeadersMiddleware(object): def __init__(self): - self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + self.global_default_headers = settings.get('DEFAULT_REQUEST_HEADERS') + self._default_headers = {} + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def process_request(self, request, spider): - for k, v in self.default_headers.iteritems(): + for k, v in self._default_headers[spider].iteritems(): if v: request.headers.setdefault(k, v) + + def spider_opened(self, spider): + self._default_headers[spider] = dict(self.global_default_headers, + **getattr(spider, 'default_request_headers', {})) + + def spider_closed(self, spider): + self._default_headers.pop(spider) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 341e9b5a1..d1af33d7a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -22,6 +22,7 @@ class HttpCacheMiddleware(object): def __init__(self, settings=conf.settings): self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) dispatcher.connect(self.spider_opened, signal=signals.spider_opened) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) @@ -35,17 +36,20 @@ class HttpCacheMiddleware(object): if not self.is_cacheable(request): return response = self.storage.retrieve_response(spider, request) - if response: + if response and self.is_cacheable_response(response): response.flags.append('cached') return response elif self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if self.is_cacheable(request): + if self.is_cacheable(request) and self.is_cacheable_response(response): self.storage.store_response(spider, request, response) return response + def is_cacheable_response(self, response): + return response.status not in self.ignore_http_codes + def is_cacheable(self, request): return urlparse_cached(request).scheme in ['http', 'https'] diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py index 0e82159e8..df9b5c184 100644 --- a/scrapy/contrib/ibl/extraction/pageparsing.py +++ b/scrapy/contrib/ibl/extraction/pageparsing.py @@ -117,7 +117,7 @@ class TemplatePageParser(InstanceLearningParser): def _handle_unpaired_tag(self, html_tag): if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img": self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1)) - elif self._read_bool_template_attribute(html_tag, "ignore-beneath") and html_tag.tag == "img": + elif self._read_bool_template_attribute(html_tag, "ignore-beneath"): self.ignored_regions.append((self.next_tag_index, None)) jannotation = self._read_template_annotation(html_tag) if jannotation: diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index 263f40f36..7b54c8173 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -3,18 +3,16 @@ SpiderManager is the class which locates and manages all website-specific spiders """ -import sys +import inspect -from twisted.plugin import getCache -from twisted.python.rebuild import rebuild - -from scrapy.spider.models import ISpider from scrapy import log from scrapy.conf import settings from scrapy.utils.url import url_is_from_spider +from scrapy.utils.misc import walk_modules +from scrapy.spider import BaseSpider -class TwistedPluginSpiderManager(object): - """Spider manager based in Twisted Plugin System""" + +class SpiderManager(object): def __init__(self): self.loaded = False @@ -25,9 +23,7 @@ class TwistedPluginSpiderManager(object): spider arguments. If the sipder name is not found, it raises a KeyError. """ - spider = self._spiders[spider_name] - spider.__dict__.update(spider_kwargs) - return spider + return self._spiders[spider_name](**spider_kwargs) def find_by_request(self, request): """Returns list of spiders names that match the given Request""" @@ -60,47 +56,23 @@ class TwistedPluginSpiderManager(object): return self._spiders.keys() def load(self, spider_modules=None): - """Load spiders from module directory.""" + """Load spiders from spider_modules or SPIDER_MODULES setting.""" if spider_modules is None: spider_modules = settings.getlist('SPIDER_MODULES') self.spider_modules = spider_modules - self._spiders = {} - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for module in modules: - for spider in self._getspiders(ISpider, module): - ISpider.validateInvariants(spider) - self._spiders[spider.name] = spider + self._spiders = {} + for name in self.spider_modules: + for module in walk_modules(name): + self._load_spiders(module) self.loaded = True - def _getspiders(self, interface, package): - """This is an override of twisted.plugin.getPlugin, because we're - interested in catching exceptions thrown when loading spiders such as - KeyboardInterrupt - """ - try: - allDropins = getCache(package) - for dropin in allDropins.itervalues(): - for plugin in dropin.plugins: - adapted = interface(plugin, None) - if adapted is not None: - yield adapted - except KeyboardInterrupt: - sys.stderr.write("Interrupted while loading Scrapy spiders\n") - sys.exit(2) + def _load_spiders(self, module): + for obj in vars(module).itervalues(): + if inspect.isclass(obj) and issubclass(obj, BaseSpider): + name = getattr(obj, 'name', None) + if name is not None: + self._spiders[name] = obj def close_spider(self, spider): - """Reload spider module to release any resources held on to by the - spider - """ - name = spider.name - if name not in self._spiders: - return - spider = self._spiders[name] - module_name = spider.__module__ - module = sys.modules[module_name] - if hasattr(module, 'SPIDER'): - log.msg("Reloading module %s" % module_name, spider=spider, \ - level=log.DEBUG) - new_module = rebuild(module, doLog=0) - self._spiders[name] = new_module.SPIDER + pass diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 14d4fe40e..4ca606d8a 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -13,84 +13,40 @@ from scrapy.utils.spider import iterate_spider_output from scrapy.contrib.spiders.init import InitSpider from scrapy.conf import settings +def identity(x): + return x + class Rule(object): - """ - A rule for crawling, which receives the following constructor arguments: - link_extractor (required) - A LinkExtractor which defines the policy for extracting links - callback (optional) - A function to use to process the page once it has been downloaded. If - callback is omitted the page is not procesed, just crawled. If callback - is a string (instead of callable) a method of the spider class with that - name is used as the callback function - cb_kwargs (optional) - A dict specifying keyword arguments to pass to the callback function - follow (optional) - If True, links will be followed from the pages crawled by this rule. - It defaults to True when no callback is specified or False when a - callback is specified - process_links (optional) - Can be either a callable, or a string with the name of a method defined - in the spider's class. - This method will be called with the list of extracted links matching - this rule (if any) and must return another list of links. - """ - - def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None): + def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity): self.link_extractor = link_extractor self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links + self.process_request = process_request if follow is None: self.follow = False if callback else True else: self.follow = follow class CrawlSpider(InitSpider): - """ - Class for spiders that crawl over web pages and extract/parse their links - given some crawling rules. - These crawling rules are established by setting the 'rules' class attribute, - which is a tuple of Rule objects. - When the spider is running, it iterates over these rules with each response - and do what it has to (extract links if follow=True, and return items/requests if - there's a parsing method defined in the rule). - """ rules = () def __init__(self, *a, **kw): - """Constructor takes care of compiling rules""" super(CrawlSpider, self).__init__(*a, **kw) self._compile_rules() def parse(self, response): - """This function is called by the framework core for all the - start_urls. Do not override this function, override parse_start_url - instead.""" return self._response_downloaded(response, self.parse_start_url, cb_kwargs={}, follow=True) def parse_start_url(self, response): - """Overrideable callback function for processing start_urls. It must - return a list of BaseItem and/or Requests""" return [] def process_results(self, response, results): - """This overridable method is called for each result (item or request) - returned by the spider, and it's intended to perform any last time - processing required before returning the results to the framework core, - for example setting the item GUIDs. It receives a list of results and - the response which originated that results. It must return a list - of results (Items or Requests).""" return results def _requests_to_follow(self, response): - """ - This method iterates over each of the spider's rules, extracts the links - matching each case, filters them (if needed), and returns a list of unique - requests per response. - """ seen = set() for rule in self._rules: links = [l for l in rule.link_extractor.extract_links(response) if l not in seen] @@ -102,14 +58,9 @@ class CrawlSpider(InitSpider): cb_kwargs=rule.cb_kwargs, follow=rule.follow) r = Request(url=link.url, callback=callback) r.meta['link_text'] = link.text - yield r + yield rule.process_request(r) def _response_downloaded(self, response, callback, cb_kwargs, follow): - """ - This is were any response arrives, and were it's decided whether - to extract links or not from it, and if it will be parsed or not. - It returns a list of requests/items. - """ if callback: cb_res = callback(response, **cb_kwargs) or () cb_res = self.process_results(response, cb_res) @@ -122,8 +73,6 @@ class CrawlSpider(InitSpider): def _compile_rules(self): - """Compile the crawling rules""" - def get_method(method): if callable(method): return method @@ -134,3 +83,4 @@ class CrawlSpider(InitSpider): for rule in self._rules: rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) + rule.process_request = get_method(rule.process_request) diff --git a/scrapy/contrib/webconsole/__init__.py b/scrapy/contrib/webconsole/__init__.py deleted file mode 100644 index 72830df17..000000000 --- a/scrapy/contrib/webconsole/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import warnings -warnings.warn("Web console is deprecated. Consider using web service instead.", \ - DeprecationWarning, stacklevel=2) diff --git a/scrapy/contrib/webconsole/enginestatus.py b/scrapy/contrib/webconsole/enginestatus.py deleted file mode 100644 index 623f777ef..000000000 --- a/scrapy/contrib/webconsole/enginestatus.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Scheduler information module for Scrapy webconsole -""" -from scrapy.xlib.pydispatch import dispatcher -from scrapy.utils.engine import get_engine_status -from scrapy.management.web import banner - -class EngineStatus(object): - webconsole_id = 'enginestatus' - webconsole_name = 'Engine status' - - def __init__(self): - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - s = banner(self) - s += "
\n"
-        s += get_engine_status()
-        s += "
\n" - s += "\n" - s += "\n" - - return s - - def webconsole_discover_module(self): - return self diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py deleted file mode 100644 index c5badee0e..000000000 --- a/scrapy/contrib/webconsole/livestats.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -Live statistics extension -""" -from datetime import datetime -from scrapy.xlib.pydispatch import dispatcher -from scrapy.core import signals -from scrapy.core.manager import scrapymanager -from scrapy.management.web import banner, webconsole_discover_module - -class SpiderStats(object): - def __init__(self): - self.scraped = 0 - self.crawled = 0 - self.started = None - -class LiveStats(object): - webconsole_id = 'livestats' - webconsole_name = 'Spider live statistics of current run' - - def __init__(self): - self.domains = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) - - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def spider_opened(self, spider): - pstats = SpiderStats() - self.domains[spider] = pstats - pstats.started = datetime.now().replace(microsecond=0) - - def spider_closed(self, spider): - del self.domains[spider] - - def item_scraped(self, item, spider): - self.domains[spider].scraped += 1 - - def response_downloaded(self, response, spider): - # sometimes we download responses without opening/closing domains, - # for example from scrapy shell - if self.domains.get(spider): - self.domains[spider].crawled += 1 - - def webconsole_render(self, wc_request): - sch = scrapymanager.engine.scheduler - dwl = scrapymanager.engine.downloader - - totdomains = totscraped = totcrawled = totscheduled = totactive = totdqueued = tottransf = 0 - s = banner(self) - s += "\n" - s += "\n" - for spider in sorted(self.domains.keys()): - scheduled = len(sch.pending_requests[spider]) if spider in sch.pending_requests else 0 - active = len(dwl.sites[spider].active) if spider in dwl.sites else 0 - dqueued = len(dwl.sites[spider].queue) if spider in dwl.sites else 0 - transf = len(dwl.sites[spider].transferring) if spider in dwl.sites else 0 - stats = self.domains[spider] - runtime = datetime.now() - stats.started - - s += '\n' % \ - (spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) - - totdomains += 1 - totscraped += stats.scraped - totcrawled += stats.crawled - totscheduled += scheduled - totactive += active - totdqueued += dqueued - tottransf += transf - s += '\n' % \ - (totdomains, totscraped, totcrawled, totscheduled, totdqueued, totactive, tottransf) - s += "
SpiderItems
Scraped		Pages
Crawled		Scheduler
Pending		Downloader
Queued		Downloader
Active		Downloader
Transferring		Start timeRun time
%s%d%d%d%d%d%d%s%s
%d domains%d%d%d%d%d%d
\n" - - s += "\n" - s += "\n" - - return s - - def webconsole_discover_module(self): - return self diff --git a/scrapy/contrib/webconsole/scheduler.py b/scrapy/contrib/webconsole/scheduler.py deleted file mode 100644 index 756bc3b6d..000000000 --- a/scrapy/contrib/webconsole/scheduler.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Scheduler queue web console module - -See documentation in docs/topics/extensions.rst -""" - -from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.manager import scrapymanager -from scrapy.management.web import banner, webconsole_discover_module - -class SchedulerQueue(object): - webconsole_id = 'scheduler' - webconsole_name = 'Scheduler queue' - - def __init__(self): - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_discover_module(self): - return self - - def webconsole_render(self, wc_request): - s = banner(self) - s += "
    \n" - for domain, request_queue in scrapymanager.engine.scheduler.pending_requests.iteritems(): - s += "
  • \n" - s += "%s (%s requests)\n" % (domain, len(request_queue)) - s += "
      \n" - for ((req, _), prio) in request_queue: - s += "
    • %s (priority: %d)
      • \n" % (req.url, req.url, prio) - s += "
    \n" - s += "
    • \n" - s += "
\n" - - s += "\n" - s += "\n" - - return s diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py deleted file mode 100644 index 0c3f24176..000000000 --- a/scrapy/contrib/webconsole/spiderctl.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -Extensions for allowing spider control from web console -""" - -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.core import signals -from scrapy.core.manager import scrapymanager -from scrapy.spider import spiders -from scrapy.management.web import banner, webconsole_discover_module -from scrapy.conf import settings - -class Spiderctl(object): - webconsole_id = 'spiderctl' - webconsole_name = 'Spider control panel' - - def __init__(self): - self.running = {} - self.finished = set() - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def spider_opened(self, spider): - self.running[spider.name] = spider - - def spider_closed(self, spider): - del self.running[spider.name] - self.finished.add(spider.name) - - def webconsole_render(self, wc_request): - if wc_request.args: - changes = self.webconsole_control(wc_request) - - self.scheduled = [s[0].name for s in scrapymanager.queue.spider_requests] - self.idle = [d for d in self.enabled_spiders if d not in self.scheduled - and d not in self.running - and d not in self.finished] - - s = banner(self) - s += '\n' - s += "\n" % \ - (len(self.idle), - len(self.scheduled), - len(self.running), - settings['CONCURRENT_SPIDERS'], - len(self.finished)) - s += "\n" - - # idle - s += "\n" - - # scheduled - s += "\n" - - # running - s += "\n" - - # finished - s += "\n" - - s += "\n" - s += "
Idle (%d)Scheduled (%d)Running (%d/%d)Finished (%d)
\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - - s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "		\n" - s += '
\n' - s += '
\n' - s += '
' - s += '\n' - s += '
\n' - s += "
\n" - - if wc_request.args: - s += changes - - s += "\n" - s += "\n" - - return s - - def webconsole_control(self, wc_request): - args = wc_request.args - s = "
\n" - - if "stop_running_spiders" in args: - s += "

" - stopped_spiders = [] - for name in args["stop_running_spiders"]: - if name in self.running: - scrapymanager.engine.close_spider(self.running[name]) - stopped_spiders.append(name) - s += "Stopped spiders:

  • %s
" % "
  • ".join(stopped_spiders) - s += "

    " - if "remove_pending_spiders" in args: - removed = [] - for name in args["remove_pending_spiders"]: - q = scrapymanager.queue - q.spider_requests = [x for x in q.spider_requests if x[0].name != name] - if removed: - s += "

    " - s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_spiders"]) - s += "

    " - if "add_pending_spiders" in args: - for name in args["add_pending_spiders"]: - if name not in scrapymanager.engine.scheduler.pending_requests: - scrapymanager.queue.append_spider_name(name) - s += "

    " - s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_spiders"]) - s += "

    " - if "rerun_finished_spiders" in args: - for name in args["rerun_finished_spiders"]: - if name not in scrapymanager.engine.scheduler.pending_requests: - scrapymanager.queue.append_spider_name(name) - self.finished.remove(name) - s += "

    " - s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_spiders"]) - s += "

    " - - return s - - def webconsole_discover_module(self): - self.enabled_spiders = spiders.list() - return self diff --git a/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py deleted file mode 100644 index e8c707f64..000000000 --- a/scrapy/contrib/webconsole/stats.py +++ /dev/null @@ -1,35 +0,0 @@ -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.stats import stats -from scrapy.management.web import banner, webconsole_discover_module - -def stats_html_table(statsdict): - s = "" - s += "\n" - for kv in statsdict.iteritems(): - s += "\n" % kv - s += "
    %s%s
    \n" - return s - -class StatsDump(object): - webconsole_id = 'stats' - webconsole_name = 'StatsCollector dump' - - def __init__(self): - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - s = banner(self) - s += "

    Global stats

    \n" - s += stats_html_table(stats.get_stats()) - for spider, spider_stats in stats.iter_spider_stats(): - s += "

    %s

    \n" % spider.name - s += stats_html_table(spider_stats) - s += "\n" - s += "\n" - - return str(s) - - def webconsole_discover_module(self): - return self - diff --git a/scrapy/log.py b/scrapy/log.py index c8e969928..0caa82ed7 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -4,42 +4,64 @@ Scrapy logging facility See documentation in docs/topics/logging.rst """ import sys -from traceback import format_exc +import logging from twisted.python import log -from scrapy.xlib.pydispatch import dispatcher from scrapy.conf import settings from scrapy.utils.python import unicode_to_str # Logging levels -SILENT, CRITICAL, ERROR, WARNING, INFO, DEBUG = range(6) +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL +SILENT = CRITICAL + 1 + level_names = { - 0: "SILENT", - 1: "CRITICAL", - 2: "ERROR", - 3: "WARNING", - 4: "INFO", - 5: "DEBUG", + logging.DEBUG: "DEBUG", + logging.INFO: "INFO", + logging.WARNING: "WARNING", + logging.ERROR: "ERROR", + logging.CRITICAL: "CRITICAL", + SILENT: "SILENT", } -BOT_NAME = settings['BOT_NAME'] - -# signal sent when log message is received -# args: message, level, spider -logmessage_received = object() - -# default values -log_level = DEBUG -log_encoding = 'utf-8' - started = False +class ScrapyFileLogObserver(log.FileLogObserver): + + def __init__(self, f, level=INFO, encoding='utf-8'): + self.level = level + self.encoding = encoding + log.FileLogObserver.__init__(self, f) + + def emit(self, eventDict): + if eventDict.get('system') != 'scrapy': + return + level = eventDict.get('logLevel') + if level < self.level: + return + spider = eventDict.get('spider') + message = eventDict.get('message') + lvlname = level_names.get(level, 'NOLEVEL') + if message: + message = [unicode_to_str(x, self.encoding) for x in message] + message[0] = "%s: %s" % (lvlname, message[0]) + why = eventDict.get('why') + if why: + why = "%s: %s" % (lvlname, unicode_to_str(why, self.encoding)) + eventDict['message'] = message + eventDict['why'] = why + eventDict['system'] = spider.name if spider else '-' + log.FileLogObserver.emit(self, eventDict) + def _get_log_level(level_name_or_id=None): if level_name_or_id is None: - lvlname = settings['LOG_LEVEL'] or settings['LOGLEVEL'] + lvlname = settings['LOG_LEVEL'] return globals()[lvlname] - elif isinstance(level_name_or_id, int) and 0 <= level_name_or_id <= 5: + elif isinstance(level_name_or_id, int): return level_name_or_id elif isinstance(level_name_or_id, basestring): return globals()[level_name_or_id] @@ -47,53 +69,31 @@ def _get_log_level(level_name_or_id=None): raise ValueError("Unknown log level: %r" % level_name_or_id) def start(logfile=None, loglevel=None, logstdout=None): - """Initialize and start logging facility""" - global log_level, log_encoding, started - + global started if started or not settings.getbool('LOG_ENABLED'): return - log_level = _get_log_level(loglevel) - log_encoding = settings['LOG_ENCODING'] started = True - # set log observer if log.defaultObserver: # check twisted log not already started - logfile = logfile or settings['LOG_FILE'] or settings['LOGFILE'] + loglevel = _get_log_level(loglevel) + logfile = logfile or settings['LOG_FILE'] + file = open(logfile, 'a') if logfile else sys.stderr if logstdout is None: logstdout = settings.getbool('LOG_STDOUT') + sflo = ScrapyFileLogObserver(file, loglevel, settings['LOG_ENCODING']) + log.startLoggingWithObserver(sflo.emit, setStdout=logstdout) + msg("Started project: %s" % settings['BOT_NAME']) - file = open(logfile, 'a') if logfile else sys.stderr - log.startLogging(file, setStdout=logstdout) - -def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): - """Log message according to the level""" - if level > log_level: - return - if domain is not None: +def msg(message, level=INFO, **kw): + if 'component' in kw: import warnings - warnings.warn("'domain' argument of scrapy.log.msg() is deprecated, " \ - "use 'spider' argument instead", DeprecationWarning, stacklevel=2) - dispatcher.send(signal=logmessage_received, message=message, level=level, \ - spider=spider) - system = domain or (spider.name if spider else component) - msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding) - log.msg(msg_txt, system=system) + warnings.warn("Argument `component` of scrapy.log.msg() is deprecated", \ + DeprecationWarning, stacklevel=2) + kw.setdefault('system', 'scrapy') + kw['logLevel'] = level + log.msg(message, **kw) -def exc(message, level=ERROR, component=BOT_NAME, domain=None, spider=None): - message = message + '\n' + format_exc() - msg(message, level, component, domain, spider) - -def err(_stuff=None, _why=None, **kwargs): - if ERROR > log_level: - return - domain = kwargs.pop('domain', None) - spider = kwargs.pop('spider', None) - component = kwargs.pop('component', BOT_NAME) - if domain is not None: - import warnings - warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \ - "use 'spider' argument instead", DeprecationWarning, stacklevel=2) - kwargs['system'] = domain or (spider.name if spider else component) - if _why: - _why = unicode_to_str("ERROR: %s" % _why, log_encoding) - log.err(_stuff, _why, **kwargs) +def err(_stuff=None, _why=None, **kw): + kw.setdefault('system', 'scrapy') + kw['logLevel'] = kw.pop('level', ERROR) + log.err(_stuff, _why, **kw) diff --git a/scrapy/management/web.py b/scrapy/management/web.py deleted file mode 100644 index d54cf4953..000000000 --- a/scrapy/management/web.py +++ /dev/null @@ -1,93 +0,0 @@ -import warnings -warnings.warn("Scrapy web console is deprecated. Consider using web service instead.", \ - DeprecationWarning, stacklevel=2) - -import re -import socket -from time import time - -from twisted.internet import reactor -from twisted.web import server, resource - -from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.exceptions import NotConfigured -from scrapy.core.manager import scrapymanager -from scrapy.conf import settings - -# web management signals -webconsole_discover_module = object() - -urlpath_re = re.compile(r"^/(\w+)/") - -def error404(module): - return """ - - 404 Not Found - -

    Not found

    -

    Web console module not found: %s

    -

    Back to main menu

    - - -""" % module - -def banner(module=None): - s = "\n" - s += "Scrapy\n" - s += "\n" - s += "

    Scrapy web console

    \n" - uptime = time() - scrapymanager.engine.start_time - s += "

    Bot: %s | Host: %s | Uptime: %ds

    \n" % \ - (settings['BOT_NAME'], socket.gethostname(), uptime) - if module: - s += "

    %s

    \n" % (module.webconsole_id, \ - module.webconsole_name) - return s - -class WebConsoleResource(resource.Resource): - isLeaf = True - - @property - def modules(self): - if not hasattr(self, '_modules'): - self._modules = {} - for _, obj in dispatcher.send(signal=webconsole_discover_module, \ - sender=self.__class__): - self._modules[obj.webconsole_id] = obj - return self._modules - - def render_GET(self, request): - m = urlpath_re.search(request.path) - if m: - module = m.group(1) - if module in self.modules: - return self.modules[m.group(1)].webconsole_render(request) - else: - request.setResponseCode(404) - return error404(module) - else: - return self.module_list() - - render_POST = render_GET - - def module_list(self): - s = banner() - s += "

    Available modules:

    \n" - s += "
      \n" - for name, obj in self.modules.iteritems(): - s += "
    • %s
      • \n" % (name, obj.webconsole_name) - s += "
    \n" - s += "\n" - s += "\n" - return s - -class WebConsole(server.Site): - - def __init__(self): - if not settings.getbool('WEBCONSOLE_ENABLED'): - raise NotConfigured - logfile = settings['WEBCONSOLE_LOGFILE'] - server.Site.__init__(self, WebConsoleResource(), logPath=logfile) - self.noisy = False - port = settings.getint('WEBCONSOLE_PORT') - reactor.callWhenRunning(reactor.listenTCP, port, self) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index d0be6f901..a58f0a0c9 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -6,25 +6,17 @@ See documentation in docs/topics/spiders.rst import warnings -from zope.interface import Interface, Attribute, invariant, implements -from twisted.plugin import IPlugin - from scrapy import log from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref -class ISpider(Interface, IPlugin) : - """Interface used by TwistedPluginSpiderManager to discover spiders""" - pass class BaseSpider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this class. """ - implements(ISpider) - # XXX: class attributes kept for backwards compatibility name = None start_urls = [] diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 246015466..63ddce401 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -9,6 +9,3 @@ class $classname(BaseSpider): def parse(self, response): pass - -SPIDER = $classname() - diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 578779c06..d0036443f 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -21,5 +21,3 @@ class $classname(CrawlSpider): #i['name'] = hxs.select('//div[@id="name"]').extract() #i['description'] = hxs.select('//div[@id="description"]').extract() return i - -SPIDER = $classname() diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index c9a723000..7551686a6 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -18,5 +18,3 @@ class $classname(CSVFeedSpider): #i['name'] = row['name'] #i['description'] = row['description'] return i - -SPIDER = $classname() diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index f5ecbd707..5e16d2325 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -12,5 +12,3 @@ class $classname(XMLFeedSpider): #i['name'] = selector.select('name').extract() #i['description'] = selector.select('description').extract() return i - -SPIDER = $classname() diff --git a/scrapy/tests/test_contrib_spidermanager/__init__.py b/scrapy/tests/test_contrib_spidermanager/__init__.py index 3f4cff83a..930ceed54 100644 --- a/scrapy/tests/test_contrib_spidermanager/__init__.py +++ b/scrapy/tests/test_contrib_spidermanager/__init__.py @@ -7,13 +7,13 @@ from twisted.trial import unittest # ugly hack to avoid cyclic imports of scrapy.spider when running this test # alone -import scrapy.spider -from scrapy.contrib.spidermanager import TwistedPluginSpiderManager +import scrapy.spider +from scrapy.contrib.spidermanager import SpiderManager from scrapy.http import Request module_dir = os.path.dirname(os.path.abspath(__file__)) -class TwistedPluginSpiderManagerTest(unittest.TestCase): +class SpiderManagerTest(unittest.TestCase): def setUp(self): orig_spiders_dir = os.path.join(module_dir, 'test_spiders') @@ -22,7 +22,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) - self.spiderman = TwistedPluginSpiderManager() + self.spiderman = SpiderManager() assert not self.spiderman.loaded self.spiderman.load(['test_spiders_xxx']) assert self.spiderman.loaded @@ -32,7 +32,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): sys.path.remove(self.tmpdir) def test_list(self): - self.assertEqual(set(self.spiderman.list()), + self.assertEqual(set(self.spiderman.list()), set(['spider1', 'spider2'])) def test_create(self): @@ -42,14 +42,6 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.assertEqual(spider2.__class__.__name__, 'Spider2') self.assertEqual(spider2.foo, 'bar') - def test_create_uses_cache(self): - # TwistedPluginSpiderManager uses an internal cache which is - # invalidated in close_spider() but this isn't necessarily the best - # thing to do in all cases. - spider1 = self.spiderman.create("spider1") - spider2 = self.spiderman.create("spider1") - assert spider1 is spider2 - def test_find_by_request(self): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) @@ -60,16 +52,10 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase): self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')), []) - def test_close_spider_remove_refs(self): - spider = self.spiderman.create("spider1") - wref = weakref.ref(spider) - assert wref() - self.spiderman.close_spider(spider) - del spider - assert not wref() + def test_load_spider_module(self): + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider1']) + assert len(self.spiderman._spiders) == 1 - def test_close_spider_invalidates_cache(self): - spider1 = self.spiderman.create("spider1") - self.spiderman.close_spider(spider1) - spider2 = self.spiderman.create("spider1") - assert spider1 is not spider2 + def test_load_base_spider(self): + self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider0']) + assert len(self.spiderman._spiders) == 0 diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py new file mode 100644 index 000000000..442d690d1 --- /dev/null +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider0.py @@ -0,0 +1,4 @@ +from scrapy.spider import BaseSpider + +class Spider0(BaseSpider): + allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py index 0a9b60989..f38af6ba6 100644 --- a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider1.py @@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider class Spider1(BaseSpider): name = "spider1" allowed_domains = ["scrapy1.org", "scrapy3.org"] - -SPIDER = Spider1() diff --git a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py index 52023277f..7af1441e7 100644 --- a/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py +++ b/scrapy/tests/test_contrib_spidermanager/test_spiders/spider2.py @@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider class Spider2(BaseSpider): name = "spider2" allowed_domains = ["scrapy2.org", "scrapy3.org"] - -SPIDER = Spider2() diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index 805289787..d24882d02 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.conf import settings from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware -from scrapy.http import Response, Request +from scrapy.http import Request from scrapy.spider import BaseSpider @@ -11,20 +11,38 @@ class TestDefaultHeadersMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('foo') self.mw = DefaultHeadersMiddleware() - self.default_headers = dict([(k, [v]) for k, v in \ + self.default_request_headers = dict([(k, [v]) for k, v in \ settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) def test_process_request(self): req = Request('http://www.scrapytest.org') + self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) - self.assertEquals(req.headers, self.default_headers) + self.mw.spider_closed(self.spider) + self.assertEquals(req.headers, self.default_request_headers) + + def test_spider_default_request_headers(self): + spider_headers = {'Unexistant-Header': ['value']} + # override one of the global default headers by spider + if self.default_request_headers: + k = set(self.default_request_headers).pop() + spider_headers[k] = ['__newvalue__'] + self.spider.default_request_headers = spider_headers + + req = Request('http://www.scrapytest.org') + self.mw.spider_opened(self.spider) + self.mw.process_request(req, self.spider) + self.mw.spider_closed(self.spider) + self.assertEquals(req.headers, dict(self.default_request_headers, **spider_headers)) def test_update_headers(self): headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} req = Request('http://www.scrapytest.org', headers=headers) self.assertEquals(req.headers, headers) + self.mw.spider_opened(self.spider) self.mw.process_request(req, self.spider) - self.default_headers.update(headers) - self.assertEquals(req.headers, self.default_headers) + self.mw.spider_closed(self.spider) + self.default_request_headers.update(headers) + self.assertEquals(req.headers, self.default_request_headers) diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index b9331110c..230352804 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -24,6 +24,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): settings = { 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, + 'HTTPCACHE_IGNORE_HTTP_CODES': [], } settings.update(new_settings) return Settings(settings) @@ -76,6 +77,22 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_middleware_ignore_http_codes(self): + # test response is not cached + mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202]) + assert mw.process_request(self.request, self.spider) is None + mw.process_response(self.request, self.response, self.spider) + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + # test response is cached + mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203]) + mw.process_response(self.request, self.response, self.spider) + response = mw.process_request(self.request, self.spider) + assert isinstance(response, HtmlResponse) + self.assertEqualResponse(self.response, response) + assert 'cached' in response.flags + def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url) self.assertEqual(response1.status, response2.status) diff --git a/scrapy/tests/test_log.py b/scrapy/tests/test_log.py index ec9c1a2db..da3baa1a8 100644 --- a/scrapy/tests/test_log.py +++ b/scrapy/tests/test_log.py @@ -1,17 +1,129 @@ -import unittest +from cStringIO import StringIO + +from twisted.python import log as txlog, failure +from twisted.trial import unittest from scrapy import log +from scrapy.spider import BaseSpider from scrapy.conf import settings -class ItemTest(unittest.TestCase): +class LogTest(unittest.TestCase): def test_get_log_level(self): default_log_level = getattr(log, settings['LOG_LEVEL']) self.assertEqual(log._get_log_level(), default_log_level) self.assertEqual(log._get_log_level('WARNING'), log.WARNING) self.assertEqual(log._get_log_level(log.WARNING), log.WARNING) - self.assertRaises(ValueError, log._get_log_level, 99999) self.assertRaises(ValueError, log._get_log_level, object()) +class ScrapyFileLogObserverTest(unittest.TestCase): + + level = log.INFO + encoding = 'utf-8' + + def setUp(self): + self.f = StringIO() + self.sflo = log.ScrapyFileLogObserver(self.f, self.level, self.encoding) + self.sflo.start() + + def tearDown(self): + self.flushLoggedErrors() + self.sflo.stop() + + def logged(self): + return self.f.getvalue().strip()[25:] + + def first_log_line(self): + logged = self.logged() + return logged.splitlines()[0] if logged else '' + + def test_msg_basic(self): + log.msg("Hello") + self.assertEqual(self.logged(), "[-] INFO: Hello") + + def test_msg_spider(self): + spider = BaseSpider("myspider") + log.msg("Hello", spider=spider) + self.assertEqual(self.logged(), "[myspider] INFO: Hello") + + def test_msg_level1(self): + log.msg("Hello", level=log.WARNING) + self.assertEqual(self.logged(), "[-] WARNING: Hello") + + def test_msg_level2(self): + log.msg("Hello", log.WARNING) + self.assertEqual(self.logged(), "[-] WARNING: Hello") + + def test_msg_wrong_level(self): + log.msg("Hello", level=9999) + self.assertEqual(self.logged(), "[-] NOLEVEL: Hello") + + def test_msg_level_spider(self): + spider = BaseSpider("myspider") + log.msg("Hello", spider=spider, level=log.WARNING) + self.assertEqual(self.logged(), "[myspider] WARNING: Hello") + + def test_msg_encoding(self): + log.msg(u"Price: \xa3100") + self.assertEqual(self.logged(), "[-] INFO: Price: \xc2\xa3100") + + def test_msg_ignore_level(self): + log.msg("Hello", level=log.DEBUG) + log.msg("World", level=log.INFO) + self.assertEqual(self.logged(), "[-] INFO: World") + + def test_msg_ignore_system(self): + txlog.msg("Hello") + self.failIf(self.logged()) + + def test_msg_ignore_system_err(self): + txlog.msg("Hello") + self.failIf(self.logged()) + + def test_err_noargs(self): + try: + a = 1/0 + except: + log.err() + self.failUnless('Traceback' in self.logged()) + self.failUnless('ZeroDivisionError' in self.logged()) + + def test_err_why(self): + log.err(TypeError("bad type"), "Wrong type") + self.assertEqual(self.first_log_line(), "[-] ERROR: Wrong type") + self.failUnless('TypeError' in self.logged()) + self.failUnless('bad type' in self.logged()) + + def test_err_why_encoding(self): + log.err(TypeError("bad type"), u"\xa3") + self.assertEqual(self.first_log_line(), "[-] ERROR: \xc2\xa3") + + def test_err_exc(self): + log.err(TypeError("bad type")) + self.failUnless('Unhandled Error' in self.logged()) + self.failUnless('TypeError' in self.logged()) + self.failUnless('bad type' in self.logged()) + + def test_err_failure(self): + log.err(failure.Failure(TypeError("bad type"))) + self.failUnless('Unhandled Error' in self.logged()) + self.failUnless('TypeError' in self.logged()) + self.failUnless('bad type' in self.logged()) + + +class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest): + + encoding = 'latin-1' + + def test_msg_encoding(self): + log.msg(u"Price: \xa3100") + logged = self.f.getvalue().strip()[25:] + self.assertEqual(self.logged(), "[-] INFO: Price: \xa3100") + + def test_err_why_encoding(self): + log.err(TypeError("bad type"), u"\xa3") + self.assertEqual(self.first_log_line(), "[-] ERROR: \xa3") + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/tests/test_utils_misc.py b/scrapy/tests/test_utils_misc.py deleted file mode 100644 index 806225c7c..000000000 --- a/scrapy/tests/test_utils_misc.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -from cStringIO import StringIO - -from scrapy.utils.misc import load_object, arg_to_iter - -class UtilsMiscTestCase(unittest.TestCase): - - def test_load_object(self): - obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object - self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') - self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - - def test_arg_to_iter(self): - assert hasattr(arg_to_iter(None), '__iter__') - assert hasattr(arg_to_iter(100), '__iter__') - assert hasattr(arg_to_iter('lala'), '__iter__') - assert hasattr(arg_to_iter([1,2,3]), '__iter__') - assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') - - self.assertEqual(list(arg_to_iter(None)), []) - self.assertEqual(list(arg_to_iter('lala')), ['lala']) - self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) - self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) - self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_utils_misc/__init__.py b/scrapy/tests/test_utils_misc/__init__.py new file mode 100644 index 000000000..5c8a1d7d9 --- /dev/null +++ b/scrapy/tests/test_utils_misc/__init__.py @@ -0,0 +1,71 @@ +import sys +import os +import unittest +from cStringIO import StringIO + +from scrapy.utils.misc import load_object, arg_to_iter, walk_modules + +class UtilsMiscTestCase(unittest.TestCase): + + def test_load_object(self): + obj = load_object('scrapy.utils.misc.load_object') + assert obj is load_object + self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') + self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + + def test_walk_modules(self): + mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules') + expected = [ + 'scrapy.tests.test_utils_misc.test_walk_modules', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod1', + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + + mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod') + expected = [ + 'scrapy.tests.test_utils_misc.test_walk_modules.mod', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0', + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + + mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod1') + expected = [ + 'scrapy.tests.test_utils_misc.test_walk_modules.mod1', + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + + self.assertRaises(ImportError, walk_modules, 'nomodule999') + + def test_walk_modules_egg(self): + egg = os.path.join(os.path.dirname(__file__), 'test.egg') + sys.path.append(egg) + try: + mods = walk_modules('testegg') + expected = [ + 'testegg.spiders', + 'testegg.spiders.a', + 'testegg.spiders.b', + 'testegg' + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + finally: + sys.path.remove(egg) + + def test_arg_to_iter(self): + assert hasattr(arg_to_iter(None), '__iter__') + assert hasattr(arg_to_iter(100), '__iter__') + assert hasattr(arg_to_iter('lala'), '__iter__') + assert hasattr(arg_to_iter([1,2,3]), '__iter__') + assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') + + self.assertEqual(list(arg_to_iter(None)), []) + self.assertEqual(list(arg_to_iter('lala')), ['lala']) + self.assertEqual(list(arg_to_iter(100)), [100]) + self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) + self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_misc/test.egg b/scrapy/tests/test_utils_misc/test.egg new file mode 100644 index 000000000..238517694 Binary files /dev/null and b/scrapy/tests/test_utils_misc/test.egg differ diff --git a/scrapy/management/__init__.py b/scrapy/tests/test_utils_misc/test_walk_modules/__init__.py similarity index 100% rename from scrapy/management/__init__.py rename to scrapy/tests/test_utils_misc/test_walk_modules/__init__.py diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py b/scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py b/scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod1.py b/scrapy/tests/test_utils_misc/test_walk_modules/mod1.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/test_utils_signal.py b/scrapy/tests/test_utils_signal.py index 139d8a632..1a1534932 100644 --- a/scrapy/tests/test_utils_signal.py +++ b/scrapy/tests/test_utils_signal.py @@ -1,5 +1,7 @@ import unittest +from twisted.python import log as txlog + from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.signal import send_catch_log from scrapy import log @@ -20,12 +22,12 @@ class SignalUtilsTest(unittest.TestCase): assert arg == 'test' return "OK" - def log_received(message, level): + def log_received(event): handlers_called.add(log_received) - assert "test_handler_error" in message - assert level == log.ERROR + assert "test_handler_error" in event['message'][0] + assert event['logLevel'] == log.ERROR - dispatcher.connect(log_received, signal=log.logmessage_received) + txlog.addObserver(log_received) dispatcher.connect(test_handler_error, signal=test_signal) dispatcher.connect(test_handler_check, signal=test_signal) result = send_catch_log(test_signal, arg='test') @@ -37,7 +39,7 @@ class SignalUtilsTest(unittest.TestCase): self.assert_(isinstance(result[0][1], Exception)) self.assertEqual(result[1], (test_handler_check, "OK")) - dispatcher.disconnect(log_received, signal=log.logmessage_received) + txlog.removeObserver(log_received) dispatcher.disconnect(test_handler_error, signal=test_signal) dispatcher.disconnect(test_handler_check, signal=test_signal) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index af61b8e7f..058b94276 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -1,8 +1,8 @@ import unittest +from scrapy.spider import BaseSpider from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, \ url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, \ - urljoin_rfc - + urljoin_rfc, url_is_from_spider class UrlUtilsTest(unittest.TestCase): @@ -19,6 +19,15 @@ class UrlUtilsTest(unittest.TestCase): self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) + def test_url_is_from_any_domain(self): + spider = BaseSpider(name='example.com', allowed_domains=['example.org', 'example.net']) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider)) + self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) + self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) + def test_urljoin_rfc(self): self.assertEqual(urljoin_rfc('http://example.com/some/path', 'newpath/test'), 'http://example.com/some/newpath/test') diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ac24591c2..e61b7aeb4 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -2,6 +2,7 @@ import re import hashlib +from pkgutil import iter_modules from scrapy.utils.python import flatten from scrapy.utils.markup import remove_entities @@ -44,6 +45,27 @@ def load_object(path): return obj +def walk_modules(path, load=False): + """Loads a module and all its submodules from a the given module path and + returns them. If *any* module throws an exception while importing, that + exception is thrown back. + + For example: walk_modules('scrapy.utils') + """ + + mods = [] + mod = __import__(path, {}, {}, ['']) + mods.append(mod) + if hasattr(mod, '__path__'): + for _, subpath, ispkg in iter_modules(mod.__path__): + fullpath = path + '.' + subpath + if ispkg: + mods += walk_modules(fullpath) + else: + submod = __import__(fullpath, {}, {}, ['']) + mods.append(submod) + return mods + def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: diff --git a/scrapy/utils/mysql.py b/scrapy/utils/mysql.py deleted file mode 100644 index a7081f9ed..000000000 --- a/scrapy/utils/mysql.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Functions for dealing with databases""" - -import re - -import MySQLdb - -from scrapy.conf import settings -from scrapy import log - -mysql_uri_re = r"mysql:\/\/(?P[^:]+)(:(?P[^@]+))?@(?P[^/:]+)(:(?P\d+))?/(?P.*)$" - -def parse_uri(mysql_uri): - """Parse mysql URI and return a dict with its parameters""" - m = re.search(mysql_uri_re, mysql_uri) - if m: - d = m.groupdict() - if d['passwd'] is None: - del(d['passwd']) - if d['port'] is None: - del(d['port']) - else: - d['port'] = int(d['port']) - return d - -def mysql_connect(db_uri_or_dict, **kwargs): - """Connects to a MySQL DB given a mysql URI""" - if isinstance(db_uri_or_dict, dict): - d = db_uri_or_dict - else: - d = parse_uri(db_uri_or_dict) - if not d: - return - d.update(settings.get("MYSQL_CONNECTION_SETTINGS", {})) - d.update(kwargs) - log.msg("Connecting to MySQL: db=%r, host=%r, user=%r" % (d['db'], \ - d['host'], d['user']), level=log.DEBUG) - conn = MySQLdb.connect(**d) - - return conn diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 196c03727..31e21789e 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, spider.allowed_domains) + return url_is_from_any_domain(url, [spider.name] + spider.allowed_domains) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref diff --git a/setup.py b/setup.py index 9e95e5666..fba882ad4 100644 --- a/setup.py +++ b/setup.py @@ -75,14 +75,7 @@ if len(sys.argv) > 1 and sys.argv[1] == 'bdist_wininst': file_info[0] = '\\PURELIB\\%s' % file_info[0] # Dynamically calculate the version based on scrapy.__version__ -version = __import__('scrapy').__version__ -if 'dev' in version: - try: - from subprocess import Popen, PIPE - hgrev = Popen(['hg', 'tip', '--template={rev}'], stdout=PIPE).communicate()[0] - version = version.replace('-dev', '-r%s' % hgrev) - except: - pass +version = ".".join(map(str, __import__('scrapy').version_info[:2])) setup_args = { 'name': 'Scrapy',