mirror of https://github.com/scrapy/scrapy.git
Automated merge with http://hg.scrapy.org/scrapy-0.9
This commit is contained in:
Pablo Hoffman
commit
49851d7f55
45
Makefile
45
Makefile
|
|
@ -1,45 +0,0 @@
|
|||
VERSION ?= $(shell python -c "import scrapy; print '.'.join(map(str, scrapy.version_info[:2]))")
|
||||
DEBREV ?= 1
|
||||
HGREV ?= $(shell hg log -l1 --template '{rev}')
|
||||
FULL_VERSION ?= $(VERSION)~r$(HGREV)
|
||||
BUILDBASE ?= $(shell pwd)/build/deb
|
||||
BUILDDIR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig
|
||||
BUILDTAR ?= $(BUILDBASE)/scrapy_$(FULL_VERSION).orig.tar.gz
|
||||
|
||||
help:
|
||||
@echo 'Available targets:'
|
||||
@echo ' deb-binary - build debian binary package'
|
||||
@echo ' deb-source - build debian source package'
|
||||
@echo ' deb-all - build source and binary debian packages'
|
||||
@echo ' tarball - build source tarball'
|
||||
@echo ' sign - sign release'
|
||||
|
||||
deb-binary: deb-prepare
|
||||
cd $(BUILDDIR); debuild -i -us -uc -b
|
||||
|
||||
deb-source: deb-prepare
|
||||
cd $(BUILDDIR); debuild -i -us -uc -S
|
||||
|
||||
deb-all: deb-prepare
|
||||
cd $(BUILDDIR); debuild -i -us -uc
|
||||
|
||||
deb-prepare:
|
||||
@if [ -d $(BUILDBASE) ]; then \
|
||||
rm -rf $(BUILDBASE); \
|
||||
fi;
|
||||
mkdir -p $(BUILDBASE)
|
||||
hg archive -t tgz -X .hgtags $(BUILDTAR)
|
||||
tar zxf $(BUILDTAR) -C $(BUILDBASE)
|
||||
rm -f $(BUILDDIR)/Makefile # to avoid confusing dh_auto_build
|
||||
cp -r debian $(BUILDDIR)
|
||||
cd $(BUILDDIR); debchange -m -D unstable --force-distribution -v $(FULL_VERSION)-$(DEBREV) "Automatic build"
|
||||
|
||||
tarball:
|
||||
hg purge --all
|
||||
python setup.py sdist
|
||||
|
||||
sign:
|
||||
md5sum dist/Scrapy-* > dist/MD5SUMS
|
||||
sha1sum dist/Scrapy-* > dist/SHA1SUMS
|
||||
gpg -ba dist/MD5SUMS
|
||||
gpg -ba dist/SHA1SUMS
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
scrapy (0.9~r2000-1) unstable; urgency=low
|
||||
scrapy (0.10) unstable; urgency=low
|
||||
|
||||
* Initial release.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
# Environment variables to use in Scrapy service
|
||||
|
||||
#export PYTHONPATH=/var/lib/scrapy/myproject
|
||||
|
|
@ -2,17 +2,22 @@ Source: scrapy
|
|||
Section: python
|
||||
Priority: optional
|
||||
Maintainer: Insophia Team <info@insophia.com>
|
||||
Build-Depends: debhelper (>= 7.0.50), python-twisted
|
||||
Build-Depends-Indep: python-support, python, python-sphinx, libjs-jquery
|
||||
Build-Depends: debhelper (>= 7.0.50), python (>=2.5), python-twisted
|
||||
Standards-Version: 3.8.4
|
||||
Homepage: http://scrapy.org/
|
||||
|
||||
Package: scrapy
|
||||
Architecture: all
|
||||
Depends: ${misc:Depends}, ${python:Depends}, python-libxml2, python-twisted, python-openssl
|
||||
Provides: ${python:Provides}
|
||||
Description: Python web scraping and crawling framework
|
||||
Depends: ${python:Depends}, python-libxml2, python-twisted, python-openssl
|
||||
Description: Python web crawling and scraping framework
|
||||
Scrapy is a fast high-level screen scraping and web crawling framework,
|
||||
used to crawl websites and extract structured data from their pages.
|
||||
It can be used for a wide range of purposes, from data mining to
|
||||
monitoring and automated testing.
|
||||
|
||||
Package: scrapy-service
|
||||
Architecture: all
|
||||
Depends: scrapy
|
||||
Description: Scrapy Service
|
||||
This package provides support for running Scrapy as a system service,
|
||||
controlled through an upstart script.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
This package was debianized by Insophia <info@insophia.com>.
|
||||
This package was debianized by the Insophia <info@insophia.com>.
|
||||
|
||||
It was downloaded from http://scrapy.org
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
2
|
||||
|
|
@ -1 +1 @@
|
|||
2.5-2.6
|
||||
2.5-
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
#!/usr/bin/make -f
|
||||
# -*- makefile -*-
|
||||
|
||||
export DH_ALWAYS_EXCLUDE=license.txt:_sources/:.buildinfo
|
||||
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_auto_install:
|
||||
dh_auto_install
|
||||
mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl
|
||||
mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-ws.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ws
|
||||
mv $(CURDIR)/debian/scrapy/usr/bin/scrapy-sqs.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-sqs
|
||||
mkdir -p $(CURDIR)/debian/scrapy/usr/bin
|
||||
mv $(CURDIR)/debian/tmp/usr/bin/scrapy-ctl.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ctl
|
||||
mv $(CURDIR)/debian/tmp/usr/bin/scrapy-ws.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-ws
|
||||
mv $(CURDIR)/debian/tmp/usr/bin/scrapy-sqs.py $(CURDIR)/debian/scrapy/usr/bin/scrapy-sqs
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
# Defaults for Scrapy service
|
||||
|
||||
export PYTHONPATH=/etc/scrapy
|
||||
export SCRAPY_SETTINGS_MODULE=service_conf
|
||||
|
|
@ -1,4 +1,2 @@
|
|||
usr/bin
|
||||
usr/share/scrapy
|
||||
var/lib/scrapy
|
||||
var/log/scrapy
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
debian/service_conf.py etc/scrapy
|
||||
extras/scrapy.tac usr/share/scrapy
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
new-package-should-close-itp-bug
|
||||
script-in-etc-init.d-not-registered-via-update-rc.d /etc/init.d/scrapy-service
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
# Create user to run the service as
|
||||
if [ -z "`id -u scrapy 2> /dev/null`" ]; then
|
||||
adduser --system --home /var/lib/scrapy --gecos "scrapy" \
|
||||
--no-create-home --disabled-password \
|
||||
--quiet scrapy || true
|
||||
fi
|
||||
if [ ! -d /var/run/scrapy ]; then
|
||||
mkdir /var/run/scrapy
|
||||
chown scrapy:nogroup /var/run/scrapy
|
||||
fi
|
||||
|
||||
chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy
|
||||
;;
|
||||
|
||||
abort-upgrade|abort-remove|abort-deconfigure)
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "postinst called with unknown argument \`$1'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
if [ purge = "$1" ]; then
|
||||
|
|
@ -10,9 +11,6 @@ if [ purge = "$1" ]; then
|
|||
rm -rf /var/run/scrapy
|
||||
fi
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
case "$1" in
|
||||
remove|upgrade|deconfigure)
|
||||
rm -f /etc/scrapy/service_conf.pyc
|
||||
;;
|
||||
|
||||
failed-upgrade)
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "prerm called with unknown argument \`$1'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
# Scrapy service
|
||||
|
||||
start on runlevel [2345]
|
||||
stop on runlevel [06]
|
||||
|
||||
script
|
||||
[ -r /etc/default/scrapy-service ] && . /etc/default/scrapy-service
|
||||
exec twistd -ny /usr/share/scrapy/scrapy.tac \
|
||||
-u scrapy -g nogroup \
|
||||
-l /var/log/scrapy/service.log \
|
||||
--pidfile /var/run/scrapy/scrapy.pid
|
||||
end script
|
||||
|
|
@ -1,3 +1 @@
|
|||
debian/conf/service_conf.py etc/scrapy
|
||||
debian/conf/environment etc/scrapy
|
||||
extras/scrapy.tac usr/share/scrapy
|
||||
debian/tmp/usr
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
new-package-should-close-itp-bug
|
||||
binary-without-manpage usr/bin/scrapy-sqs
|
||||
binary-without-manpage usr/bin/scrapy-ws
|
||||
extra-license-file usr/share/pyshared/scrapy/xlib/pydispatch/license.txt
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
#!/bin/sh
|
||||
# postinst script for scrapy
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <postinst> `configure' <most-recently-configured-version>
|
||||
# * <old-postinst> `abort-upgrade' <new version>
|
||||
# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
|
||||
# <new-version>
|
||||
# * <postinst> `abort-remove'
|
||||
# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
|
||||
# <failed-install-package> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see http://www.debian.org/doc/debian-policy/ or
|
||||
# the debian-policy package
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
# Fix for old packages not managing pyc files
|
||||
for d in /usr/lib/python*/site-packages/scrapy ; do
|
||||
if [ -d "$d" ] ; then
|
||||
echo "Removing old python byte-compiled files in $d"
|
||||
rm -rf "$d"
|
||||
fi
|
||||
done
|
||||
|
||||
# Create user to run the service as
|
||||
if [ -z "`id -u scrapy 2> /dev/null`" ]; then
|
||||
adduser --system --home /var/lib/scrapy --gecos "scrapy" \
|
||||
--no-create-home --disabled-password \
|
||||
--quiet scrapy || true
|
||||
fi
|
||||
if [ ! -d /var/run/scrapy ]; then
|
||||
mkdir /var/run/scrapy
|
||||
chown scrapy:nogroup /var/run/scrapy
|
||||
fi
|
||||
|
||||
chown scrapy:nogroup /var/log/scrapy /var/lib/scrapy /var/run/scrapy
|
||||
;;
|
||||
|
||||
abort-upgrade|abort-remove|abort-deconfigure)
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "postinst called with unknown argument \`$1'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# Scrapy service
|
||||
|
||||
description "Scrapy web crawler"
|
||||
author "Pablo Hoffman <pablo@pablohoffman.com>"
|
||||
|
||||
start on runlevel [2345]
|
||||
stop on runlevel [!2345]
|
||||
|
||||
script
|
||||
[ -r /etc/scrapy/environment ] && . /etc/scrapy/environment
|
||||
export PYTHONPATH=/etc/scrapy:$PYTHONPATH
|
||||
export SCRAPY_SETTINGS_MODULE=service_conf
|
||||
exec twistd -ny /usr/share/scrapy/scrapy.tac \
|
||||
-u scrapy -g nogroup \
|
||||
-l /var/log/scrapy/service.log \
|
||||
--pidfile /var/run/scrapy/scrapy.pid
|
||||
end script
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
version=3
|
||||
http://scrapy.org/releases/(\d\.\d)/Scrapy-([\d\.]+)\.tar\.gz
|
||||
|
|
@ -138,8 +138,6 @@ This is the code for our first Spider, save it in a file named
|
|||
def parse(self, response):
|
||||
filename = response.url.split("/")[-2]
|
||||
open(filename, 'wb').write(response.body)
|
||||
|
||||
SPIDER = DmozSpider()
|
||||
|
||||
Crawling
|
||||
--------
|
||||
|
|
@ -370,8 +368,6 @@ Let's add this code to our spider::
|
|||
link = site.select('a/@href').extract()
|
||||
desc = site.select('text()').extract()
|
||||
print title, link, desc
|
||||
|
||||
SPIDER = DmozSpider()
|
||||
|
||||
Now try crawling the dmoz.org domain again and you'll see sites being printed
|
||||
in your output, run::
|
||||
|
|
@ -418,8 +414,6 @@ scraped so far, the code for our Spider should be like this::
|
|||
item['desc'] = site.select('text()').extract()
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
SPIDER = DmozSpider()
|
||||
|
||||
Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s::
|
||||
|
||||
|
|
|
|||
|
|
@ -177,7 +177,9 @@ DefaultHeadersMiddleware
|
|||
.. class:: DefaultHeadersMiddleware
|
||||
|
||||
This middleware sets all default requests headers specified in the
|
||||
:setting:`DEFAULT_REQUEST_HEADERS` setting.
|
||||
:setting:`DEFAULT_REQUEST_HEADERS` setting plus those found in spider
|
||||
``default_request_headers`` attribute. Spider headers has precedence over
|
||||
global headers.
|
||||
|
||||
HttpAuthMiddleware
|
||||
------------------
|
||||
|
|
@ -276,6 +278,17 @@ Number of seconds to use for HTTP cache expiration. Requests that were cached
|
|||
before this time will be re-downloaded. If zero, cached requests will always
|
||||
expire. A negative number means requests will never expire.
|
||||
|
||||
.. setting:: HTTPCACHE_IGNORE_HTTP_CODES
|
||||
|
||||
HTTPCACHE_IGNORE_HTTP_CODES
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 0.10
|
||||
|
||||
Default: ``[]``
|
||||
|
||||
Don't cache response with these HTTP codes.
|
||||
|
||||
.. setting:: HTTPCACHE_IGNORE_MISSING
|
||||
|
||||
HTTPCACHE_IGNORE_MISSING
|
||||
|
|
|
|||
|
|
@ -93,8 +93,6 @@ This is how the spider would look so far::
|
|||
# write the category page data extraction code here
|
||||
pass
|
||||
|
||||
SPIDER = GoogleDirectorySpider()
|
||||
|
||||
|
||||
Extracting the data
|
||||
===================
|
||||
|
|
|
|||
|
|
@ -53,10 +53,6 @@ scrapy.log module
|
|||
.. module:: scrapy.log
|
||||
:synopsis: Logging facility
|
||||
|
||||
.. attribute:: log_level
|
||||
|
||||
The current log level being used
|
||||
|
||||
.. attribute:: started
|
||||
|
||||
A boolean which is ``True`` is logging has been started or ``False`` otherwise.
|
||||
|
|
@ -81,7 +77,7 @@ scrapy.log module
|
|||
setting will be used.
|
||||
:type logstdout: boolean
|
||||
|
||||
.. function:: msg(message, level=INFO, component=BOT_NAME, spider=None)
|
||||
.. function:: msg(message, level=INFO, spider=None)
|
||||
|
||||
Log a message
|
||||
|
||||
|
|
@ -91,24 +87,11 @@ scrapy.log module
|
|||
:param level: the log level for this message. See
|
||||
:ref:`topics-logging-levels`.
|
||||
|
||||
:param component: the component to use for logging, it defaults to
|
||||
:setting:`BOT_NAME`
|
||||
:type component: str
|
||||
|
||||
:param spider: the spider to use for logging this message. This parameter
|
||||
should always be used when logging things related to a particular
|
||||
spider.
|
||||
:type spider: :class:`~scrapy.spider.BaseSpider` object
|
||||
|
||||
.. function:: exc(message, level=ERROR, component=BOT_NAME, spider=None)
|
||||
|
||||
Log an exception. Similar to ``msg()`` but it also appends a stack trace
|
||||
report using `traceback.format_exc`.
|
||||
|
||||
.. _traceback.format_exc: http://docs.python.org/library/traceback.html#traceback.format_exc
|
||||
|
||||
It accepts the same parameters as the :func:`msg` function.
|
||||
|
||||
.. data:: CRITICAL
|
||||
|
||||
Log level for critical errors
|
||||
|
|
|
|||
|
|
@ -177,8 +177,6 @@ Let's see an example::
|
|||
def parse(self, response):
|
||||
self.log('A response from %s just arrived!' % response.url)
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
||||
Another example returning multiples Requests and Items from a single callback::
|
||||
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
|
|
@ -203,8 +201,6 @@ Another example returning multiples Requests and Items from a single callback::
|
|||
for url in hxs.select('//a/@href').extract():
|
||||
yield Request(url, callback=self.parse)
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
||||
.. module:: scrapy.contrib.spiders
|
||||
:synopsis: Collection of generic spiders
|
||||
|
||||
|
|
@ -230,7 +226,7 @@ CrawlSpider
|
|||
|
||||
Crawling rules
|
||||
~~~~~~~~~~~~~~
|
||||
.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None)
|
||||
.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
|
||||
|
||||
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
|
||||
defines how links will be extracted from each crawled page.
|
||||
|
|
@ -253,6 +249,10 @@ Crawling rules
|
|||
of links extracted from each response using the specified ``link_extractor``.
|
||||
This is mainly used for filtering purposes.
|
||||
|
||||
``process_request`` is a callable, or a string (in which case a method from
|
||||
the spider object with that name will be used) which will be called with
|
||||
every request extracted by this rule, and must return a request or None (to
|
||||
filter out the request).
|
||||
|
||||
CrawlSpider example
|
||||
-------------------
|
||||
|
|
@ -288,8 +288,6 @@ Let's now take a look at an example CrawlSpider with rules::
|
|||
item['description'] = hxs.select('//td[@id="item_description"]/text()').extract()
|
||||
return item
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
||||
|
||||
This spider would start crawling example.com's home page, collecting category
|
||||
links, and item links, parsing the latter with the ``parse_item`` method. For
|
||||
|
|
@ -405,8 +403,6 @@ These spiders are pretty easy to use, let's have at one example::
|
|||
item['description'] = node.select('description').extract()
|
||||
return item
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
||||
Basically what we did up there was creating a spider that downloads a feed from
|
||||
the given ``start_urls``, and then iterates through each of its ``item`` tags,
|
||||
prints them out, and stores some random data in an :class:`~scrapy.item.Item`.
|
||||
|
|
@ -462,6 +458,3 @@ Let's see an example similar to the previous one, but using a
|
|||
item['name'] = row['name']
|
||||
item['description'] = row['description']
|
||||
return item
|
||||
|
||||
SPIDER = MySpider()
|
||||
|
||||
|
|
|
|||
|
|
@ -36,6 +36,3 @@ class GoogleDirectorySpider(CrawlSpider):
|
|||
|
||||
# Here we populate the item and yield it
|
||||
yield l.load_item()
|
||||
|
||||
SPIDER = GoogleDirectorySpider()
|
||||
|
||||
|
|
|
|||
|
|
@ -137,5 +137,3 @@ class ImdbSiteSpider(CrawlSpider):
|
|||
def _urljoin(self, response, url):
|
||||
"""Helper to convert relative urls to absolute"""
|
||||
return urljoin_rfc(response.url, url, response.encoding)
|
||||
|
||||
SPIDER = ImdbSiteSpider()
|
||||
|
|
|
|||
|
|
@ -38,5 +38,3 @@ class GoogleDirectorySpider(CrawlSpider):
|
|||
|
||||
# Here we populate the item and yield it
|
||||
yield l.load_item()
|
||||
|
||||
SPIDER = GoogleDirectorySpider()
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
Scrapy - a screen scraping framework written in Python
|
||||
"""
|
||||
|
||||
version_info = (0, 9, 0, '')
|
||||
__version__ = "0.9"
|
||||
version_info = (0, 10, 0, 'dev')
|
||||
__version__ = "0.10-dev"
|
||||
|
||||
import sys, os, warnings
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,9 @@ class Command(ScrapyCommand):
|
|||
return
|
||||
else:
|
||||
spider = spiders.create_for_request(request)
|
||||
if spider is None:
|
||||
log.msg('Unable to find spider for URL: %s' % args[0], log.ERROR)
|
||||
return
|
||||
|
||||
scrapymanager.configure()
|
||||
scrapymanager.queue.append_request(request, spider)
|
||||
|
|
@ -112,7 +115,7 @@ class Command(ScrapyCommand):
|
|||
for rule in spider.rules:
|
||||
if rule.link_extractor.matches(response.url) \
|
||||
and rule.callback:
|
||||
|
||||
|
||||
items, links = self.run_callback(spider,
|
||||
response, rule.callback,
|
||||
args, opts)
|
||||
|
|
|
|||
|
|
@ -123,6 +123,7 @@ HTTPCACHE_DIR = ''
|
|||
HTTPCACHE_IGNORE_MISSING = False
|
||||
HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage'
|
||||
HTTPCACHE_EXPIRATION_SECS = 0
|
||||
HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
|
||||
ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
|
||||
|
||||
|
|
@ -190,7 +191,7 @@ SCHEDULER_ORDER = 'DFO'
|
|||
|
||||
SERVICE_QUEUE = 'scrapy.core.queue.KeepAliveExecutionQueue'
|
||||
|
||||
SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager'
|
||||
SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.SpiderManager'
|
||||
|
||||
SPIDER_MIDDLEWARES = {}
|
||||
|
||||
|
|
@ -233,10 +234,6 @@ USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
|
|||
TELNETCONSOLE_ENABLED = 1
|
||||
TELNETCONSOLE_PORT = 6023 # if None, uses a dynamic port
|
||||
|
||||
WEBCONSOLE_ENABLED = True
|
||||
WEBCONSOLE_PORT = 6080
|
||||
WEBCONSOLE_LOGFILE = None
|
||||
|
||||
WEBSERVICE_ENABLED = True
|
||||
WEBSERVICE_LOGFILE = None
|
||||
WEBSERVICE_PORT = 6080
|
||||
|
|
|
|||
|
|
@ -3,15 +3,27 @@ DefaultHeaders downloader middleware
|
|||
|
||||
See documentation in docs/topics/downloader-middleware.rst
|
||||
"""
|
||||
|
||||
from scrapy.conf import settings
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.core import signals
|
||||
|
||||
|
||||
class DefaultHeadersMiddleware(object):
|
||||
|
||||
def __init__(self):
|
||||
self.default_headers = settings.get('DEFAULT_REQUEST_HEADERS')
|
||||
self.global_default_headers = settings.get('DEFAULT_REQUEST_HEADERS')
|
||||
self._default_headers = {}
|
||||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
for k, v in self.default_headers.iteritems():
|
||||
for k, v in self._default_headers[spider].iteritems():
|
||||
if v:
|
||||
request.headers.setdefault(k, v)
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self._default_headers[spider] = dict(self.global_default_headers,
|
||||
**getattr(spider, 'default_request_headers', {}))
|
||||
|
||||
def spider_closed(self, spider):
|
||||
self._default_headers.pop(spider)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ class HttpCacheMiddleware(object):
|
|||
def __init__(self, settings=conf.settings):
|
||||
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
|
||||
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
|
||||
self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
|
||||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
|
||||
|
|
@ -35,17 +36,20 @@ class HttpCacheMiddleware(object):
|
|||
if not self.is_cacheable(request):
|
||||
return
|
||||
response = self.storage.retrieve_response(spider, request)
|
||||
if response:
|
||||
if response and self.is_cacheable_response(response):
|
||||
response.flags.append('cached')
|
||||
return response
|
||||
elif self.ignore_missing:
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if self.is_cacheable(request):
|
||||
if self.is_cacheable(request) and self.is_cacheable_response(response):
|
||||
self.storage.store_response(spider, request, response)
|
||||
return response
|
||||
|
||||
def is_cacheable_response(self, response):
|
||||
return response.status not in self.ignore_http_codes
|
||||
|
||||
def is_cacheable(self, request):
|
||||
return urlparse_cached(request).scheme in ['http', 'https']
|
||||
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ class TemplatePageParser(InstanceLearningParser):
|
|||
def _handle_unpaired_tag(self, html_tag):
|
||||
if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
|
||||
self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
|
||||
elif self._read_bool_template_attribute(html_tag, "ignore-beneath") and html_tag.tag == "img":
|
||||
elif self._read_bool_template_attribute(html_tag, "ignore-beneath"):
|
||||
self.ignored_regions.append((self.next_tag_index, None))
|
||||
jannotation = self._read_template_annotation(html_tag)
|
||||
if jannotation:
|
||||
|
|
|
|||
|
|
@ -3,18 +3,16 @@ SpiderManager is the class which locates and manages all website-specific
|
|||
spiders
|
||||
"""
|
||||
|
||||
import sys
|
||||
import inspect
|
||||
|
||||
from twisted.plugin import getCache
|
||||
from twisted.python.rebuild import rebuild
|
||||
|
||||
from scrapy.spider.models import ISpider
|
||||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
from scrapy.utils.misc import walk_modules
|
||||
from scrapy.spider import BaseSpider
|
||||
|
||||
class TwistedPluginSpiderManager(object):
|
||||
"""Spider manager based in Twisted Plugin System"""
|
||||
|
||||
class SpiderManager(object):
|
||||
|
||||
def __init__(self):
|
||||
self.loaded = False
|
||||
|
|
@ -25,9 +23,7 @@ class TwistedPluginSpiderManager(object):
|
|||
spider arguments. If the sipder name is not found, it raises a
|
||||
KeyError.
|
||||
"""
|
||||
spider = self._spiders[spider_name]
|
||||
spider.__dict__.update(spider_kwargs)
|
||||
return spider
|
||||
return self._spiders[spider_name](**spider_kwargs)
|
||||
|
||||
def find_by_request(self, request):
|
||||
"""Returns list of spiders names that match the given Request"""
|
||||
|
|
@ -60,47 +56,23 @@ class TwistedPluginSpiderManager(object):
|
|||
return self._spiders.keys()
|
||||
|
||||
def load(self, spider_modules=None):
|
||||
"""Load spiders from module directory."""
|
||||
"""Load spiders from spider_modules or SPIDER_MODULES setting."""
|
||||
if spider_modules is None:
|
||||
spider_modules = settings.getlist('SPIDER_MODULES')
|
||||
self.spider_modules = spider_modules
|
||||
self._spiders = {}
|
||||
|
||||
modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules]
|
||||
for module in modules:
|
||||
for spider in self._getspiders(ISpider, module):
|
||||
ISpider.validateInvariants(spider)
|
||||
self._spiders[spider.name] = spider
|
||||
self._spiders = {}
|
||||
for name in self.spider_modules:
|
||||
for module in walk_modules(name):
|
||||
self._load_spiders(module)
|
||||
self.loaded = True
|
||||
|
||||
def _getspiders(self, interface, package):
|
||||
"""This is an override of twisted.plugin.getPlugin, because we're
|
||||
interested in catching exceptions thrown when loading spiders such as
|
||||
KeyboardInterrupt
|
||||
"""
|
||||
try:
|
||||
allDropins = getCache(package)
|
||||
for dropin in allDropins.itervalues():
|
||||
for plugin in dropin.plugins:
|
||||
adapted = interface(plugin, None)
|
||||
if adapted is not None:
|
||||
yield adapted
|
||||
except KeyboardInterrupt:
|
||||
sys.stderr.write("Interrupted while loading Scrapy spiders\n")
|
||||
sys.exit(2)
|
||||
def _load_spiders(self, module):
|
||||
for obj in vars(module).itervalues():
|
||||
if inspect.isclass(obj) and issubclass(obj, BaseSpider):
|
||||
name = getattr(obj, 'name', None)
|
||||
if name is not None:
|
||||
self._spiders[name] = obj
|
||||
|
||||
def close_spider(self, spider):
|
||||
"""Reload spider module to release any resources held on to by the
|
||||
spider
|
||||
"""
|
||||
name = spider.name
|
||||
if name not in self._spiders:
|
||||
return
|
||||
spider = self._spiders[name]
|
||||
module_name = spider.__module__
|
||||
module = sys.modules[module_name]
|
||||
if hasattr(module, 'SPIDER'):
|
||||
log.msg("Reloading module %s" % module_name, spider=spider, \
|
||||
level=log.DEBUG)
|
||||
new_module = rebuild(module, doLog=0)
|
||||
self._spiders[name] = new_module.SPIDER
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -13,84 +13,40 @@ from scrapy.utils.spider import iterate_spider_output
|
|||
from scrapy.contrib.spiders.init import InitSpider
|
||||
from scrapy.conf import settings
|
||||
|
||||
def identity(x):
|
||||
return x
|
||||
|
||||
class Rule(object):
|
||||
"""
|
||||
A rule for crawling, which receives the following constructor arguments:
|
||||
|
||||
link_extractor (required)
|
||||
A LinkExtractor which defines the policy for extracting links
|
||||
callback (optional)
|
||||
A function to use to process the page once it has been downloaded. If
|
||||
callback is omitted the page is not procesed, just crawled. If callback
|
||||
is a string (instead of callable) a method of the spider class with that
|
||||
name is used as the callback function
|
||||
cb_kwargs (optional)
|
||||
A dict specifying keyword arguments to pass to the callback function
|
||||
follow (optional)
|
||||
If True, links will be followed from the pages crawled by this rule.
|
||||
It defaults to True when no callback is specified or False when a
|
||||
callback is specified
|
||||
process_links (optional)
|
||||
Can be either a callable, or a string with the name of a method defined
|
||||
in the spider's class.
|
||||
This method will be called with the list of extracted links matching
|
||||
this rule (if any) and must return another list of links.
|
||||
"""
|
||||
|
||||
def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None):
|
||||
def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
|
||||
self.link_extractor = link_extractor
|
||||
self.callback = callback
|
||||
self.cb_kwargs = cb_kwargs or {}
|
||||
self.process_links = process_links
|
||||
self.process_request = process_request
|
||||
if follow is None:
|
||||
self.follow = False if callback else True
|
||||
else:
|
||||
self.follow = follow
|
||||
|
||||
class CrawlSpider(InitSpider):
|
||||
"""
|
||||
Class for spiders that crawl over web pages and extract/parse their links
|
||||
given some crawling rules.
|
||||
|
||||
These crawling rules are established by setting the 'rules' class attribute,
|
||||
which is a tuple of Rule objects.
|
||||
When the spider is running, it iterates over these rules with each response
|
||||
and do what it has to (extract links if follow=True, and return items/requests if
|
||||
there's a parsing method defined in the rule).
|
||||
"""
|
||||
rules = ()
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
"""Constructor takes care of compiling rules"""
|
||||
super(CrawlSpider, self).__init__(*a, **kw)
|
||||
self._compile_rules()
|
||||
|
||||
def parse(self, response):
|
||||
"""This function is called by the framework core for all the
|
||||
start_urls. Do not override this function, override parse_start_url
|
||||
instead."""
|
||||
return self._response_downloaded(response, self.parse_start_url, cb_kwargs={}, follow=True)
|
||||
|
||||
def parse_start_url(self, response):
|
||||
"""Overrideable callback function for processing start_urls. It must
|
||||
return a list of BaseItem and/or Requests"""
|
||||
return []
|
||||
|
||||
def process_results(self, response, results):
|
||||
"""This overridable method is called for each result (item or request)
|
||||
returned by the spider, and it's intended to perform any last time
|
||||
processing required before returning the results to the framework core,
|
||||
for example setting the item GUIDs. It receives a list of results and
|
||||
the response which originated that results. It must return a list
|
||||
of results (Items or Requests)."""
|
||||
return results
|
||||
|
||||
def _requests_to_follow(self, response):
|
||||
"""
|
||||
This method iterates over each of the spider's rules, extracts the links
|
||||
matching each case, filters them (if needed), and returns a list of unique
|
||||
requests per response.
|
||||
"""
|
||||
seen = set()
|
||||
for rule in self._rules:
|
||||
links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
|
||||
|
|
@ -102,14 +58,9 @@ class CrawlSpider(InitSpider):
|
|||
cb_kwargs=rule.cb_kwargs, follow=rule.follow)
|
||||
r = Request(url=link.url, callback=callback)
|
||||
r.meta['link_text'] = link.text
|
||||
yield r
|
||||
yield rule.process_request(r)
|
||||
|
||||
def _response_downloaded(self, response, callback, cb_kwargs, follow):
|
||||
"""
|
||||
This is were any response arrives, and were it's decided whether
|
||||
to extract links or not from it, and if it will be parsed or not.
|
||||
It returns a list of requests/items.
|
||||
"""
|
||||
if callback:
|
||||
cb_res = callback(response, **cb_kwargs) or ()
|
||||
cb_res = self.process_results(response, cb_res)
|
||||
|
|
@ -122,8 +73,6 @@ class CrawlSpider(InitSpider):
|
|||
|
||||
|
||||
def _compile_rules(self):
|
||||
"""Compile the crawling rules"""
|
||||
|
||||
def get_method(method):
|
||||
if callable(method):
|
||||
return method
|
||||
|
|
@ -134,3 +83,4 @@ class CrawlSpider(InitSpider):
|
|||
for rule in self._rules:
|
||||
rule.callback = get_method(rule.callback)
|
||||
rule.process_links = get_method(rule.process_links)
|
||||
rule.process_request = get_method(rule.process_request)
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
import warnings
|
||||
warnings.warn("Web console is deprecated. Consider using web service instead.", \
|
||||
DeprecationWarning, stacklevel=2)
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
"""
|
||||
Scheduler information module for Scrapy webconsole
|
||||
"""
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.utils.engine import get_engine_status
|
||||
from scrapy.management.web import banner
|
||||
|
||||
class EngineStatus(object):
|
||||
webconsole_id = 'enginestatus'
|
||||
webconsole_name = 'Engine status'
|
||||
|
||||
def __init__(self):
|
||||
from scrapy.management.web import webconsole_discover_module
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
s = banner(self)
|
||||
s += "<pre><code>\n"
|
||||
s += get_engine_status()
|
||||
s += "</pre></code>\n"
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return s
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
return self
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
"""
|
||||
Live statistics extension
|
||||
"""
|
||||
from datetime import datetime
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.management.web import banner, webconsole_discover_module
|
||||
|
||||
class SpiderStats(object):
|
||||
def __init__(self):
|
||||
self.scraped = 0
|
||||
self.crawled = 0
|
||||
self.started = None
|
||||
|
||||
class LiveStats(object):
|
||||
webconsole_id = 'livestats'
|
||||
webconsole_name = 'Spider live statistics of current run'
|
||||
|
||||
def __init__(self):
|
||||
self.domains = {}
|
||||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
|
||||
dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded)
|
||||
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def spider_opened(self, spider):
|
||||
pstats = SpiderStats()
|
||||
self.domains[spider] = pstats
|
||||
pstats.started = datetime.now().replace(microsecond=0)
|
||||
|
||||
def spider_closed(self, spider):
|
||||
del self.domains[spider]
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
self.domains[spider].scraped += 1
|
||||
|
||||
def response_downloaded(self, response, spider):
|
||||
# sometimes we download responses without opening/closing domains,
|
||||
# for example from scrapy shell
|
||||
if self.domains.get(spider):
|
||||
self.domains[spider].crawled += 1
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
sch = scrapymanager.engine.scheduler
|
||||
dwl = scrapymanager.engine.downloader
|
||||
|
||||
totdomains = totscraped = totcrawled = totscheduled = totactive = totdqueued = tottransf = 0
|
||||
s = banner(self)
|
||||
s += "<table border='1'>\n"
|
||||
s += "<tr><th>Spider</th><th>Items<br>Scraped</th><th>Pages<br>Crawled</th><th>Scheduler<br>Pending</th><th>Downloader<br/>Queued</th><th>Downloader<br/>Active</th><th>Downloader<br/>Transferring</th><th>Start time</th><th>Run time</th></tr>\n"
|
||||
for spider in sorted(self.domains.keys()):
|
||||
scheduled = len(sch.pending_requests[spider]) if spider in sch.pending_requests else 0
|
||||
active = len(dwl.sites[spider].active) if spider in dwl.sites else 0
|
||||
dqueued = len(dwl.sites[spider].queue) if spider in dwl.sites else 0
|
||||
transf = len(dwl.sites[spider].transferring) if spider in dwl.sites else 0
|
||||
stats = self.domains[spider]
|
||||
runtime = datetime.now() - stats.started
|
||||
|
||||
s += '<tr><td>%s</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td>%s</td><td>%s</td></tr>\n' % \
|
||||
(spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime))
|
||||
|
||||
totdomains += 1
|
||||
totscraped += stats.scraped
|
||||
totcrawled += stats.crawled
|
||||
totscheduled += scheduled
|
||||
totactive += active
|
||||
totdqueued += dqueued
|
||||
tottransf += transf
|
||||
s += '<tr><td><b>%d domains</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td align="right"><b>%d</b></td><td/><td/></tr>\n' % \
|
||||
(totdomains, totscraped, totcrawled, totscheduled, totdqueued, totactive, tottransf)
|
||||
s += "</table>\n"
|
||||
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return s
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
return self
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
"""
|
||||
Scheduler queue web console module
|
||||
|
||||
See documentation in docs/topics/extensions.rst
|
||||
"""
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.management.web import banner, webconsole_discover_module
|
||||
|
||||
class SchedulerQueue(object):
|
||||
webconsole_id = 'scheduler'
|
||||
webconsole_name = 'Scheduler queue'
|
||||
|
||||
def __init__(self):
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
return self
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
s = banner(self)
|
||||
s += "<ul>\n"
|
||||
for domain, request_queue in scrapymanager.engine.scheduler.pending_requests.iteritems():
|
||||
s += "<li>\n"
|
||||
s += "%s (<b>%s</b> requests)\n" % (domain, len(request_queue))
|
||||
s += "<ul>\n"
|
||||
for ((req, _), prio) in request_queue:
|
||||
s += "<li><a href='%s'>%s</a> (priority: %d)</li>\n" % (req.url, req.url, prio)
|
||||
s += "</ul>\n"
|
||||
s += "</li>\n"
|
||||
s += "</ul>\n"
|
||||
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return s
|
||||
|
|
@ -1,152 +0,0 @@
|
|||
"""
|
||||
Extensions for allowing spider control from web console
|
||||
"""
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.management.web import banner, webconsole_discover_module
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Spiderctl(object):
|
||||
webconsole_id = 'spiderctl'
|
||||
webconsole_name = 'Spider control panel'
|
||||
|
||||
def __init__(self):
|
||||
self.running = {}
|
||||
self.finished = set()
|
||||
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
|
||||
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def spider_opened(self, spider):
|
||||
self.running[spider.name] = spider
|
||||
|
||||
def spider_closed(self, spider):
|
||||
del self.running[spider.name]
|
||||
self.finished.add(spider.name)
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
if wc_request.args:
|
||||
changes = self.webconsole_control(wc_request)
|
||||
|
||||
self.scheduled = [s[0].name for s in scrapymanager.queue.spider_requests]
|
||||
self.idle = [d for d in self.enabled_spiders if d not in self.scheduled
|
||||
and d not in self.running
|
||||
and d not in self.finished]
|
||||
|
||||
s = banner(self)
|
||||
s += '<table border=1">\n'
|
||||
s += "<tr><th>Idle (%d)</th><th>Scheduled (%d)</th><th>Running (%d/%d)</th><th>Finished (%d)</th></tr>\n" % \
|
||||
(len(self.idle),
|
||||
len(self.scheduled),
|
||||
len(self.running),
|
||||
settings['CONCURRENT_SPIDERS'],
|
||||
len(self.finished))
|
||||
s += "<tr>\n"
|
||||
|
||||
# idle
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="add_pending_spiders" multiple="multiple">\n'
|
||||
for name in sorted(self.idle):
|
||||
s += "<option>%s</option>\n" % name
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Schedule selected">\n'
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
# scheduled
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="remove_pending_spiders" multiple="multiple">\n'
|
||||
for name in self.scheduled:
|
||||
s += "<option>%s</option>\n" % name
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Remove selected">\n'
|
||||
s += '</form>\n'
|
||||
|
||||
s += "</td>\n"
|
||||
|
||||
# running
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="stop_running_spiders" multiple="multiple">\n'
|
||||
for name in sorted(self.running):
|
||||
s += "<option>%s</option>\n" % name
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Stop selected">\n'
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
# finished
|
||||
s += "<td valign='top'>\n"
|
||||
s += '<form method="post" action=".">\n'
|
||||
s += '<select name="rerun_finished_spiders" multiple="multiple">\n'
|
||||
for name in sorted(self.finished):
|
||||
s += "<option>%s</option>\n" % name
|
||||
s += '</select><br>\n'
|
||||
s += '<br />'
|
||||
s += '<input type="submit" value="Re-schedule selected">\n'
|
||||
s += '</form>\n'
|
||||
s += "</td>\n"
|
||||
|
||||
s += "</tr>\n"
|
||||
s += "</table>\n"
|
||||
|
||||
if wc_request.args:
|
||||
s += changes
|
||||
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return s
|
||||
|
||||
def webconsole_control(self, wc_request):
|
||||
args = wc_request.args
|
||||
s = "<hr />\n"
|
||||
|
||||
if "stop_running_spiders" in args:
|
||||
s += "<p>"
|
||||
stopped_spiders = []
|
||||
for name in args["stop_running_spiders"]:
|
||||
if name in self.running:
|
||||
scrapymanager.engine.close_spider(self.running[name])
|
||||
stopped_spiders.append(name)
|
||||
s += "Stopped spiders: <ul><li>%s</li></ul>" % "</li><li>".join(stopped_spiders)
|
||||
s += "</p>"
|
||||
if "remove_pending_spiders" in args:
|
||||
removed = []
|
||||
for name in args["remove_pending_spiders"]:
|
||||
q = scrapymanager.queue
|
||||
q.spider_requests = [x for x in q.spider_requests if x[0].name != name]
|
||||
if removed:
|
||||
s += "<p>"
|
||||
s += "Removed scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["remove_pending_spiders"])
|
||||
s += "</p>"
|
||||
if "add_pending_spiders" in args:
|
||||
for name in args["add_pending_spiders"]:
|
||||
if name not in scrapymanager.engine.scheduler.pending_requests:
|
||||
scrapymanager.queue.append_spider_name(name)
|
||||
s += "<p>"
|
||||
s += "Scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["add_pending_spiders"])
|
||||
s += "</p>"
|
||||
if "rerun_finished_spiders" in args:
|
||||
for name in args["rerun_finished_spiders"]:
|
||||
if name not in scrapymanager.engine.scheduler.pending_requests:
|
||||
scrapymanager.queue.append_spider_name(name)
|
||||
self.finished.remove(name)
|
||||
s += "<p>"
|
||||
s += "Re-scheduled finished spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["rerun_finished_spiders"])
|
||||
s += "</p>"
|
||||
|
||||
return s
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
self.enabled_spiders = spiders.list()
|
||||
return self
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.stats import stats
|
||||
from scrapy.management.web import banner, webconsole_discover_module
|
||||
|
||||
def stats_html_table(statsdict):
|
||||
s = ""
|
||||
s += "<table border='1'>\n"
|
||||
for kv in statsdict.iteritems():
|
||||
s += "<tr><th align='left'>%s</th><td>%s</td></tr>\n" % kv
|
||||
s += "</table>\n"
|
||||
return s
|
||||
|
||||
class StatsDump(object):
|
||||
webconsole_id = 'stats'
|
||||
webconsole_name = 'StatsCollector dump'
|
||||
|
||||
def __init__(self):
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
s = banner(self)
|
||||
s += "<h3>Global stats</h3>\n"
|
||||
s += stats_html_table(stats.get_stats())
|
||||
for spider, spider_stats in stats.iter_spider_stats():
|
||||
s += "<h3>%s</h3>\n" % spider.name
|
||||
s += stats_html_table(spider_stats)
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return str(s)
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
return self
|
||||
|
||||
122
scrapy/log.py
122
scrapy/log.py
|
|
@ -4,42 +4,64 @@ Scrapy logging facility
|
|||
See documentation in docs/topics/logging.rst
|
||||
"""
|
||||
import sys
|
||||
from traceback import format_exc
|
||||
import logging
|
||||
|
||||
from twisted.python import log
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.conf import settings
|
||||
from scrapy.utils.python import unicode_to_str
|
||||
|
||||
# Logging levels
|
||||
SILENT, CRITICAL, ERROR, WARNING, INFO, DEBUG = range(6)
|
||||
DEBUG = logging.DEBUG
|
||||
INFO = logging.INFO
|
||||
WARNING = logging.WARNING
|
||||
ERROR = logging.ERROR
|
||||
CRITICAL = logging.CRITICAL
|
||||
SILENT = CRITICAL + 1
|
||||
|
||||
level_names = {
|
||||
0: "SILENT",
|
||||
1: "CRITICAL",
|
||||
2: "ERROR",
|
||||
3: "WARNING",
|
||||
4: "INFO",
|
||||
5: "DEBUG",
|
||||
logging.DEBUG: "DEBUG",
|
||||
logging.INFO: "INFO",
|
||||
logging.WARNING: "WARNING",
|
||||
logging.ERROR: "ERROR",
|
||||
logging.CRITICAL: "CRITICAL",
|
||||
SILENT: "SILENT",
|
||||
}
|
||||
|
||||
BOT_NAME = settings['BOT_NAME']
|
||||
|
||||
# signal sent when log message is received
|
||||
# args: message, level, spider
|
||||
logmessage_received = object()
|
||||
|
||||
# default values
|
||||
log_level = DEBUG
|
||||
log_encoding = 'utf-8'
|
||||
|
||||
started = False
|
||||
|
||||
class ScrapyFileLogObserver(log.FileLogObserver):
|
||||
|
||||
def __init__(self, f, level=INFO, encoding='utf-8'):
|
||||
self.level = level
|
||||
self.encoding = encoding
|
||||
log.FileLogObserver.__init__(self, f)
|
||||
|
||||
def emit(self, eventDict):
|
||||
if eventDict.get('system') != 'scrapy':
|
||||
return
|
||||
level = eventDict.get('logLevel')
|
||||
if level < self.level:
|
||||
return
|
||||
spider = eventDict.get('spider')
|
||||
message = eventDict.get('message')
|
||||
lvlname = level_names.get(level, 'NOLEVEL')
|
||||
if message:
|
||||
message = [unicode_to_str(x, self.encoding) for x in message]
|
||||
message[0] = "%s: %s" % (lvlname, message[0])
|
||||
why = eventDict.get('why')
|
||||
if why:
|
||||
why = "%s: %s" % (lvlname, unicode_to_str(why, self.encoding))
|
||||
eventDict['message'] = message
|
||||
eventDict['why'] = why
|
||||
eventDict['system'] = spider.name if spider else '-'
|
||||
log.FileLogObserver.emit(self, eventDict)
|
||||
|
||||
def _get_log_level(level_name_or_id=None):
|
||||
if level_name_or_id is None:
|
||||
lvlname = settings['LOG_LEVEL'] or settings['LOGLEVEL']
|
||||
lvlname = settings['LOG_LEVEL']
|
||||
return globals()[lvlname]
|
||||
elif isinstance(level_name_or_id, int) and 0 <= level_name_or_id <= 5:
|
||||
elif isinstance(level_name_or_id, int):
|
||||
return level_name_or_id
|
||||
elif isinstance(level_name_or_id, basestring):
|
||||
return globals()[level_name_or_id]
|
||||
|
|
@ -47,53 +69,31 @@ def _get_log_level(level_name_or_id=None):
|
|||
raise ValueError("Unknown log level: %r" % level_name_or_id)
|
||||
|
||||
def start(logfile=None, loglevel=None, logstdout=None):
|
||||
"""Initialize and start logging facility"""
|
||||
global log_level, log_encoding, started
|
||||
|
||||
global started
|
||||
if started or not settings.getbool('LOG_ENABLED'):
|
||||
return
|
||||
log_level = _get_log_level(loglevel)
|
||||
log_encoding = settings['LOG_ENCODING']
|
||||
started = True
|
||||
|
||||
# set log observer
|
||||
if log.defaultObserver: # check twisted log not already started
|
||||
logfile = logfile or settings['LOG_FILE'] or settings['LOGFILE']
|
||||
loglevel = _get_log_level(loglevel)
|
||||
logfile = logfile or settings['LOG_FILE']
|
||||
file = open(logfile, 'a') if logfile else sys.stderr
|
||||
if logstdout is None:
|
||||
logstdout = settings.getbool('LOG_STDOUT')
|
||||
sflo = ScrapyFileLogObserver(file, loglevel, settings['LOG_ENCODING'])
|
||||
log.startLoggingWithObserver(sflo.emit, setStdout=logstdout)
|
||||
msg("Started project: %s" % settings['BOT_NAME'])
|
||||
|
||||
file = open(logfile, 'a') if logfile else sys.stderr
|
||||
log.startLogging(file, setStdout=logstdout)
|
||||
|
||||
def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None):
|
||||
"""Log message according to the level"""
|
||||
if level > log_level:
|
||||
return
|
||||
if domain is not None:
|
||||
def msg(message, level=INFO, **kw):
|
||||
if 'component' in kw:
|
||||
import warnings
|
||||
warnings.warn("'domain' argument of scrapy.log.msg() is deprecated, " \
|
||||
"use 'spider' argument instead", DeprecationWarning, stacklevel=2)
|
||||
dispatcher.send(signal=logmessage_received, message=message, level=level, \
|
||||
spider=spider)
|
||||
system = domain or (spider.name if spider else component)
|
||||
msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding)
|
||||
log.msg(msg_txt, system=system)
|
||||
warnings.warn("Argument `component` of scrapy.log.msg() is deprecated", \
|
||||
DeprecationWarning, stacklevel=2)
|
||||
kw.setdefault('system', 'scrapy')
|
||||
kw['logLevel'] = level
|
||||
log.msg(message, **kw)
|
||||
|
||||
def exc(message, level=ERROR, component=BOT_NAME, domain=None, spider=None):
|
||||
message = message + '\n' + format_exc()
|
||||
msg(message, level, component, domain, spider)
|
||||
|
||||
def err(_stuff=None, _why=None, **kwargs):
|
||||
if ERROR > log_level:
|
||||
return
|
||||
domain = kwargs.pop('domain', None)
|
||||
spider = kwargs.pop('spider', None)
|
||||
component = kwargs.pop('component', BOT_NAME)
|
||||
if domain is not None:
|
||||
import warnings
|
||||
warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \
|
||||
"use 'spider' argument instead", DeprecationWarning, stacklevel=2)
|
||||
kwargs['system'] = domain or (spider.name if spider else component)
|
||||
if _why:
|
||||
_why = unicode_to_str("ERROR: %s" % _why, log_encoding)
|
||||
log.err(_stuff, _why, **kwargs)
|
||||
def err(_stuff=None, _why=None, **kw):
|
||||
kw.setdefault('system', 'scrapy')
|
||||
kw['logLevel'] = kw.pop('level', ERROR)
|
||||
log.err(_stuff, _why, **kw)
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
import warnings
|
||||
warnings.warn("Scrapy web console is deprecated. Consider using web service instead.", \
|
||||
DeprecationWarning, stacklevel=2)
|
||||
|
||||
import re
|
||||
import socket
|
||||
from time import time
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.web import server, resource
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.conf import settings
|
||||
|
||||
# web management signals
|
||||
webconsole_discover_module = object()
|
||||
|
||||
urlpath_re = re.compile(r"^/(\w+)/")
|
||||
|
||||
def error404(module):
|
||||
return """
|
||||
<html>
|
||||
<head><title>404 Not Found</title></head>
|
||||
<body>
|
||||
<h1>Not found</h1>
|
||||
<p>Web console module not found: <b>%s</b></p>
|
||||
<p><a href="/">Back to main menu</a></p>
|
||||
</body>
|
||||
</html>
|
||||
""" % module
|
||||
|
||||
def banner(module=None):
|
||||
s = "<html>\n"
|
||||
s += "<head><title>Scrapy</title></head>\n"
|
||||
s += "<body>\n"
|
||||
s += "<h1><a href='/'>Scrapy web console</a></h1>\n"
|
||||
uptime = time() - scrapymanager.engine.start_time
|
||||
s += "<p>Bot: <b>%s</b> | Host: <b>%s</b> | Uptime: <b>%ds</b></p>\n" % \
|
||||
(settings['BOT_NAME'], socket.gethostname(), uptime)
|
||||
if module:
|
||||
s += "<h2><a href='/%s/'>%s</a></h2>\n" % (module.webconsole_id, \
|
||||
module.webconsole_name)
|
||||
return s
|
||||
|
||||
class WebConsoleResource(resource.Resource):
|
||||
isLeaf = True
|
||||
|
||||
@property
|
||||
def modules(self):
|
||||
if not hasattr(self, '_modules'):
|
||||
self._modules = {}
|
||||
for _, obj in dispatcher.send(signal=webconsole_discover_module, \
|
||||
sender=self.__class__):
|
||||
self._modules[obj.webconsole_id] = obj
|
||||
return self._modules
|
||||
|
||||
def render_GET(self, request):
|
||||
m = urlpath_re.search(request.path)
|
||||
if m:
|
||||
module = m.group(1)
|
||||
if module in self.modules:
|
||||
return self.modules[m.group(1)].webconsole_render(request)
|
||||
else:
|
||||
request.setResponseCode(404)
|
||||
return error404(module)
|
||||
else:
|
||||
return self.module_list()
|
||||
|
||||
render_POST = render_GET
|
||||
|
||||
def module_list(self):
|
||||
s = banner()
|
||||
s += "<p>Available modules:</p>\n"
|
||||
s += "<ul>\n"
|
||||
for name, obj in self.modules.iteritems():
|
||||
s += "<li><a href='/%s/'>%s</a></li>\n" % (name, obj.webconsole_name)
|
||||
s += "</ul>\n"
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
return s
|
||||
|
||||
class WebConsole(server.Site):
|
||||
|
||||
def __init__(self):
|
||||
if not settings.getbool('WEBCONSOLE_ENABLED'):
|
||||
raise NotConfigured
|
||||
logfile = settings['WEBCONSOLE_LOGFILE']
|
||||
server.Site.__init__(self, WebConsoleResource(), logPath=logfile)
|
||||
self.noisy = False
|
||||
port = settings.getint('WEBCONSOLE_PORT')
|
||||
reactor.callWhenRunning(reactor.listenTCP, port, self)
|
||||
|
|
@ -6,25 +6,17 @@ See documentation in docs/topics/spiders.rst
|
|||
|
||||
import warnings
|
||||
|
||||
from zope.interface import Interface, Attribute, invariant, implements
|
||||
from twisted.plugin import IPlugin
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.trackref import object_ref
|
||||
|
||||
class ISpider(Interface, IPlugin) :
|
||||
"""Interface used by TwistedPluginSpiderManager to discover spiders"""
|
||||
pass
|
||||
|
||||
class BaseSpider(object_ref):
|
||||
"""Base class for scrapy spiders. All spiders must inherit from this
|
||||
class.
|
||||
"""
|
||||
|
||||
implements(ISpider)
|
||||
|
||||
# XXX: class attributes kept for backwards compatibility
|
||||
name = None
|
||||
start_urls = []
|
||||
|
|
|
|||
|
|
@ -9,6 +9,3 @@ class $classname(BaseSpider):
|
|||
|
||||
def parse(self, response):
|
||||
pass
|
||||
|
||||
SPIDER = $classname()
|
||||
|
||||
|
|
|
|||
|
|
@ -21,5 +21,3 @@ class $classname(CrawlSpider):
|
|||
#i['name'] = hxs.select('//div[@id="name"]').extract()
|
||||
#i['description'] = hxs.select('//div[@id="description"]').extract()
|
||||
return i
|
||||
|
||||
SPIDER = $classname()
|
||||
|
|
|
|||
|
|
@ -18,5 +18,3 @@ class $classname(CSVFeedSpider):
|
|||
#i['name'] = row['name']
|
||||
#i['description'] = row['description']
|
||||
return i
|
||||
|
||||
SPIDER = $classname()
|
||||
|
|
|
|||
|
|
@ -12,5 +12,3 @@ class $classname(XMLFeedSpider):
|
|||
#i['name'] = selector.select('name').extract()
|
||||
#i['description'] = selector.select('description').extract()
|
||||
return i
|
||||
|
||||
SPIDER = $classname()
|
||||
|
|
|
|||
|
|
@ -7,13 +7,13 @@ from twisted.trial import unittest
|
|||
|
||||
# ugly hack to avoid cyclic imports of scrapy.spider when running this test
|
||||
# alone
|
||||
import scrapy.spider
|
||||
from scrapy.contrib.spidermanager import TwistedPluginSpiderManager
|
||||
import scrapy.spider
|
||||
from scrapy.contrib.spidermanager import SpiderManager
|
||||
from scrapy.http import Request
|
||||
|
||||
module_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
class TwistedPluginSpiderManagerTest(unittest.TestCase):
|
||||
class SpiderManagerTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
|
||||
|
|
@ -22,7 +22,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
|
|||
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
|
||||
shutil.copytree(orig_spiders_dir, self.spiders_dir)
|
||||
sys.path.append(self.tmpdir)
|
||||
self.spiderman = TwistedPluginSpiderManager()
|
||||
self.spiderman = SpiderManager()
|
||||
assert not self.spiderman.loaded
|
||||
self.spiderman.load(['test_spiders_xxx'])
|
||||
assert self.spiderman.loaded
|
||||
|
|
@ -32,7 +32,7 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
|
|||
sys.path.remove(self.tmpdir)
|
||||
|
||||
def test_list(self):
|
||||
self.assertEqual(set(self.spiderman.list()),
|
||||
self.assertEqual(set(self.spiderman.list()),
|
||||
set(['spider1', 'spider2']))
|
||||
|
||||
def test_create(self):
|
||||
|
|
@ -42,14 +42,6 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
|
|||
self.assertEqual(spider2.__class__.__name__, 'Spider2')
|
||||
self.assertEqual(spider2.foo, 'bar')
|
||||
|
||||
def test_create_uses_cache(self):
|
||||
# TwistedPluginSpiderManager uses an internal cache which is
|
||||
# invalidated in close_spider() but this isn't necessarily the best
|
||||
# thing to do in all cases.
|
||||
spider1 = self.spiderman.create("spider1")
|
||||
spider2 = self.spiderman.create("spider1")
|
||||
assert spider1 is spider2
|
||||
|
||||
def test_find_by_request(self):
|
||||
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy1.org/test')),
|
||||
['spider1'])
|
||||
|
|
@ -60,16 +52,10 @@ class TwistedPluginSpiderManagerTest(unittest.TestCase):
|
|||
self.assertEqual(self.spiderman.find_by_request(Request('http://scrapy999.org/test')),
|
||||
[])
|
||||
|
||||
def test_close_spider_remove_refs(self):
|
||||
spider = self.spiderman.create("spider1")
|
||||
wref = weakref.ref(spider)
|
||||
assert wref()
|
||||
self.spiderman.close_spider(spider)
|
||||
del spider
|
||||
assert not wref()
|
||||
def test_load_spider_module(self):
|
||||
self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider1'])
|
||||
assert len(self.spiderman._spiders) == 1
|
||||
|
||||
def test_close_spider_invalidates_cache(self):
|
||||
spider1 = self.spiderman.create("spider1")
|
||||
self.spiderman.close_spider(spider1)
|
||||
spider2 = self.spiderman.create("spider1")
|
||||
assert spider1 is not spider2
|
||||
def test_load_base_spider(self):
|
||||
self.spiderman.load(['scrapy.tests.test_contrib_spidermanager.test_spiders.spider0'])
|
||||
assert len(self.spiderman._spiders) == 0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
from scrapy.spider import BaseSpider
|
||||
|
||||
class Spider0(BaseSpider):
|
||||
allowed_domains = ["scrapy1.org", "scrapy3.org"]
|
||||
|
|
@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider
|
|||
class Spider1(BaseSpider):
|
||||
name = "spider1"
|
||||
allowed_domains = ["scrapy1.org", "scrapy3.org"]
|
||||
|
||||
SPIDER = Spider1()
|
||||
|
|
|
|||
|
|
@ -3,5 +3,3 @@ from scrapy.spider import BaseSpider
|
|||
class Spider2(BaseSpider):
|
||||
name = "spider2"
|
||||
allowed_domains = ["scrapy2.org", "scrapy3.org"]
|
||||
|
||||
SPIDER = Spider2()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from unittest import TestCase
|
|||
|
||||
from scrapy.conf import settings
|
||||
from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.http import Request
|
||||
from scrapy.spider import BaseSpider
|
||||
|
||||
|
||||
|
|
@ -11,20 +11,38 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
def setUp(self):
|
||||
self.spider = BaseSpider('foo')
|
||||
self.mw = DefaultHeadersMiddleware()
|
||||
self.default_headers = dict([(k, [v]) for k, v in \
|
||||
self.default_request_headers = dict([(k, [v]) for k, v in \
|
||||
settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])
|
||||
|
||||
def test_process_request(self):
|
||||
req = Request('http://www.scrapytest.org')
|
||||
self.mw.spider_opened(self.spider)
|
||||
self.mw.process_request(req, self.spider)
|
||||
self.assertEquals(req.headers, self.default_headers)
|
||||
self.mw.spider_closed(self.spider)
|
||||
self.assertEquals(req.headers, self.default_request_headers)
|
||||
|
||||
def test_spider_default_request_headers(self):
|
||||
spider_headers = {'Unexistant-Header': ['value']}
|
||||
# override one of the global default headers by spider
|
||||
if self.default_request_headers:
|
||||
k = set(self.default_request_headers).pop()
|
||||
spider_headers[k] = ['__newvalue__']
|
||||
self.spider.default_request_headers = spider_headers
|
||||
|
||||
req = Request('http://www.scrapytest.org')
|
||||
self.mw.spider_opened(self.spider)
|
||||
self.mw.process_request(req, self.spider)
|
||||
self.mw.spider_closed(self.spider)
|
||||
self.assertEquals(req.headers, dict(self.default_request_headers, **spider_headers))
|
||||
|
||||
def test_update_headers(self):
|
||||
headers = {'Accept-Language': ['es'], 'Test-Header': ['test']}
|
||||
req = Request('http://www.scrapytest.org', headers=headers)
|
||||
self.assertEquals(req.headers, headers)
|
||||
|
||||
self.mw.spider_opened(self.spider)
|
||||
self.mw.process_request(req, self.spider)
|
||||
self.default_headers.update(headers)
|
||||
self.assertEquals(req.headers, self.default_headers)
|
||||
self.mw.spider_closed(self.spider)
|
||||
self.default_request_headers.update(headers)
|
||||
self.assertEquals(req.headers, self.default_request_headers)
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
settings = {
|
||||
'HTTPCACHE_DIR': self.tmpdir,
|
||||
'HTTPCACHE_EXPIRATION_SECS': 1,
|
||||
'HTTPCACHE_IGNORE_HTTP_CODES': [],
|
||||
}
|
||||
settings.update(new_settings)
|
||||
return Settings(settings)
|
||||
|
|
@ -76,6 +77,22 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
self.assertEqualResponse(self.response, response)
|
||||
assert 'cached' in response.flags
|
||||
|
||||
def test_middleware_ignore_http_codes(self):
|
||||
# test response is not cached
|
||||
mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[202])
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
mw.process_response(self.request, self.response, self.spider)
|
||||
assert mw.storage.retrieve_response(self.spider, self.request) is None
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
|
||||
# test response is cached
|
||||
mw = self._get_middleware(HTTPCACHE_IGNORE_HTTP_CODES=[203])
|
||||
mw.process_response(self.request, self.response, self.spider)
|
||||
response = mw.process_request(self.request, self.spider)
|
||||
assert isinstance(response, HtmlResponse)
|
||||
self.assertEqualResponse(self.response, response)
|
||||
assert 'cached' in response.flags
|
||||
|
||||
def assertEqualResponse(self, response1, response2):
|
||||
self.assertEqual(response1.url, response2.url)
|
||||
self.assertEqual(response1.status, response2.status)
|
||||
|
|
|
|||
|
|
@ -1,17 +1,129 @@
|
|||
import unittest
|
||||
from cStringIO import StringIO
|
||||
|
||||
from twisted.python import log as txlog, failure
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.conf import settings
|
||||
|
||||
class ItemTest(unittest.TestCase):
|
||||
class LogTest(unittest.TestCase):
|
||||
|
||||
def test_get_log_level(self):
|
||||
default_log_level = getattr(log, settings['LOG_LEVEL'])
|
||||
self.assertEqual(log._get_log_level(), default_log_level)
|
||||
self.assertEqual(log._get_log_level('WARNING'), log.WARNING)
|
||||
self.assertEqual(log._get_log_level(log.WARNING), log.WARNING)
|
||||
self.assertRaises(ValueError, log._get_log_level, 99999)
|
||||
self.assertRaises(ValueError, log._get_log_level, object())
|
||||
|
||||
class ScrapyFileLogObserverTest(unittest.TestCase):
|
||||
|
||||
level = log.INFO
|
||||
encoding = 'utf-8'
|
||||
|
||||
def setUp(self):
|
||||
self.f = StringIO()
|
||||
self.sflo = log.ScrapyFileLogObserver(self.f, self.level, self.encoding)
|
||||
self.sflo.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.flushLoggedErrors()
|
||||
self.sflo.stop()
|
||||
|
||||
def logged(self):
|
||||
return self.f.getvalue().strip()[25:]
|
||||
|
||||
def first_log_line(self):
|
||||
logged = self.logged()
|
||||
return logged.splitlines()[0] if logged else ''
|
||||
|
||||
def test_msg_basic(self):
|
||||
log.msg("Hello")
|
||||
self.assertEqual(self.logged(), "[-] INFO: Hello")
|
||||
|
||||
def test_msg_spider(self):
|
||||
spider = BaseSpider("myspider")
|
||||
log.msg("Hello", spider=spider)
|
||||
self.assertEqual(self.logged(), "[myspider] INFO: Hello")
|
||||
|
||||
def test_msg_level1(self):
|
||||
log.msg("Hello", level=log.WARNING)
|
||||
self.assertEqual(self.logged(), "[-] WARNING: Hello")
|
||||
|
||||
def test_msg_level2(self):
|
||||
log.msg("Hello", log.WARNING)
|
||||
self.assertEqual(self.logged(), "[-] WARNING: Hello")
|
||||
|
||||
def test_msg_wrong_level(self):
|
||||
log.msg("Hello", level=9999)
|
||||
self.assertEqual(self.logged(), "[-] NOLEVEL: Hello")
|
||||
|
||||
def test_msg_level_spider(self):
|
||||
spider = BaseSpider("myspider")
|
||||
log.msg("Hello", spider=spider, level=log.WARNING)
|
||||
self.assertEqual(self.logged(), "[myspider] WARNING: Hello")
|
||||
|
||||
def test_msg_encoding(self):
|
||||
log.msg(u"Price: \xa3100")
|
||||
self.assertEqual(self.logged(), "[-] INFO: Price: \xc2\xa3100")
|
||||
|
||||
def test_msg_ignore_level(self):
|
||||
log.msg("Hello", level=log.DEBUG)
|
||||
log.msg("World", level=log.INFO)
|
||||
self.assertEqual(self.logged(), "[-] INFO: World")
|
||||
|
||||
def test_msg_ignore_system(self):
|
||||
txlog.msg("Hello")
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_msg_ignore_system_err(self):
|
||||
txlog.msg("Hello")
|
||||
self.failIf(self.logged())
|
||||
|
||||
def test_err_noargs(self):
|
||||
try:
|
||||
a = 1/0
|
||||
except:
|
||||
log.err()
|
||||
self.failUnless('Traceback' in self.logged())
|
||||
self.failUnless('ZeroDivisionError' in self.logged())
|
||||
|
||||
def test_err_why(self):
|
||||
log.err(TypeError("bad type"), "Wrong type")
|
||||
self.assertEqual(self.first_log_line(), "[-] ERROR: Wrong type")
|
||||
self.failUnless('TypeError' in self.logged())
|
||||
self.failUnless('bad type' in self.logged())
|
||||
|
||||
def test_err_why_encoding(self):
|
||||
log.err(TypeError("bad type"), u"\xa3")
|
||||
self.assertEqual(self.first_log_line(), "[-] ERROR: \xc2\xa3")
|
||||
|
||||
def test_err_exc(self):
|
||||
log.err(TypeError("bad type"))
|
||||
self.failUnless('Unhandled Error' in self.logged())
|
||||
self.failUnless('TypeError' in self.logged())
|
||||
self.failUnless('bad type' in self.logged())
|
||||
|
||||
def test_err_failure(self):
|
||||
log.err(failure.Failure(TypeError("bad type")))
|
||||
self.failUnless('Unhandled Error' in self.logged())
|
||||
self.failUnless('TypeError' in self.logged())
|
||||
self.failUnless('bad type' in self.logged())
|
||||
|
||||
|
||||
class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest):
|
||||
|
||||
encoding = 'latin-1'
|
||||
|
||||
def test_msg_encoding(self):
|
||||
log.msg(u"Price: \xa3100")
|
||||
logged = self.f.getvalue().strip()[25:]
|
||||
self.assertEqual(self.logged(), "[-] INFO: Price: \xa3100")
|
||||
|
||||
def test_err_why_encoding(self):
|
||||
log.err(TypeError("bad type"), u"\xa3")
|
||||
self.assertEqual(self.first_log_line(), "[-] ERROR: \xa3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
import unittest
|
||||
from cStringIO import StringIO
|
||||
|
||||
from scrapy.utils.misc import load_object, arg_to_iter
|
||||
|
||||
class UtilsMiscTestCase(unittest.TestCase):
|
||||
|
||||
def test_load_object(self):
|
||||
obj = load_object('scrapy.utils.misc.load_object')
|
||||
assert obj is load_object
|
||||
self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
|
||||
self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
|
||||
|
||||
def test_arg_to_iter(self):
|
||||
assert hasattr(arg_to_iter(None), '__iter__')
|
||||
assert hasattr(arg_to_iter(100), '__iter__')
|
||||
assert hasattr(arg_to_iter('lala'), '__iter__')
|
||||
assert hasattr(arg_to_iter([1,2,3]), '__iter__')
|
||||
assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
|
||||
|
||||
self.assertEqual(list(arg_to_iter(None)), [])
|
||||
self.assertEqual(list(arg_to_iter('lala')), ['lala'])
|
||||
self.assertEqual(list(arg_to_iter(100)), [100])
|
||||
self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
|
||||
self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3])
|
||||
self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
import sys
|
||||
import os
|
||||
import unittest
|
||||
from cStringIO import StringIO
|
||||
|
||||
from scrapy.utils.misc import load_object, arg_to_iter, walk_modules
|
||||
|
||||
class UtilsMiscTestCase(unittest.TestCase):
|
||||
|
||||
def test_load_object(self):
|
||||
obj = load_object('scrapy.utils.misc.load_object')
|
||||
assert obj is load_object
|
||||
self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
|
||||
self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
|
||||
|
||||
def test_walk_modules(self):
|
||||
mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules')
|
||||
expected = [
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules',
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules.mod',
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0',
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules.mod1',
|
||||
]
|
||||
self.assertEquals(set([m.__name__ for m in mods]), set(expected))
|
||||
|
||||
mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod')
|
||||
expected = [
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules.mod',
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0',
|
||||
]
|
||||
self.assertEquals(set([m.__name__ for m in mods]), set(expected))
|
||||
|
||||
mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod1')
|
||||
expected = [
|
||||
'scrapy.tests.test_utils_misc.test_walk_modules.mod1',
|
||||
]
|
||||
self.assertEquals(set([m.__name__ for m in mods]), set(expected))
|
||||
|
||||
self.assertRaises(ImportError, walk_modules, 'nomodule999')
|
||||
|
||||
def test_walk_modules_egg(self):
|
||||
egg = os.path.join(os.path.dirname(__file__), 'test.egg')
|
||||
sys.path.append(egg)
|
||||
try:
|
||||
mods = walk_modules('testegg')
|
||||
expected = [
|
||||
'testegg.spiders',
|
||||
'testegg.spiders.a',
|
||||
'testegg.spiders.b',
|
||||
'testegg'
|
||||
]
|
||||
self.assertEquals(set([m.__name__ for m in mods]), set(expected))
|
||||
finally:
|
||||
sys.path.remove(egg)
|
||||
|
||||
def test_arg_to_iter(self):
|
||||
assert hasattr(arg_to_iter(None), '__iter__')
|
||||
assert hasattr(arg_to_iter(100), '__iter__')
|
||||
assert hasattr(arg_to_iter('lala'), '__iter__')
|
||||
assert hasattr(arg_to_iter([1,2,3]), '__iter__')
|
||||
assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__')
|
||||
|
||||
self.assertEqual(list(arg_to_iter(None)), [])
|
||||
self.assertEqual(list(arg_to_iter('lala')), ['lala'])
|
||||
self.assertEqual(list(arg_to_iter(100)), [100])
|
||||
self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
|
||||
self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3])
|
||||
self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Binary file not shown.
|
|
@ -1,5 +1,7 @@
|
|||
import unittest
|
||||
|
||||
from twisted.python import log as txlog
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
from scrapy import log
|
||||
|
|
@ -20,12 +22,12 @@ class SignalUtilsTest(unittest.TestCase):
|
|||
assert arg == 'test'
|
||||
return "OK"
|
||||
|
||||
def log_received(message, level):
|
||||
def log_received(event):
|
||||
handlers_called.add(log_received)
|
||||
assert "test_handler_error" in message
|
||||
assert level == log.ERROR
|
||||
assert "test_handler_error" in event['message'][0]
|
||||
assert event['logLevel'] == log.ERROR
|
||||
|
||||
dispatcher.connect(log_received, signal=log.logmessage_received)
|
||||
txlog.addObserver(log_received)
|
||||
dispatcher.connect(test_handler_error, signal=test_signal)
|
||||
dispatcher.connect(test_handler_check, signal=test_signal)
|
||||
result = send_catch_log(test_signal, arg='test')
|
||||
|
|
@ -37,7 +39,7 @@ class SignalUtilsTest(unittest.TestCase):
|
|||
self.assert_(isinstance(result[0][1], Exception))
|
||||
self.assertEqual(result[1], (test_handler_check, "OK"))
|
||||
|
||||
dispatcher.disconnect(log_received, signal=log.logmessage_received)
|
||||
txlog.removeObserver(log_received)
|
||||
dispatcher.disconnect(test_handler_error, signal=test_signal)
|
||||
dispatcher.disconnect(test_handler_check, signal=test_signal)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import unittest
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, \
|
||||
url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url, \
|
||||
urljoin_rfc
|
||||
|
||||
urljoin_rfc, url_is_from_spider
|
||||
|
||||
class UrlUtilsTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -19,6 +19,15 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
|
||||
self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com']))
|
||||
|
||||
def test_url_is_from_any_domain(self):
|
||||
spider = BaseSpider(name='example.com', allowed_domains=['example.org', 'example.net'])
|
||||
self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
|
||||
self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', spider))
|
||||
self.assertTrue(url_is_from_spider('http://example.com/some/page.html', spider))
|
||||
self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider))
|
||||
self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
|
||||
self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
|
||||
|
||||
def test_urljoin_rfc(self):
|
||||
self.assertEqual(urljoin_rfc('http://example.com/some/path', 'newpath/test'),
|
||||
'http://example.com/some/newpath/test')
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import re
|
||||
import hashlib
|
||||
from pkgutil import iter_modules
|
||||
|
||||
from scrapy.utils.python import flatten
|
||||
from scrapy.utils.markup import remove_entities
|
||||
|
|
@ -44,6 +45,27 @@ def load_object(path):
|
|||
|
||||
return obj
|
||||
|
||||
def walk_modules(path, load=False):
|
||||
"""Loads a module and all its submodules from a the given module path and
|
||||
returns them. If *any* module throws an exception while importing, that
|
||||
exception is thrown back.
|
||||
|
||||
For example: walk_modules('scrapy.utils')
|
||||
"""
|
||||
|
||||
mods = []
|
||||
mod = __import__(path, {}, {}, [''])
|
||||
mods.append(mod)
|
||||
if hasattr(mod, '__path__'):
|
||||
for _, subpath, ispkg in iter_modules(mod.__path__):
|
||||
fullpath = path + '.' + subpath
|
||||
if ispkg:
|
||||
mods += walk_modules(fullpath)
|
||||
else:
|
||||
submod = __import__(fullpath, {}, {}, [''])
|
||||
mods.append(submod)
|
||||
return mods
|
||||
|
||||
def extract_regex(regex, text, encoding='utf-8'):
|
||||
"""Extract a list of unicode strings from the given text/encoding using the following policies:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
"""Functions for dealing with databases"""
|
||||
|
||||
import re
|
||||
|
||||
import MySQLdb
|
||||
|
||||
from scrapy.conf import settings
|
||||
from scrapy import log
|
||||
|
||||
mysql_uri_re = r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/:]+)(:(?P<port>\d+))?/(?P<db>.*)$"
|
||||
|
||||
def parse_uri(mysql_uri):
|
||||
"""Parse mysql URI and return a dict with its parameters"""
|
||||
m = re.search(mysql_uri_re, mysql_uri)
|
||||
if m:
|
||||
d = m.groupdict()
|
||||
if d['passwd'] is None:
|
||||
del(d['passwd'])
|
||||
if d['port'] is None:
|
||||
del(d['port'])
|
||||
else:
|
||||
d['port'] = int(d['port'])
|
||||
return d
|
||||
|
||||
def mysql_connect(db_uri_or_dict, **kwargs):
|
||||
"""Connects to a MySQL DB given a mysql URI"""
|
||||
if isinstance(db_uri_or_dict, dict):
|
||||
d = db_uri_or_dict
|
||||
else:
|
||||
d = parse_uri(db_uri_or_dict)
|
||||
if not d:
|
||||
return
|
||||
d.update(settings.get("MYSQL_CONNECTION_SETTINGS", {}))
|
||||
d.update(kwargs)
|
||||
log.msg("Connecting to MySQL: db=%r, host=%r, user=%r" % (d['db'], \
|
||||
d['host'], d['user']), level=log.DEBUG)
|
||||
conn = MySQLdb.connect(**d)
|
||||
|
||||
return conn
|
||||
|
|
@ -22,7 +22,7 @@ def url_is_from_any_domain(url, domains):
|
|||
|
||||
def url_is_from_spider(url, spider):
|
||||
"""Return True if the url belongs to the given spider"""
|
||||
return url_is_from_any_domain(url, spider.allowed_domains)
|
||||
return url_is_from_any_domain(url, [spider.name] + spider.allowed_domains)
|
||||
|
||||
def urljoin_rfc(base, ref, encoding='utf-8'):
|
||||
"""Same as urlparse.urljoin but supports unicode values in base and ref
|
||||
|
|
|
|||
9
setup.py
9
setup.py
|
|
@ -75,14 +75,7 @@ if len(sys.argv) > 1 and sys.argv[1] == 'bdist_wininst':
|
|||
file_info[0] = '\\PURELIB\\%s' % file_info[0]
|
||||
|
||||
# Dynamically calculate the version based on scrapy.__version__
|
||||
version = __import__('scrapy').__version__
|
||||
if 'dev' in version:
|
||||
try:
|
||||
from subprocess import Popen, PIPE
|
||||
hgrev = Popen(['hg', 'tip', '--template={rev}'], stdout=PIPE).communicate()[0]
|
||||
version = version.replace('-dev', '-r%s' % hgrev)
|
||||
except:
|
||||
pass
|
||||
version = ".".join(map(str, __import__('scrapy').version_info[:2]))
|
||||
|
||||
setup_args = {
|
||||
'name': 'Scrapy',
|
||||
|
|
|
|||
Loading…
Reference in New Issue