mirror of https://github.com/scrapy/scrapy.git
Automated merge with http://hg.scrapy.org/scrapy-stable
This commit is contained in:
Pablo Hoffman
commit
d03bf1f27f
|
|
@ -247,6 +247,18 @@ HttpCompressionMiddleware
|
|||
This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites.
|
||||
|
||||
HttpProxyMiddleware
|
||||
-------------------
|
||||
|
||||
.. module:: scrapy.contrib.downloadermiddleware.httpproxy
|
||||
:synopsis: Http Proxy Middleware
|
||||
|
||||
.. class:: HttpProxyMiddleware
|
||||
|
||||
This middleware sets proxy to use for requests, it obeys enviroment
|
||||
variables 'http_proxy', 'https_proxy', and 'no_proxy'
|
||||
|
||||
|
||||
RedirectMiddleware
|
||||
-------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -725,6 +725,33 @@ Default: ``+2``
|
|||
Adjust redirect request priority relative to original request.
|
||||
A negative priority adjust means more priority.
|
||||
|
||||
.. setting:: REQUEST_HANDLERS
|
||||
|
||||
REQUEST_HANDLERS
|
||||
----------------
|
||||
|
||||
Default: ``{}``
|
||||
|
||||
A dict containing the request downloader handlers enabled in your project.
|
||||
See `REQUEST_HANDLERS_BASE` for example format.
|
||||
|
||||
.. setting:: REQUEST_HANDLERS_BASE
|
||||
|
||||
REQUEST_HANDLERS_BASE
|
||||
---------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
'file': 'scrapy.core.downloader.handlers.file.download_file',
|
||||
'http': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
'https': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
}
|
||||
|
||||
A dict containing the request download handlers enabled by default in Scrapy.
|
||||
You should never modify this setting in your project, modify
|
||||
:setting:`REQUEST_HANDLERS` instead.
|
||||
|
||||
.. setting:: REQUESTS_PER_DOMAIN
|
||||
|
||||
REQUESTS_PER_DOMAIN
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
hg purge --all
|
||||
|
||||
# build packages
|
||||
version=$(python -c "import scrapy; print scrapy.__version__")
|
||||
python setup.py sdist
|
||||
#version=$(python -c "import scrapy; print scrapy.__version__")
|
||||
#python setup.py sdist
|
||||
# FIXME: bdist_wininst doesn't work on Unix (it doesn't include the data_files)
|
||||
# To build the win32 release you need to use Windows for now.
|
||||
#python setup.py bdist_wininst -t "Scrapy $version" -p "win32"
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ Scrapy - a screen scraping framework written in Python
|
|||
"""
|
||||
|
||||
# IMPORTANT: remember to also update the version in docs/conf.py
|
||||
version_info = (0, 7, 0, 'final', 0)
|
||||
__version__ = "0.7"
|
||||
version_info = (0, 8, 0, '', 0)
|
||||
__version__ = "0.8-dev"
|
||||
|
||||
import sys, os
|
||||
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
|
|||
'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
|
||||
'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
|
||||
'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
|
||||
'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
|
||||
'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
|
||||
'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
|
||||
'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
|
||||
|
|
@ -100,6 +101,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
|
|||
ITEM_PIPELINES = []
|
||||
|
||||
LOG_ENABLED = True
|
||||
LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline'
|
||||
LOG_STDOUT = False
|
||||
LOG_LEVEL = 'DEBUG'
|
||||
LOG_FILE = None
|
||||
|
|
@ -124,6 +126,13 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
|
|||
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
||||
REDIRECT_PRIORITY_ADJUST = +2
|
||||
|
||||
REQUEST_HANDLERS = {}
|
||||
REQUEST_HANDLERS_BASE = {
|
||||
'file': 'scrapy.core.downloader.handlers.file.download_file',
|
||||
'http': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
'https': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
}
|
||||
|
||||
REQUESTS_QUEUE_SIZE = 0
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
import base64
|
||||
from urllib import getproxies, unquote, proxy_bypass
|
||||
from urllib2 import _parse_proxy
|
||||
from urlparse import urlunparse
|
||||
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
|
||||
|
||||
class HttpProxyMiddleware(object):
|
||||
|
||||
def __init__(self):
|
||||
self.proxies = {}
|
||||
for type, url in getproxies().items():
|
||||
self.proxies[type] = self._get_proxy(url, type)
|
||||
|
||||
if not self.proxies:
|
||||
raise NotConfigured
|
||||
|
||||
def _get_proxy(self, url, orig_type):
|
||||
proxy_type, user, password, hostport = _parse_proxy(url)
|
||||
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
|
||||
|
||||
if user and password:
|
||||
user_pass = '%s:%s' % (unquote(user), unquote(password))
|
||||
creds = base64.b64encode(user_pass).strip()
|
||||
else:
|
||||
creds = None
|
||||
|
||||
return creds, proxy_url
|
||||
|
||||
def process_request(self, request, spider):
|
||||
# ignore if proxy is already seted
|
||||
if 'proxy' in request.meta:
|
||||
return
|
||||
|
||||
parsed = urlparse_cached(request)
|
||||
scheme = parsed.scheme
|
||||
|
||||
# 'no_proxy' is only supported by http schemes
|
||||
if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
|
||||
return
|
||||
|
||||
if scheme in self.proxies:
|
||||
self._set_proxy(request, scheme)
|
||||
|
||||
def _set_proxy(self, request, scheme):
|
||||
creds, proxy = self.proxies[scheme]
|
||||
request.meta['proxy'] = proxy
|
||||
if creds:
|
||||
request.headers['Proxy-Authorization'] = 'Basic ' + creds
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
"""Functions for logging diferent actions"""
|
||||
|
||||
def crawled_logline(request, response):
|
||||
referer = request.headers.get('Referer')
|
||||
flags = ' %s' % str(response.flags) if response.flags else ''
|
||||
return "Crawled (%d) %s (referer: %s)%s" % (response.status, \
|
||||
request, referer, flags)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
from lxml import html, etree
|
||||
|
||||
from scrapy.contrib.loader import ItemLoader
|
||||
|
||||
|
||||
class LxmlItemLoader(ItemLoader):
|
||||
|
||||
def __init__(self, response, item=None, **context):
|
||||
self.tree = html.fromstring(response.body_as_unicode())
|
||||
context.update(response=response)
|
||||
super(LxmlItemLoader, self).__init__(item, **context)
|
||||
|
||||
def add_xpath(self, field_name, xpath):
|
||||
self.add_value(field_name, self._get_xpath(xpath))
|
||||
|
||||
def replace_xpath(self, field_name, xpath):
|
||||
self.replace_value(field_name, self._get_xpath(xpath))
|
||||
|
||||
def _get_xpath(self, xpath):
|
||||
return self._get_values(self.tree.xpath(xpath))
|
||||
|
||||
def add_css(self, field_name, css):
|
||||
self.add_value(field_name, self._get_css(css))
|
||||
|
||||
def replace_css(self, field_name, css):
|
||||
self.replace_value(field_name, self._get_css(css))
|
||||
|
||||
def _get_css(self, css):
|
||||
return self._get_values(self.tree.cssselect(css))
|
||||
|
||||
def _get_values(self, elems):
|
||||
for e in elems:
|
||||
yield etree.tostring(e) if isinstance(e, etree.ElementBase) else e
|
||||
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
"""
|
||||
Download handlers for different schemes
|
||||
"""
|
||||
from __future__ import with_statement
|
||||
|
||||
import urlparse
|
||||
|
||||
from twisted.internet import reactor
|
||||
try:
|
||||
from twisted.internet import ssl
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from scrapy import optional_features
|
||||
from scrapy.core import signals
|
||||
from scrapy.http import Headers
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.defer import defer_succeed
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
|
||||
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
|
||||
ssl_supported = 'ssl' in optional_features
|
||||
|
||||
def download_any(request, spider):
|
||||
scheme = urlparse_cached(request).scheme
|
||||
if scheme == 'http':
|
||||
return download_http(request, spider)
|
||||
elif scheme == 'https':
|
||||
if ssl_supported:
|
||||
return download_https(request, spider)
|
||||
else:
|
||||
raise NotSupported("HTTPS not supported: install pyopenssl library")
|
||||
elif scheme == 'file':
|
||||
return download_file(request, spider)
|
||||
else:
|
||||
raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
|
||||
|
||||
def create_factory(request, spider):
|
||||
"""Return HTTPClientFactory for the given Request"""
|
||||
url = urlparse.urldefrag(request.url)[0]
|
||||
timeout = getattr(spider, "download_timeout", None) or default_timeout
|
||||
factory = HTTPClientFactory.from_request(request, timeout)
|
||||
|
||||
def _create_response(body):
|
||||
body = body or ''
|
||||
status = int(factory.status)
|
||||
headers = Headers(factory.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
r = respcls(url=request.url, status=status, headers=headers, body=body)
|
||||
send_catch_log(signal=signals.request_uploaded, sender='download_http', \
|
||||
request=request, spider=spider)
|
||||
send_catch_log(signal=signals.response_downloaded, sender='download_http', \
|
||||
response=r, spider=spider)
|
||||
return r
|
||||
|
||||
factory.deferred.addCallbacks(_create_response)
|
||||
return factory
|
||||
|
||||
def download_http(request, spider):
|
||||
"""Return a deferred for the HTTP download"""
|
||||
factory = create_factory(request, spider)
|
||||
url = urlparse_cached(request)
|
||||
port = url.port
|
||||
reactor.connectTCP(url.hostname, port or 80, factory)
|
||||
return factory.deferred
|
||||
|
||||
def download_https(request, spider):
|
||||
"""Return a deferred for the HTTPS download"""
|
||||
factory = create_factory(request, spider)
|
||||
url = urlparse_cached(request)
|
||||
port = url.port
|
||||
contextFactory = ssl.ClientContextFactory()
|
||||
reactor.connectSSL(url.hostname, port or 443, factory, contextFactory)
|
||||
return factory.deferred
|
||||
|
||||
def download_file(request, spider) :
|
||||
"""Return a deferred for a file download."""
|
||||
filepath = request.url.split("file://")[1]
|
||||
with open(filepath) as f:
|
||||
body = f.read()
|
||||
respcls = responsetypes.from_args(filename=filepath, body=body)
|
||||
response = respcls(url=request.url, body=body)
|
||||
|
||||
return defer_succeed(response)
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""Download handlers for different schemes"""
|
||||
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.conf import settings
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
class RequestHandlers(object):
|
||||
|
||||
def __init__(self):
|
||||
self._handlers = {}
|
||||
handlers = settings.get('REQUEST_HANDLERS_BASE')
|
||||
handlers.update(settings.get('REQUEST_HANDLERS', {}))
|
||||
for scheme, cls in handlers.iteritems():
|
||||
self._handlers[scheme] = load_object(cls)
|
||||
|
||||
def download_request(self, request, spider):
|
||||
scheme = urlparse_cached(request).scheme
|
||||
try:
|
||||
handler = self._handlers[scheme]
|
||||
except KeyError:
|
||||
raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
|
||||
return handler(request, spider)
|
||||
|
||||
|
||||
download_any = RequestHandlers().download_request
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
"""Download handler for file:// scheme"""
|
||||
from __future__ import with_statement
|
||||
|
||||
from twisted.internet import defer
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
|
||||
|
||||
def download_file(request, spider):
|
||||
"""Return a deferred for a file download."""
|
||||
return defer.maybeDeferred(_all_in_one_read_download_file, request, spider)
|
||||
|
||||
def _all_in_one_read_download_file(request, spider):
|
||||
filepath = request.url.split("file://")[1]
|
||||
with open(filepath) as f:
|
||||
body = f.read()
|
||||
respcls = responsetypes.from_args(filename=filepath, body=body)
|
||||
return respcls(url=request.url, body=body)
|
||||
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
"""Download handlers for http and https schemes"""
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.conf import settings
|
||||
from scrapy import optional_features
|
||||
|
||||
ssl_supported = 'ssl' in optional_features
|
||||
if ssl_supported:
|
||||
from twisted.internet.ssl import ClientContextFactory
|
||||
|
||||
|
||||
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
|
||||
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
|
||||
|
||||
def _create_factory(request, spider):
|
||||
def _download_signals(response):
|
||||
send_catch_log(signal=signals.request_uploaded, \
|
||||
sender='download_http', request=request, spider=spider)
|
||||
send_catch_log(signal=signals.response_downloaded, \
|
||||
sender='download_http', response=response, spider=spider)
|
||||
return response
|
||||
|
||||
timeout = getattr(spider, "download_timeout", None) or default_timeout
|
||||
factory = HTTPClientFactory(request, timeout)
|
||||
factory.deferred.addCallbacks(_download_signals)
|
||||
return factory
|
||||
|
||||
|
||||
def _connect(factory):
|
||||
host, port = factory.host, factory.port
|
||||
if factory.scheme == 'https':
|
||||
if ssl_supported:
|
||||
return reactor.connectSSL(host, port, factory, ClientContextFactory())
|
||||
raise NotSupported("HTTPS not supported: install pyopenssl library")
|
||||
else:
|
||||
return reactor.connectTCP(host, port, factory)
|
||||
|
||||
|
||||
def download_http(request, spider):
|
||||
"""Return a deferred for the HTTP download"""
|
||||
factory = _create_factory(request, spider)
|
||||
_connect(factory)
|
||||
return factory.deferred
|
||||
|
||||
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
from urlparse import urlparse, urlunparse
|
||||
from urlparse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.python import failure
|
||||
from twisted.web.client import HTTPClientFactory, PartialDownloadError
|
||||
from twisted.web.client import PartialDownloadError, HTTPClientFactory
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
|
||||
|
||||
def _parsed_url_args(parsed):
|
||||
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
|
||||
|
|
@ -18,11 +20,13 @@ def _parsed_url_args(parsed):
|
|||
port = 443 if scheme == 'https' else 80
|
||||
return scheme, netloc, host, port, path
|
||||
|
||||
|
||||
def _parse(url):
|
||||
url = url.strip()
|
||||
parsed = urlparse(url)
|
||||
return _parsed_url_args(parsed)
|
||||
|
||||
|
||||
class ScrapyHTTPPageGetter(HTTPClient):
|
||||
|
||||
def connectionMade(self):
|
||||
|
|
@ -64,7 +68,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
|
||||
def timeout(self):
|
||||
self.transport.loseConnection()
|
||||
self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout)))
|
||||
self.factory.noPage(\
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." % \
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
|
|
@ -74,22 +80,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
"""
|
||||
|
||||
protocol = ScrapyHTTPPageGetter
|
||||
response_headers = None
|
||||
waiting = 1
|
||||
noisy = False
|
||||
|
||||
def __init__(self, url, method='GET', body=None, headers=None, timeout=0, parsedurl=None):
|
||||
self.url = url
|
||||
self.method = method
|
||||
self.body = body or None
|
||||
if parsedurl:
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsedurl)
|
||||
else:
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parse(url)
|
||||
|
||||
def __init__(self, request, timeout=0):
|
||||
self.url = urldefrag(request.url)[0]
|
||||
self.method = request.method
|
||||
self.body = request.body or None
|
||||
self.headers = request.headers
|
||||
self.response_headers = None
|
||||
self.timeout = timeout
|
||||
self.headers = Headers(headers or {})
|
||||
self.deferred = defer.Deferred()
|
||||
self.deferred = defer.Deferred().addCallback(self._build_response)
|
||||
|
||||
self._set_connection_attributes(request)
|
||||
|
||||
# set Host header based on url
|
||||
self.headers.setdefault('Host', self.netloc)
|
||||
|
|
@ -100,33 +103,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
# just in case a broken http/1.1 decides to keep connection alive
|
||||
self.headers.setdefault("Connection", "close")
|
||||
|
||||
@classmethod
|
||||
def from_request(cls, request, timeout):
|
||||
return cls(request.url,
|
||||
method=request.method,
|
||||
body=request.body or None, # see http://dev.scrapy.org/ticket/60
|
||||
headers=Headers(request.headers or {}),
|
||||
timeout=timeout,
|
||||
parsedurl=urlparse_cached(request),
|
||||
)
|
||||
def _build_response(self, body):
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self.url)
|
||||
return respcls(url=self.url, status=status, headers=headers, body=body)
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
parsed = urlparse_cached(request)
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
|
||||
proxy = request.meta.get('proxy')
|
||||
if proxy:
|
||||
self.scheme, _, self.host, self.port, _ = _parse(proxy)
|
||||
self.path = self.url
|
||||
|
||||
def gotHeaders(self, headers):
|
||||
self.response_headers = headers
|
||||
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, *args, **kwargs):
|
||||
"""
|
||||
Download a web page as a string.
|
||||
|
||||
Download a page. Return a deferred, which will callback with a
|
||||
page (as a string) or errback with a description of the error.
|
||||
|
||||
See HTTPClientFactory to see what extra args can be passed.
|
||||
"""
|
||||
from twisted.web.client import _makeGetterFactory
|
||||
return _makeGetterFactory(
|
||||
url,
|
||||
ScrapyHTTPClientFactory,
|
||||
contextFactory=contextFactory,
|
||||
*args, **kwargs).deferred
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class ExecutionEngine(object):
|
|||
self.paused = False
|
||||
self._next_request_pending = set()
|
||||
self._mainloop_task = task.LoopingCall(self._mainloop)
|
||||
self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED'])
|
||||
|
||||
def configure(self):
|
||||
"""
|
||||
|
|
@ -198,15 +199,14 @@ class ExecutionEngine(object):
|
|||
|
||||
def download(self, request, spider):
|
||||
domain = spider.domain_name
|
||||
referer = request.headers.get('Referer')
|
||||
|
||||
def _on_success(response):
|
||||
"""handle the result of a page download"""
|
||||
assert isinstance(response, (Response, Request))
|
||||
if isinstance(response, Response):
|
||||
response.request = request # tie request to response received
|
||||
log.msg("Crawled %s (referer: <%s>)" % (request, referer), \
|
||||
level=log.DEBUG, domain=domain)
|
||||
log.msg(self._crawled_logline(request, response), \
|
||||
level=log.DEBUG, domain=spider.domain_name)
|
||||
return response
|
||||
elif isinstance(response, Request):
|
||||
newrequest = response
|
||||
|
|
@ -224,14 +224,17 @@ class ExecutionEngine(object):
|
|||
errmsg = str(_failure)
|
||||
level = log.ERROR
|
||||
if errmsg:
|
||||
log.msg("Downloading <%s> (referer: <%s>): %s" % (request.url, \
|
||||
referer, errmsg), level=level, domain=domain)
|
||||
log.msg("Crawling <%s>: %s" % (request.url, errmsg), \
|
||||
level=level, domain=domain)
|
||||
return Failure(IgnoreRequest(str(exc)))
|
||||
|
||||
def _on_complete(_):
|
||||
self.next_request(spider)
|
||||
return _
|
||||
|
||||
if spider not in self.downloader.sites:
|
||||
return defer.fail(Failure(IgnoreRequest())).addBoth(_on_complete)
|
||||
|
||||
dwld = mustbe_deferred(self.downloader.fetch, request, spider)
|
||||
dwld.addCallbacks(_on_success, _on_error)
|
||||
dwld.addBoth(_on_complete)
|
||||
|
|
|
|||
|
|
@ -85,10 +85,7 @@ class Request(object_ref):
|
|||
return self._encoding
|
||||
|
||||
def __str__(self):
|
||||
if self.method == 'GET':
|
||||
return "<%s>" % self.url
|
||||
else:
|
||||
return "<%s %s>" % (self.method, self.url)
|
||||
return "<%s %s>" % (self.method, self.url)
|
||||
|
||||
def __repr__(self):
|
||||
attrs = ['url', 'method', 'body', 'headers', 'cookies', 'meta']
|
||||
|
|
|
|||
|
|
@ -65,9 +65,7 @@ class Response(object_ref):
|
|||
return "%s(%s)" % (self.__class__.__name__, args)
|
||||
|
||||
def __str__(self):
|
||||
flags = "(%s) " % ",".join(self.flags) if self.flags else ""
|
||||
status = "%d " % self.status + " " if self.status != 200 else ""
|
||||
return "%s<%s%s>" % (flags, status, self.url)
|
||||
return "<%d %s>" % (self.status, self.url)
|
||||
|
||||
def copy(self):
|
||||
"""Return a copy of this Response"""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.contrib.loader.processor import MapCompose
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.http import HtmlResponse
|
||||
|
||||
try:
|
||||
import lxml
|
||||
except ImportError:
|
||||
lxml = False
|
||||
|
||||
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
||||
|
||||
if lxml:
|
||||
from scrapy.contrib_exp.loader.lxmlloader import LxmlItemLoader
|
||||
|
||||
class TestLxmlItemLoader(LxmlItemLoader):
|
||||
default_item_class = TestItem
|
||||
|
||||
|
||||
class LxmlItemLoaderTest(unittest.TestCase):
|
||||
response = HtmlResponse(url="", body='<html><body><div id="id">marta</div><p>paragraph</p></body></html>')
|
||||
|
||||
def setUp(self):
|
||||
if not lxml:
|
||||
raise unittest.SkipTest("lxml is not available")
|
||||
|
||||
def test_constructor_with_response(self):
|
||||
l = TestLxmlItemLoader(response=self.response)
|
||||
self.assert_(l.tree)
|
||||
|
||||
def test_add_xpath(self):
|
||||
l = TestLxmlItemLoader(response=self.response)
|
||||
l.add_xpath('name', '//div')
|
||||
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
|
||||
|
||||
def test_add_xpath_text(self):
|
||||
l = TestLxmlItemLoader(response=self.response)
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
|
||||
def test_replace_xpath(self):
|
||||
l = TestLxmlItemLoader(response=self.response)
|
||||
l.add_xpath('name', '//div/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
l.replace_xpath('name', '//p/text()')
|
||||
self.assertEqual(l.get_output_value('name'), [u'paragraph'])
|
||||
|
||||
def test_add_css(self):
|
||||
l = TestLxmlItemLoader(response=self.response)
|
||||
l.add_css('name', '#id')
|
||||
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
|
||||
|
||||
def test_replace_css(self):
|
||||
l = TestLxmlItemLoader(response=self.response)
|
||||
l.add_css('name', '#id')
|
||||
self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
|
||||
l.replace_css('name', 'p')
|
||||
self.assertEqual(l.get_output_value('name'), [u'<p>paragraph</p>'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.contrib.logformatter import crawled_logline
|
||||
|
||||
|
||||
class LoggingContribTest(unittest.TestCase):
|
||||
|
||||
def test_crawled_logline(self):
|
||||
req = Request("http://www.example.com")
|
||||
res = Response("http://www.example.com")
|
||||
self.assertEqual(crawled_logline(req, res),
|
||||
"Crawled (200) <GET http://www.example.com> (referer: None)")
|
||||
|
||||
req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
|
||||
res = Response("http://www.example.com", flags=['cached'])
|
||||
self.assertEqual(crawled_logline(req, res),
|
||||
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
import os
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.python.filepath import FilePath
|
||||
from twisted.internet import reactor, defer
|
||||
from twisted.web import server, static, util, resource
|
||||
from twisted.web.test.test_webclient import ForeverTakingResource, \
|
||||
NoLengthResource, HostHeaderResource, \
|
||||
PayloadResource, BrokenDownloadResource
|
||||
|
||||
from scrapy.core.downloader.webclient import PartialDownloadError
|
||||
from scrapy.core.downloader.handlers.file import download_file
|
||||
from scrapy.core.downloader.handlers.http import download_http
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.http import Request
|
||||
|
||||
|
||||
class FileTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmpname = self.mktemp()
|
||||
fd = open(self.tmpname, 'w')
|
||||
fd.write('0123456789')
|
||||
fd.close()
|
||||
|
||||
def test_download(self):
|
||||
def _test(response):
|
||||
self.assertEquals(response.url, request.url)
|
||||
self.assertEquals(response.status, 200)
|
||||
self.assertEquals(response.body, '0123456789')
|
||||
|
||||
request = Request('file://%s' % self.tmpname)
|
||||
return download_file(request, BaseSpider()).addCallback(_test)
|
||||
|
||||
def test_non_existent(self):
|
||||
request = Request('file://%s' % self.mktemp())
|
||||
d = download_file(request, BaseSpider())
|
||||
return self.assertFailure(d, IOError)
|
||||
|
||||
|
||||
class HttpTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
name = self.mktemp()
|
||||
os.mkdir(name)
|
||||
FilePath(name).child("file").setContent("0123456789")
|
||||
r = static.File(name)
|
||||
r.putChild("redirect", util.Redirect("/file"))
|
||||
r.putChild("wait", ForeverTakingResource())
|
||||
r.putChild("nolength", NoLengthResource())
|
||||
r.putChild("host", HostHeaderResource())
|
||||
r.putChild("payload", PayloadResource())
|
||||
r.putChild("broken", BrokenDownloadResource())
|
||||
self.site = server.Site(r, timeout=None)
|
||||
self.wrapper = WrappingFactory(self.site)
|
||||
self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
|
||||
self.portno = self.port.getHost().port
|
||||
|
||||
def tearDown(self):
|
||||
return self.port.stopListening()
|
||||
|
||||
def getURL(self, path):
|
||||
return "http://127.0.0.1:%d/%s" % (self.portno, path)
|
||||
|
||||
def test_download(self):
|
||||
request = Request(self.getURL('file'))
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.body)
|
||||
d.addCallback(self.assertEquals, "0123456789")
|
||||
return d
|
||||
|
||||
def test_redirect_status(self):
|
||||
request = Request(self.getURL('redirect'))
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.status)
|
||||
d.addCallback(self.assertEquals, 302)
|
||||
return d
|
||||
|
||||
def test_timeout_download_from_spider(self):
|
||||
spider = BaseSpider()
|
||||
spider.download_timeout = 0.000001
|
||||
request = Request(self.getURL('wait'))
|
||||
d = download_http(request, spider)
|
||||
return self.assertFailure(d, defer.TimeoutError)
|
||||
|
||||
def test_host_header(self):
|
||||
request = Request(self.getURL('host'))
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.body)
|
||||
d.addCallback(self.assertEquals, '127.0.0.1:%d' % self.portno)
|
||||
return d
|
||||
|
||||
def test_payload(self):
|
||||
body = '1'*100 # PayloadResource requires body length to be 100
|
||||
request = Request(self.getURL('payload'), method='POST', body=body)
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.body)
|
||||
d.addCallback(self.assertEquals, body)
|
||||
return d
|
||||
|
||||
def test_broken_download(self):
|
||||
request = Request(self.getURL('broken'))
|
||||
d = download_http(request, BaseSpider())
|
||||
return self.assertFailure(d, PartialDownloadError)
|
||||
|
||||
|
||||
class UriResource(resource.Resource):
|
||||
"""Return the full uri that was requested"""
|
||||
|
||||
def getChild(self, path, request):
|
||||
return self
|
||||
|
||||
def render(self, request):
|
||||
return request.uri
|
||||
|
||||
|
||||
class HttpProxyTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
site = server.Site(UriResource(), timeout=None)
|
||||
wrapper = WrappingFactory(site)
|
||||
self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
|
||||
self.portno = self.port.getHost().port
|
||||
|
||||
def tearDown(self):
|
||||
return self.port.stopListening()
|
||||
|
||||
def getURL(self, path):
|
||||
return "http://127.0.0.1:%d/%s" % (self.portno, path)
|
||||
|
||||
def test_download_with_proxy(self):
|
||||
def _test(response):
|
||||
self.assertEquals(response.status, 200)
|
||||
self.assertEquals(response.url, request.url)
|
||||
self.assertEquals(response.body, 'https://example.com')
|
||||
|
||||
http_proxy = self.getURL('')
|
||||
request = Request('https://example.com', meta={'proxy': http_proxy})
|
||||
return download_http(request, BaseSpider()).addCallback(_test)
|
||||
|
||||
def test_download_without_proxy(self):
|
||||
def _test(response):
|
||||
self.assertEquals(response.status, 200)
|
||||
self.assertEquals(response.url, request.url)
|
||||
self.assertEquals(response.body, '/path/to/resource')
|
||||
|
||||
request = Request(self.getURL('path/to/resource'))
|
||||
return download_http(request, BaseSpider()).addCallback(_test)
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
import os
|
||||
from unittest import TestCase
|
||||
|
||||
from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.conf import settings
|
||||
|
||||
spider = BaseSpider()
|
||||
|
||||
class TestDefaultHeadersMiddleware(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self._oldenv = os.environ.copy()
|
||||
|
||||
def tearDown(self):
|
||||
os.environ = self._oldenv
|
||||
|
||||
def test_no_proxies(self):
|
||||
os.environ = {}
|
||||
self.assertRaises(NotConfigured, HttpProxyMiddleware)
|
||||
|
||||
def test_no_enviroment_proxies(self):
|
||||
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
|
||||
mw = HttpProxyMiddleware()
|
||||
|
||||
for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
|
||||
req = Request(url)
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.url, url)
|
||||
self.assertEquals(req.meta, {})
|
||||
|
||||
def test_enviroment_proxies(self):
|
||||
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
|
||||
os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
|
||||
os.environ.pop('file_proxy', None)
|
||||
mw = HttpProxyMiddleware()
|
||||
|
||||
for url, proxy in [('http://e.com', http_proxy),
|
||||
('https://e.com', https_proxy), ('file://tmp/a', None)]:
|
||||
req = Request(url)
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.url, url)
|
||||
self.assertEquals(req.meta.get('proxy'), proxy)
|
||||
|
||||
def test_proxy_auth(self):
|
||||
os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
|
||||
mw = HttpProxyMiddleware()
|
||||
req = Request('http://scrapytest.org')
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
|
||||
|
||||
def test_proxy_already_seted(self):
|
||||
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
|
||||
mw = HttpProxyMiddleware()
|
||||
req = Request('http://noproxy.com', meta={'proxy': None})
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'proxy' in req.meta and req.meta['proxy'] is None
|
||||
|
||||
|
||||
def test_no_proxy(self):
|
||||
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
|
||||
mw = HttpProxyMiddleware()
|
||||
|
||||
os.environ['no_proxy'] = '*'
|
||||
req = Request('http://noproxy.com')
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'proxy' not in req.meta
|
||||
|
||||
os.environ['no_proxy'] = 'other.com'
|
||||
req = Request('http://noproxy.com')
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'proxy' in req.meta
|
||||
|
||||
os.environ['no_proxy'] = 'other.com,noproxy.com'
|
||||
req = Request('http://noproxy.com')
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'proxy' not in req.meta
|
||||
|
||||
|
|
@ -13,14 +13,27 @@ from twisted.python.filepath import FilePath
|
|||
from twisted.protocols.policies import WrappingFactory
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.http import Headers
|
||||
from scrapy.http import Request, Headers
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, *args, **kwargs):
|
||||
"""Adapted version of twisted.web.client.getPage"""
|
||||
def _clientfactory(*args, **kwargs):
|
||||
timeout = kwargs.pop('timeout', 0)
|
||||
f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout)
|
||||
f.deferred.addCallback(lambda r: r.body)
|
||||
return f
|
||||
|
||||
from twisted.web.client import _makeGetterFactory
|
||||
return _makeGetterFactory(url, _clientfactory,
|
||||
contextFactory=contextFactory, *args, **kwargs).deferred
|
||||
|
||||
|
||||
class ParseUrlTestCase(unittest.TestCase):
|
||||
"""Test URL parsing facility and defaults values."""
|
||||
|
||||
def _parse(self, url):
|
||||
f = client.ScrapyHTTPClientFactory(url)
|
||||
f = client.ScrapyHTTPClientFactory(Request(url))
|
||||
return (f.scheme, f.netloc, f.host, f.port, f.path)
|
||||
|
||||
def testParse(self):
|
||||
|
|
@ -75,7 +88,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
|
||||
def test_earlyHeaders(self):
|
||||
# basic test stolen from twisted HTTPageGetter
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
url='http://foo/bar',
|
||||
body="some data",
|
||||
headers={
|
||||
|
|
@ -83,7 +96,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
'User-Agent': 'fooble',
|
||||
'Cookie': 'blah blah',
|
||||
'Content-Length': '12981',
|
||||
'Useful': 'value'})
|
||||
'Useful': 'value'}))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -97,18 +110,18 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
"some data")
|
||||
|
||||
# test minimal sent headers
|
||||
factory = client.ScrapyHTTPClientFactory('http://foo/bar')
|
||||
factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
"Host: foo\r\n"
|
||||
"\r\n")
|
||||
|
||||
# test a simple POST with body and content-type
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
method='POST',
|
||||
url='http://foo/bar',
|
||||
body='name=value',
|
||||
headers={'Content-Type': 'application/x-www-form-urlencoded'})
|
||||
headers={'Content-Type': 'application/x-www-form-urlencoded'}))
|
||||
|
||||
self._test(factory,
|
||||
"POST /bar HTTP/1.0\r\n"
|
||||
|
|
@ -120,12 +133,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
"name=value")
|
||||
|
||||
# test with single and multivalued headers
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
url='http://foo/bar',
|
||||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})
|
||||
}))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -136,12 +149,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
"\r\n")
|
||||
|
||||
# same test with single and multivalued headers but using Headers class
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
url='http://foo/bar',
|
||||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
})))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -193,11 +206,11 @@ class WebClientTestCase(unittest.TestCase):
|
|||
|
||||
def testPayload(self):
|
||||
s = "0123456789" * 10
|
||||
return client.getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
|
||||
return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
|
||||
|
||||
def testBrokenDownload(self):
|
||||
# test what happens when download gets disconnected in the middle
|
||||
d = client.getPage(self.getURL("broken"))
|
||||
d = getPage(self.getURL("broken"))
|
||||
d = self.assertFailure(d, client.PartialDownloadError)
|
||||
d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
|
||||
return d
|
||||
|
|
@ -206,8 +219,8 @@ class WebClientTestCase(unittest.TestCase):
|
|||
# if we pass Host header explicitly, it should be used, otherwise
|
||||
# it should extract from url
|
||||
return defer.gatherResults([
|
||||
client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
|
||||
client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
|
||||
getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
|
||||
getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
|
||||
|
||||
|
||||
def test_getPage(self):
|
||||
|
|
@ -215,7 +228,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the body of the response if the default method B{GET} is used.
|
||||
"""
|
||||
d = client.getPage(self.getURL("file"))
|
||||
d = getPage(self.getURL("file"))
|
||||
d.addCallback(self.assertEquals, "0123456789")
|
||||
return d
|
||||
|
||||
|
|
@ -226,11 +239,11 @@ class WebClientTestCase(unittest.TestCase):
|
|||
the empty string if the method is C{HEAD} and there is a successful
|
||||
response code.
|
||||
"""
|
||||
def getPage(method):
|
||||
return client.getPage(self.getURL("file"), method=method)
|
||||
def _getPage(method):
|
||||
return getPage(self.getURL("file"), method=method)
|
||||
return defer.gatherResults([
|
||||
getPage("head").addCallback(self.assertEqual, ""),
|
||||
getPage("HEAD").addCallback(self.assertEqual, "")])
|
||||
_getPage("head").addCallback(self.assertEqual, ""),
|
||||
_getPage("HEAD").addCallback(self.assertEqual, "")])
|
||||
|
||||
|
||||
def test_timeoutNotTriggering(self):
|
||||
|
|
@ -239,7 +252,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
retrieved before the timeout period elapses, the L{Deferred} is
|
||||
called back with the contents of the page.
|
||||
"""
|
||||
d = client.getPage(self.getURL("host"), timeout=100)
|
||||
d = getPage(self.getURL("host"), timeout=100)
|
||||
d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno)
|
||||
return d
|
||||
|
||||
|
|
@ -251,7 +264,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
L{Deferred} is errbacked with a L{error.TimeoutError}.
|
||||
"""
|
||||
finished = self.assertFailure(
|
||||
client.getPage(self.getURL("wait"), timeout=0.000001),
|
||||
getPage(self.getURL("wait"), timeout=0.000001),
|
||||
defer.TimeoutError)
|
||||
def cleanup(passthrough):
|
||||
# Clean up the server which is hanging around not doing
|
||||
|
|
@ -266,7 +279,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
return finished
|
||||
|
||||
def testNotFound(self):
|
||||
return client.getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
|
||||
return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
|
||||
|
||||
def _cbNoSuchFile(self, pageData):
|
||||
self.assert_('404 - No Such Resource' in pageData)
|
||||
|
|
@ -274,7 +287,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
def testFactoryInfo(self):
|
||||
url = self.getURL('file')
|
||||
scheme, netloc, host, port, path = client._parse(url)
|
||||
factory = client.ScrapyHTTPClientFactory(url)
|
||||
factory = client.ScrapyHTTPClientFactory(Request(url))
|
||||
reactor.connectTCP(host, port, factory)
|
||||
return factory.deferred.addCallback(self._cbFactoryInfo, factory)
|
||||
|
||||
|
|
@ -285,7 +298,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
self.assertEquals(factory.response_headers['content-length'], '10')
|
||||
|
||||
def testRedirect(self):
|
||||
return client.getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
|
||||
return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
|
||||
|
||||
def _cbRedirect(self, pageData):
|
||||
self.assertEquals(pageData,
|
||||
|
|
|
|||
Loading…
Reference in New Issue