diff --git a/scrapy/trunk/scrapy/command/commands/scrape.py b/scrapy/trunk/scrapy/command/commands/scrape.py index 840c5f0df..4288960c9 100644 --- a/scrapy/trunk/scrapy/command/commands/scrape.py +++ b/scrapy/trunk/scrapy/command/commands/scrape.py @@ -7,6 +7,14 @@ from scrapy.utils.misc import load_class from scrapy.extension import extensions from scrapy.conf import settings +def get_url(url): + from scrapy.http import Request + from scrapy.core.downloader.handlers import download_any + request = Request(url) + spider = spiders.fromurl(url) + + return download_any(request, spider) + def load_url(url, response): vars = {} itemcls = load_class(settings['DEFAULT_ITEM_CLASS']) @@ -17,6 +25,7 @@ def load_url(url, response): vars['xxs'] = XmlXPathSelector(response) vars['hxs'] = HtmlXPathSelector(response) vars['spider'] = spiders.fromurl(url) + vars['get'] = get_url return vars def print_vars(vars): diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py index 356c51127..8e499a3b0 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py @@ -153,10 +153,9 @@ class Cache(object): url = metadata['url'] original_url = metadata.get('original_url', url) headers = Headers(responseheaders) - parent = metadata.get('parent') status = metadata['status'] - response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody, parent=parent) + response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody) response.cached = True return response @@ -168,7 +167,6 @@ class Cache(object): metadata = { 'url':request.url, 'method': request.method, - 'parent': response.parent, 'status': response.status, 'domain': response.domain, 'original_url': response.original_url, diff --git a/scrapy/trunk/scrapy/contrib/history/middleware.py b/scrapy/trunk/scrapy/contrib/history/middleware.py index 2c97bf4b0..7eb83744e 100644 --- a/scrapy/trunk/scrapy/contrib/history/middleware.py +++ b/scrapy/trunk/scrapy/contrib/history/middleware.py @@ -64,7 +64,7 @@ class HistoryMiddleware(object): key = urlkey(url) if response: redirect_url = response.url - parentkey = urlkey(response.parent) if response.parent else None + parentkey = urlkey(response.request.headers.get('referer')) if response.request else None version = response.version() else: redirect_url, parentkey, version = url, None, None diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py index 1862f191a..bbcd69aa4 100644 --- a/scrapy/trunk/scrapy/core/downloader/handlers.py +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -43,9 +43,8 @@ def download_http(request, spider): def _response(body): body = body or '' status = factory.status - parent = request.headers.get('Referer') headers = Headers(factory.response_headers) - r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body, parent=parent) + r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body) signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider) signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider) return r diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py index 40b06631a..2022e0d16 100644 --- a/scrapy/trunk/scrapy/core/engine.py +++ b/scrapy/trunk/scrapy/core/engine.py @@ -345,7 +345,7 @@ class ExecutionEngine(object): if isinstance(response, Response): response.request = request # tie request to obtained response cached = 'cached' if response.cached else 'live' - log.msg("Crawled %s <%s> from <%s>" % (cached, response.url, response.parent), level=log.DEBUG, domain=domain) + log.msg("Crawled %s <%s> from <%s>" % (cached, response.url, request.headers.get('referer')), level=log.DEBUG, domain=domain) return response elif isinstance(response, Request): redirected = response # proper alias diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py index 6c619c2d4..19e68def9 100644 --- a/scrapy/trunk/scrapy/http/response.py +++ b/scrapy/trunk/scrapy/http/response.py @@ -20,11 +20,10 @@ class Response(object) : * headers - HTTP headers * status - HTTP status code * body - Body object containing the content of the response - * parent - the URL of the referring page """ _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None, parent=None): + def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None): self.domain = domain self.url = Url(url) self.original_url = Url(original_url) if original_url else url # different if redirected or escaped @@ -34,7 +33,6 @@ class Response(object) : self.body = ResponseBody(body, self.headers_encoding()) else: self.body = body - self.parent = parent self.cached = False self.request = None # request which originated this response @@ -52,8 +50,8 @@ class Response(object) : return encoding.group(1) def __repr__(self): - return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s, parent=%s)" % \ - (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body), repr(self.parent)) + return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s)" % \ + (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body)) def __str__(self): version = '%s..%s' % (self.version()[:4], self.version()[-4:]) @@ -84,8 +82,7 @@ class Response(object) : original_url=kw.get('original_url', self.original_url), headers=kw.get('headers', sameheaders()), status=kw.get('status', self.status), - body=kw.get('body', samebody()), - parent=kw.get('parent', self.parent)) + body=kw.get('body', samebody())) return newresp def to_string(self):