removed parent attribute from Response class

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40104

scrapy/trunk/scrapy/command/commands/scrape.py

@@ -7,6 +7,14 @@ from scrapy.utils.misc import load_class
 from scrapy.extension import extensions
 from scrapy.conf import settings
 
+def get_url(url):
+    from scrapy.http import Request
+    from scrapy.core.downloader.handlers import download_any
+    request = Request(url)
+    spider = spiders.fromurl(url)
+
+    return download_any(request, spider)
+
 def load_url(url, response):
     vars = {}
     itemcls = load_class(settings['DEFAULT_ITEM_CLASS'])
@@ -17,6 +25,7 @@ def load_url(url, response):
     vars['xxs'] = XmlXPathSelector(response)
     vars['hxs'] = HtmlXPathSelector(response)
     vars['spider'] = spiders.fromurl(url)
+    vars['get'] = get_url
     return vars
 
 def print_vars(vars):

scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py

@@ -153,10 +153,9 @@ class Cache(object):
         url = metadata['url']
         original_url = metadata.get('original_url', url)
         headers = Headers(responseheaders)
-        parent = metadata.get('parent')
         status = metadata['status']
 
-        response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody, parent=parent)
+        response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody)
         response.cached = True
         return response
 
@@ -168,7 +167,6 @@ class Cache(object):
         metadata = {
                 'url':request.url,
                 'method': request.method,
-                'parent': response.parent,
                 'status': response.status,
                 'domain': response.domain,
                 'original_url': response.original_url,

scrapy/trunk/scrapy/contrib/history/middleware.py

@@ -64,7 +64,7 @@ class HistoryMiddleware(object):
         key = urlkey(url)
         if response:
             redirect_url = response.url
-            parentkey = urlkey(response.parent) if response.parent else None
+            parentkey = urlkey(response.request.headers.get('referer')) if response.request else None
             version = response.version()
         else:
             redirect_url, parentkey, version = url, None, None

scrapy/trunk/scrapy/core/downloader/handlers.py

@@ -43,9 +43,8 @@ def download_http(request, spider):
     def _response(body):
         body = body or ''
         status = factory.status
-        parent = request.headers.get('Referer')
         headers = Headers(factory.response_headers)
-        r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body, parent=parent)
+        r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body)
         signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider)
         signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider)
         return r

scrapy/trunk/scrapy/core/engine.py

@@ -345,7 +345,7 @@ class ExecutionEngine(object):
             if isinstance(response, Response):
                 response.request = request # tie request to obtained response
                 cached = 'cached' if response.cached else 'live'
-                log.msg("Crawled %s <%s> from <%s>" % (cached, response.url, response.parent), level=log.DEBUG, domain=domain)
+                log.msg("Crawled %s <%s> from <%s>" % (cached, response.url, request.headers.get('referer')), level=log.DEBUG, domain=domain)
                 return response
             elif isinstance(response, Request):
                 redirected = response # proper alias

scrapy/trunk/scrapy/http/response.py

@@ -20,11 +20,10 @@ class Response(object) :
         * headers - HTTP headers
         * status - HTTP status code
         * body - Body object containing the content of the response
-        * parent - the URL of the referring page
     """
     _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I)
 
-    def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None, parent=None):
+    def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None):
         self.domain = domain
         self.url = Url(url)
         self.original_url = Url(original_url) if original_url else url # different if redirected or escaped
@@ -34,7 +33,6 @@ class Response(object) :
             self.body = ResponseBody(body, self.headers_encoding())
         else:
             self.body = body
-        self.parent = parent
         self.cached = False
         self.request = None # request which originated this response
 
@@ -52,8 +50,8 @@ class Response(object) :
                 return encoding.group(1)
 
     def __repr__(self):
-        return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s, parent=%s)" % \
-                (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body), repr(self.parent))
+        return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s)" % \
+                (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body))
 
     def __str__(self):
         version = '%s..%s' % (self.version()[:4], self.version()[-4:])
@@ -84,8 +82,7 @@ class Response(object) :
                            original_url=kw.get('original_url', self.original_url),
                            headers=kw.get('headers', sameheaders()),
                            status=kw.get('status', self.status),
-                           body=kw.get('body', samebody()),
-                           parent=kw.get('parent', self.parent))
+                           body=kw.get('body', samebody()))
         return newresp
 
     def to_string(self):