performance improvement to cookie handling when a single spider is crawling a large number of domains

scrapy/http/cookies.py

@@ -1,11 +1,15 @@
-from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy
+import time
+from cookielib import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
 
 from scrapy.utils.httpobj import urlparse_cached
 
 class CookieJar(object):
-    def __init__(self, policy=None):
-        self.jar = _CookieJar(policy or DefaultCookiePolicy())
+    def __init__(self, policy=None, check_expired_frequency=10000):
+        self.policy = policy or DefaultCookiePolicy()
+        self.jar = _CookieJar(self.policy)
         self.jar._cookies_lock = _DummyLock()
+        self.check_expired_frequency = check_expired_frequency
+        self.processed = 0
 
     def extract_cookies(self, response, request):
         wreq = WrappedRequest(request)
@@ -14,8 +18,42 @@ class CookieJar(object):
 
     def add_cookie_header(self, request):
         wreq = WrappedRequest(request)
-        self.jar.add_cookie_header(wreq)
+        self.policy._now = self.jar._now = int(time.time())
+        
+        # the cookiejar implementation iterates through all domains
+        # instead we restrict to potential matches on the domain
+        
+        req_host = urlparse_cached(request).netloc
+        if not IPV4_RE.search(req_host):
+            hosts = potential_domain_matches(req_host)
+            if req_host.find(".") == -1:
+                hosts += req_host + ".local"
+        else:
+            hosts = [req_host]
 
+        cookies = []
+        for host in hosts:
+            if host in self.jar._cookies:
+                cookies += self.jar._cookies_for_domain(host, wreq)
+
+        attrs = self.jar._cookie_attrs(cookies)
+        if attrs:
+            if not wreq.has_header("Cookie"):
+                wreq.add_unredirected_header("Cookie", "; ".join(attrs))
+
+        # if necessary, advertise that we know RFC 2965
+        if (self.policy.rfc2965 and not self.policy.hide_cookie2 and
+            not request.has_header("Cookie2")):
+            for cookie in cookies:
+                if cookie.version != 1:
+                    request.add_unredirected_header("Cookie2", '$Version="1"')
+                    break
+        
+        self.processed += 1
+        if self.processed % self.check_expired_frequency == 0:
+            # This is still quite inefficient for large number of cookies
+            self.jar.clear_expired_cookies()
+    
     @property
     def _cookies(self):
         return self.jar._cookies
@@ -47,6 +85,24 @@ class CookieJar(object):
         self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))
 
 
+def potential_domain_matches(domain):
+    """Potential domain matches for a cookie
+    
+    >>> potential_domain_matches('www.example.com')
+    ['www.example.com', 'example.com']
+
+    """
+    matches = [domain]
+    try:
+        start = domain.index('.') + 1
+        end = domain.rindex('.')
+        while start < end:
+            matches.append(domain[start:])
+            start = domain.index('.', start) + 1
+    except ValueError:
+        pass
+    return matches
+
 class _DummyLock(object):
     def acquire(self):
         pass