From 37ad4f879141679c0a90d0c9abc4e36ec9e102db Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Fri, 28 Oct 2011 16:33:12 -0200
Subject: [PATCH] added support for ajax crawleable urls

---
 scrapy/http/request/__init__.py   |  3 ++-
 scrapy/tests/test_http_request.py |  4 ++++
 scrapy/tests/test_utils_url.py    |  2 ++
 scrapy/utils/url.py               | 12 ++++++++++++
 4 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index ce1552bb2..c2649c967 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -12,6 +12,7 @@ from w3lib.url import safe_url_string
 from scrapy.http.headers import Headers
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.decorator import deprecated
+from scrapy.utils.url import escape_ajax
 from scrapy.http.common import deprecated_setter
 
 class Request(object_ref):
@@ -48,7 +49,7 @@ class Request(object_ref):
 
     def _set_url(self, url):
         if isinstance(url, str):
-            self._url = safe_url_string(url)
+            self._url = escape_ajax(safe_url_string(url))
         elif isinstance(url, unicode):
             if self.encoding is None:
                 raise TypeError('Cannot convert unicode url - %s has no encoding' %
diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py
index 2a3015488..f18823235 100644
--- a/scrapy/tests/test_http_request.py
+++ b/scrapy/tests/test_http_request.py
@@ -107,6 +107,10 @@ class RequestTest(unittest.TestCase):
         assert isinstance(r4.body, str)
         self.assertEqual(r4.body, "Price: \xa3100")
 
+    def test_ajax_url(self):
+        r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
+        self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key=value")
+
     def test_copy(self):
         """Test Request copy"""
         
diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py
index fdfe86955..fc17ccf6a 100644
--- a/scrapy/tests/test_utils_url.py
+++ b/scrapy/tests/test_utils_url.py
@@ -2,6 +2,8 @@ import unittest
 from scrapy.spider import BaseSpider
 from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url
 
+__doctests__ = ['scrapy.utils.url']
+
 class UrlUtilsTest(unittest.TestCase):
 
     def test_url_is_from_any_domain(self):
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 696d8b288..8a309bdf6 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -62,3 +62,15 @@ def parse_url(url, encoding=None):
     """
     return url if isinstance(url, urlparse.ParseResult) else \
         urlparse.urlparse(unicode_to_str(url, encoding))
+
+def escape_ajax(url):
+    """
+    Return the crawleable url according to:
+    http://code.google.com/web/ajaxcrawling/docs/getting-started.html
+
+    TODO: add support for urls with query arguments
+
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key=value'
+    """
+    return url.replace('#!', '?_escaped_fragment_=')