added support for ajax crawleable urls

scrapy/http/request/__init__.py

@@ -12,6 +12,7 @@ from w3lib.url import safe_url_string
 from scrapy.http.headers import Headers
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.decorator import deprecated
+from scrapy.utils.url import escape_ajax
 from scrapy.http.common import deprecated_setter
 
 class Request(object_ref):
@@ -48,7 +49,7 @@ class Request(object_ref):
 
     def _set_url(self, url):
         if isinstance(url, str):
-            self._url = safe_url_string(url)
+            self._url = escape_ajax(safe_url_string(url))
         elif isinstance(url, unicode):
             if self.encoding is None:
                 raise TypeError('Cannot convert unicode url - %s has no encoding' %

scrapy/tests/test_http_request.py

@@ -107,6 +107,10 @@ class RequestTest(unittest.TestCase):
         assert isinstance(r4.body, str)
         self.assertEqual(r4.body, "Price: \xa3100")
 
+    def test_ajax_url(self):
+        r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
+        self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key=value")
+
     def test_copy(self):
         """Test Request copy"""
         

scrapy/tests/test_utils_url.py

@@ -2,6 +2,8 @@ import unittest
 from scrapy.spider import BaseSpider
 from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url
 
+__doctests__ = ['scrapy.utils.url']
+
 class UrlUtilsTest(unittest.TestCase):
 
     def test_url_is_from_any_domain(self):

scrapy/utils/url.py

@@ -62,3 +62,15 @@ def parse_url(url, encoding=None):
     """
     return url if isinstance(url, urlparse.ParseResult) else \
         urlparse.urlparse(unicode_to_str(url, encoding))
+
+def escape_ajax(url):
+    """
+    Return the crawleable url according to:
+    http://code.google.com/web/ajaxcrawling/docs/getting-started.html
+
+    TODO: add support for urls with query arguments
+
+    >>> escape_ajax("www.example.com/ajax.html#!key=value")
+    'www.example.com/ajax.html?_escaped_fragment_=key=value'
+    """
+    return url.replace('#!', '?_escaped_fragment_=')