diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ce1552bb2..c2649c967 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,6 +12,7 @@ from w3lib.url import safe_url_string from scrapy.http.headers import Headers from scrapy.utils.trackref import object_ref from scrapy.utils.decorator import deprecated +from scrapy.utils.url import escape_ajax from scrapy.http.common import deprecated_setter class Request(object_ref): @@ -48,7 +49,7 @@ class Request(object_ref): def _set_url(self, url): if isinstance(url, str): - self._url = safe_url_string(url) + self._url = escape_ajax(safe_url_string(url)) elif isinstance(url, unicode): if self.encoding is None: raise TypeError('Cannot convert unicode url - %s has no encoding' % diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index 2a3015488..f18823235 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -107,6 +107,10 @@ class RequestTest(unittest.TestCase): assert isinstance(r4.body, str) self.assertEqual(r4.body, "Price: \xa3100") + def test_ajax_url(self): + r = self.request_class(url="http://www.example.com/ajax.html#!key=value") + self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key=value") + def test_copy(self): """Test Request copy""" diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index fdfe86955..fc17ccf6a 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -2,6 +2,8 @@ import unittest from scrapy.spider import BaseSpider from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url +__doctests__ = ['scrapy.utils.url'] + class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 696d8b288..8a309bdf6 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -62,3 +62,15 @@ def parse_url(url, encoding=None): """ return url if isinstance(url, urlparse.ParseResult) else \ urlparse.urlparse(unicode_to_str(url, encoding)) + +def escape_ajax(url): + """ + Return the crawleable url according to: + http://code.google.com/web/ajaxcrawling/docs/getting-started.html + + TODO: add support for urls with query arguments + + >>> escape_ajax("www.example.com/ajax.html#!key=value") + 'www.example.com/ajax.html?_escaped_fragment_=key=value' + """ + return url.replace('#!', '?_escaped_fragment_=')