From a87b3bd1c830ebc03066283f2f04f0bf4af15449 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 10 Jul 2013 04:30:05 +0600 Subject: [PATCH 1/7] AjaxCrawlableMiddleware --- docs/topics/downloader-middleware.rst | 37 ++++++++- .../downloadermiddleware/ajaxcrawlable.py | 80 +++++++++++++++++++ scrapy/settings/default_settings.py | 3 + ...test_downloadermiddleware_ajaxcrawlable.py | 52 ++++++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 scrapy/contrib/downloadermiddleware/ajaxcrawlable.py create mode 100644 scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0a336582c..541cff31c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -96,7 +96,7 @@ single Python class that defines one or more of the following methods: .. method:: process_response(request, response, spider) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` - object, return a :class:`~scrapy.http.Request` object or + object, return a :class:`~scrapy.http.Request` object or raise a :exc:`~scrapy.exceptions.IgnoreRequest` exception. If it returns a :class:`~scrapy.http.Response` (it could be the same given @@ -796,6 +796,41 @@ UserAgentMiddleware attribute must be set. +AjaxCrawlableMiddleware +----------------------- + +.. module:: scrapy.contrib.downloadermiddleware.ajaxcrawlable + +.. class:: AjaxCrawlableMiddleware + + Middleware that finds 'AJAX crawlable' page variants based + on meta-fragment html tag. See + https://developers.google.com/webmasters/ajax-crawling/docs/getting-started + for more info. + + .. note:: + + Scrapy finds 'AJAX crawlable' pages for URLs like + ``'http://example.com/!#foo=bar'`` even without this middleware. + AjaxCrawlableMiddleware is necessary when URL doesn't contain ``'!#'``. + This is often a case for 'index' or 'main' website pages. + +AjaxCrawlableMiddleware Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: AJAXCRAWLABLE_ENABLED + +AJAXCRAWLABLE_ENABLED +^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.17 + +Default: ``False`` + +Whether the AjaxCrawlableMiddleware will be enabled. You may want to +enable it for :ref:`broad crawls `. + + .. _DBM: http://en.wikipedia.org/wiki/Dbm .. _anydbm: http://docs.python.org/library/anydbm.html .. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py new file mode 100644 index 000000000..b796a503a --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +import re +from scrapy import log +from scrapy.exceptions import NotConfigured +from scrapy.http import HtmlResponse +from scrapy.utils.response import _noscript_re, _script_re +from w3lib import html + +class AjaxCrawlableMiddleware(object): + """ + Handle 'AJAX crawlable' pages marked as crawlable via meta tag. + For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. + """ + + # XXX: Google parses at least first 100k bytes; scrapy's redirect + # middleware parses first 4k. 4k turns out to be insufficient + # for this middleware, and parsing 100k could be slow. + _lookup_bytes = 32768 + + enabled_setting = 'AJAXCRAWLABLE_ENABLED' + + def __init__(self, settings): + if not settings.getbool(self.enabled_setting): + raise NotConfigured + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings) + + def process_response(self, request, response, spider): + + if not isinstance(response, HtmlResponse) or response.status != 200: + return response + + if request.method != 'GET': + # other HTTP methods are either not safe or don't have a body + return response + + if 'ajax_crawlable' in request.meta: # prevent loops + return response + + if not self._has_ajax_crawlable_variant(response): + return response + + # scrapy already handles #! links properly + ajax_crawlable = request.replace(url=request.url+'#!') + log.msg(format="Downloading AJAX crawlable %(ajax_crawlable)s instead of %(request)s", + level=log.DEBUG, spider=spider, ajax_crawlable=ajax_crawlable, + request=request) + + ajax_crawlable.meta['ajax_crawlable'] = True + return ajax_crawlable + + def _has_ajax_crawlable_variant(self, response): + """ + Return True if a page without hash fragment could be "AJAX crawlable" + according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. + """ + body = response.body_as_unicode()[:self._lookup_bytes] + return _has_ajaxcrawlable_meta(body) + + +# XXX: move it to w3lib? +_ajax_crawlable_re = re.compile(ur'') +def _has_ajaxcrawlable_meta(text): + """ + >>> _has_ajaxcrawlable_meta('') + True + >>> _has_ajaxcrawlable_meta("") + True + >>> _has_ajaxcrawlable_meta('') + False + >>> _has_ajaxcrawlable_meta('') + False + """ + text = _script_re.sub(u'', text) + text = _noscript_re.sub(u'', text) + text = html.remove_comments(html.remove_entities(text)) + return _ajax_crawlable_re.search(text) is not None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 91b8d8b48..e74f80fd7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,6 +18,8 @@ import sys from importlib import import_module from os.path import join, abspath, dirname +AJAXCRAWLABLE_ENABLED = False + BOT_NAME = 'scrapybot' CLOSESPIDER_TIMEOUT = 0 @@ -79,6 +81,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, + 'scrapy.contrib.downloadermiddleware.ajaxcrawlable.AjaxCrawlableMiddleware': 560, 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590, 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, diff --git a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py new file mode 100644 index 000000000..ac5d86f9f --- /dev/null +++ b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -0,0 +1,52 @@ +import unittest + +from scrapy.contrib.downloadermiddleware.ajaxcrawlable import AjaxCrawlableMiddleware +from scrapy.spider import BaseSpider +from scrapy.http import Request, HtmlResponse +from scrapy.utils.test import get_crawler + +__doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawlable'] + +class AjaxCrawlableMiddlewareTest(unittest.TestCase): + def setUp(self): + self.spider = BaseSpider('foo') + crawler = get_crawler({'AJAXCRAWLABLE_ENABLED': True}) + self.mw = AjaxCrawlableMiddleware.from_crawler(crawler) + + def _ajaxcrawlable_body(self): + return '' + + def _req_resp(self, url, req_kwargs=None, resp_kwargs=None): + req = Request(url, **(req_kwargs or {})) + resp = HtmlResponse(url, request=req, **(resp_kwargs or {})) + return req, resp + + def test_non_get(self): + req, resp = self._req_resp('http://example.com/', {'method': 'HEAD'}) + resp2 = self.mw.process_response(req, resp, self.spider) + self.assertEqual(resp, resp2) + + def test_ajax_crawlable(self): + req, resp = self._req_resp( + 'http://example.com/', + {'meta': {'foo': 'bar'}}, + {'body': self._ajaxcrawlable_body()} + ) + req2 = self.mw.process_response(req, resp, self.spider) + self.assertEqual(req2.url, 'http://example.com/?_escaped_fragment_=') + self.assertEqual(req2.meta['foo'], 'bar') + + def test_ajax_crawlable_loop(self): + req, resp = self._req_resp('http://example.com/', {}, {'body': self._ajaxcrawlable_body()}) + req2 = self.mw.process_response(req, resp, self.spider) + resp2 = HtmlResponse(req2.url, body=resp.body, request=req2) + resp3 = self.mw.process_response(req2, resp2, self.spider) + + assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3) + self.assertEqual(resp3.request.url, 'http://example.com/?_escaped_fragment_=') + assert resp3 is resp2 + + def test_noncrawlable_body(self): + req, resp = self._req_resp('http://example.com/', {}, {'body': ''}) + resp2 = self.mw.process_response(req, resp, self.spider) + assert resp2 is resp From 71c59e1c948f2e9e3dad32a8aaabf1ddd795fd39 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 19 Dec 2013 00:23:38 +0600 Subject: [PATCH 2/7] add an undocumented setting for lookup body size --- .../contrib/downloadermiddleware/ajaxcrawlable.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py index b796a503a..4ccfdddd8 100644 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py @@ -13,17 +13,18 @@ class AjaxCrawlableMiddleware(object): For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - # XXX: Google parses at least first 100k bytes; scrapy's redirect - # middleware parses first 4k. 4k turns out to be insufficient - # for this middleware, and parsing 100k could be slow. - _lookup_bytes = 32768 - enabled_setting = 'AJAXCRAWLABLE_ENABLED' def __init__(self, settings): if not settings.getbool(self.enabled_setting): raise NotConfigured + # XXX: Google parses at least first 100k bytes; scrapy's redirect + # middleware parses first 4k. 4k turns out to be insufficient + # for this middleware, and parsing 100k could be slow. + # We use something in between (32K) by default. + self.lookup_bytes = settings.getint('AJAXCRAWLABLE_MAXSIZE', 32768) + @classmethod def from_crawler(cls, crawler): return cls(crawler.settings) @@ -57,7 +58,7 @@ class AjaxCrawlableMiddleware(object): Return True if a page without hash fragment could be "AJAX crawlable" according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - body = response.body_as_unicode()[:self._lookup_bytes] + body = response.body_as_unicode()[:self.lookup_bytes] return _has_ajaxcrawlable_meta(body) From 503ab58f59886a2306a69b2662cd2219b80ae3d1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 19 Dec 2013 00:28:47 +0600 Subject: [PATCH 3/7] Fail-fast path. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For me middleware now can process about 2-3k ajax crawlable pages/sec and 50k+ regular pages/sec (if they don’t contain «fragment» or «content» words). --- scrapy/contrib/downloadermiddleware/ajaxcrawlable.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py index 4ccfdddd8..2bd5de59e 100644 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py @@ -75,7 +75,17 @@ def _has_ajaxcrawlable_meta(text): >>> _has_ajaxcrawlable_meta('') False """ + + # Stripping scripts and comments is slow (about 20x slower than + # just checking if a string is in text); this is a quick fail-fast + # path that should work for most pages. + if 'fragment' not in text: + return False + if 'content' not in text: + return False + text = _script_re.sub(u'', text) text = _noscript_re.sub(u'', text) text = html.remove_comments(html.remove_entities(text)) return _ajax_crawlable_re.search(text) is not None + From 84a3a9daac5536fa8f76927935b5235fafba9b0e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 19 Dec 2013 00:35:46 +0600 Subject: [PATCH 4/7] extra test --- .../tests/test_downloadermiddleware_ajaxcrawlable.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py index ac5d86f9f..c820f78d3 100644 --- a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -2,7 +2,7 @@ import unittest from scrapy.contrib.downloadermiddleware.ajaxcrawlable import AjaxCrawlableMiddleware from scrapy.spider import BaseSpider -from scrapy.http import Request, HtmlResponse +from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler __doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawlable'] @@ -26,6 +26,12 @@ class AjaxCrawlableMiddlewareTest(unittest.TestCase): resp2 = self.mw.process_response(req, resp, self.spider) self.assertEqual(resp, resp2) + def test_binary_response(self): + req = Request('http://example.com/') + resp = Response('http://example.com/', body=b'foobar\x00\x01\x02', request=req) + resp2 = self.mw.process_response(req, resp, self.spider) + self.assertIs(resp, resp2) + def test_ajax_crawlable(self): req, resp = self._req_resp( 'http://example.com/', @@ -49,4 +55,4 @@ class AjaxCrawlableMiddlewareTest(unittest.TestCase): def test_noncrawlable_body(self): req, resp = self._req_resp('http://example.com/', {}, {'body': ''}) resp2 = self.mw.process_response(req, resp, self.spider) - assert resp2 is resp + self.assertIs(resp, resp2) From 943a0bd264a598be4c812fa525e46f64bb28ee96 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 19 Dec 2013 01:01:26 +0600 Subject: [PATCH 5/7] AjaxCrawlableMiddleware in Broad Crawl docs --- docs/topics/broad-crawls.rst | 23 +++++++++++++++++++++++ docs/topics/downloader-middleware.rst | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 189620a0b..3264ebe23 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -118,3 +118,26 @@ crawler to dedicate too many resources on any specific domain. To disable redirects use:: REDIRECT_ENABLED = False + +Enable crawling of "Ajax Crawlable Pages" +========================================= + +Some pages (up to 1%) declare themselves as `ajax crawlable`_. This means they +provide plain HTML version of content that is usually available only via AJAX. +Pages can indicate it in two ways: + +1) by using ``#!`` in URL - this is the default way; +2) by using a special meta tag - this way is used on + "main", "index" website pages. + +Scrapy handles (1) automatically; to handle (2) enable +:ref:`AjaxCrawlableMiddleware `:: + + AJAXCRAWLABLE_ENABLED = True + +When doing broad crawls it's common to crawl a lot of "index" web pages; +AjaxCrawlableMiddleware helps to crawl them correctly. +It is turned OFF by default because it has some performance overhead, +and enabling it for focused crawls doesn't make much sense. + +.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 541cff31c..82c01d1e6 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -795,6 +795,7 @@ UserAgentMiddleware In order for a spider to override the default user agent, its `user_agent` attribute must be set. +.. _ajaxcrawlable-middleware: AjaxCrawlableMiddleware ----------------------- @@ -823,7 +824,7 @@ AjaxCrawlableMiddleware Settings AJAXCRAWLABLE_ENABLED ^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: 0.17 +.. versionadded:: 0.21 Default: ``False`` From ee46ec892061f01656e294441b2836a2f5b722c8 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 20 Dec 2013 19:08:49 +0600 Subject: [PATCH 6/7] kill AjaxCrawlableMiddlewar.enabled_setting --- scrapy/contrib/downloadermiddleware/ajaxcrawlable.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py index 2bd5de59e..c76d1de68 100644 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py @@ -13,10 +13,8 @@ class AjaxCrawlableMiddleware(object): For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ - enabled_setting = 'AJAXCRAWLABLE_ENABLED' - def __init__(self, settings): - if not settings.getbool(self.enabled_setting): + if not settings.getbool('AJAXCRAWLABLE_ENABLED'): raise NotConfigured # XXX: Google parses at least first 100k bytes; scrapy's redirect From e0cebbfc8f47eebfb1b10796f4798193e6ef9e08 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 20 Dec 2013 23:12:37 +0600 Subject: [PATCH 7/7] add a remark about 1% --- docs/topics/broad-crawls.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 3264ebe23..a4b34a540 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -122,8 +122,9 @@ To disable redirects use:: Enable crawling of "Ajax Crawlable Pages" ========================================= -Some pages (up to 1%) declare themselves as `ajax crawlable`_. This means they -provide plain HTML version of content that is usually available only via AJAX. +Some pages (up to 1%, based on empirical data from year 2013) declare +themselves as `ajax crawlable`_. This means they provide plain HTML +version of content that is usually available only via AJAX. Pages can indicate it in two ways: 1) by using ``#!`` in URL - this is the default way;