From 503ab58f59886a2306a69b2662cd2219b80ae3d1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 19 Dec 2013 00:28:47 +0600 Subject: [PATCH] Fail-fast path. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For me middleware now can process about 2-3k ajax crawlable pages/sec and 50k+ regular pages/sec (if they don’t contain «fragment» or «content» words). --- scrapy/contrib/downloadermiddleware/ajaxcrawlable.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py index 4ccfdddd8..2bd5de59e 100644 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py @@ -75,7 +75,17 @@ def _has_ajaxcrawlable_meta(text): >>> _has_ajaxcrawlable_meta('') False """ + + # Stripping scripts and comments is slow (about 20x slower than + # just checking if a string is in text); this is a quick fail-fast + # path that should work for most pages. + if 'fragment' not in text: + return False + if 'content' not in text: + return False + text = _script_re.sub(u'', text) text = _noscript_re.sub(u'', text) text = html.remove_comments(html.remove_entities(text)) return _ajax_crawlable_re.search(text) is not None +