From b03fe049990962f254d52aea4f56a148cce62fc3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Jan 2014 23:09:37 +0600 Subject: [PATCH] Rename AjaxCrawlableMiddleware to AjaxCrawlMiddleware --- docs/topics/broad-crawls.rst | 6 ++--- docs/topics/downloader-middleware.rst | 24 +++++++++---------- .../{ajaxcrawlable.py => ajaxcrawl.py} | 18 +++++++------- scrapy/settings/default_settings.py | 4 ++-- ...test_downloadermiddleware_ajaxcrawlable.py | 14 +++++------ 5 files changed, 33 insertions(+), 33 deletions(-) rename scrapy/contrib/downloadermiddleware/{ajaxcrawlable.py => ajaxcrawl.py} (84%) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index a4b34a540..b95974f5d 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -132,12 +132,12 @@ Pages can indicate it in two ways: "main", "index" website pages. Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlableMiddleware `:: +:ref:`AjaxCrawlMiddleware `:: - AJAXCRAWLABLE_ENABLED = True + AJAXCRAWL_ENABLED = True When doing broad crawls it's common to crawl a lot of "index" web pages; -AjaxCrawlableMiddleware helps to crawl them correctly. +AjaxCrawlMiddleware helps to crawl them correctly. It is turned OFF by default because it has some performance overhead, and enabling it for focused crawls doesn't make much sense. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f763d5b73..cf963e7ae 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -797,14 +797,14 @@ UserAgentMiddleware In order for a spider to override the default user agent, its `user_agent` attribute must be set. -.. _ajaxcrawlable-middleware: +.. _ajaxcrawl-middleware: -AjaxCrawlableMiddleware ------------------------ +AjaxCrawlMiddleware +------------------- -.. module:: scrapy.contrib.downloadermiddleware.ajaxcrawlable +.. module:: scrapy.contrib.downloadermiddleware.ajaxcrawl -.. class:: AjaxCrawlableMiddleware +.. class:: AjaxCrawlMiddleware Middleware that finds 'AJAX crawlable' page variants based on meta-fragment html tag. See @@ -815,22 +815,22 @@ AjaxCrawlableMiddleware Scrapy finds 'AJAX crawlable' pages for URLs like ``'http://example.com/!#foo=bar'`` even without this middleware. - AjaxCrawlableMiddleware is necessary when URL doesn't contain ``'!#'``. + AjaxCrawlMiddleware is necessary when URL doesn't contain ``'!#'``. This is often a case for 'index' or 'main' website pages. -AjaxCrawlableMiddleware Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +AjaxCrawlMiddleware Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. setting:: AJAXCRAWLABLE_ENABLED +.. setting:: AJAXCRAWL_ENABLED -AJAXCRAWLABLE_ENABLED -^^^^^^^^^^^^^^^^^^^^^ +AJAXCRAWL_ENABLED +^^^^^^^^^^^^^^^^^ .. versionadded:: 0.21 Default: ``False`` -Whether the AjaxCrawlableMiddleware will be enabled. You may want to +Whether the AjaxCrawlMiddleware will be enabled. You may want to enable it for :ref:`broad crawls `. diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py similarity index 84% rename from scrapy/contrib/downloadermiddleware/ajaxcrawlable.py rename to scrapy/contrib/downloadermiddleware/ajaxcrawl.py index c76d1de68..c2ab67ae7 100644 --- a/scrapy/contrib/downloadermiddleware/ajaxcrawlable.py +++ b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py @@ -7,21 +7,21 @@ from scrapy.http import HtmlResponse from scrapy.utils.response import _noscript_re, _script_re from w3lib import html -class AjaxCrawlableMiddleware(object): +class AjaxCrawlMiddleware(object): """ Handle 'AJAX crawlable' pages marked as crawlable via meta tag. For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. """ def __init__(self, settings): - if not settings.getbool('AJAXCRAWLABLE_ENABLED'): + if not settings.getbool('AJAXCRAWL_ENABLED'): raise NotConfigured # XXX: Google parses at least first 100k bytes; scrapy's redirect # middleware parses first 4k. 4k turns out to be insufficient # for this middleware, and parsing 100k could be slow. # We use something in between (32K) by default. - self.lookup_bytes = settings.getint('AJAXCRAWLABLE_MAXSIZE', 32768) + self.lookup_bytes = settings.getint('AJAXCRAWL_MAXSIZE', 32768) @classmethod def from_crawler(cls, crawler): @@ -43,13 +43,13 @@ class AjaxCrawlableMiddleware(object): return response # scrapy already handles #! links properly - ajax_crawlable = request.replace(url=request.url+'#!') - log.msg(format="Downloading AJAX crawlable %(ajax_crawlable)s instead of %(request)s", - level=log.DEBUG, spider=spider, ajax_crawlable=ajax_crawlable, - request=request) + ajax_crawl_request = request.replace(url=request.url+'#!') + log.msg(format="Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", + level=log.DEBUG, spider=spider, + ajax_crawl_request=ajax_crawl_request, request=request) - ajax_crawlable.meta['ajax_crawlable'] = True - return ajax_crawlable + ajax_crawl_request.meta['ajax_crawlable'] = True + return ajax_crawl_request def _has_ajax_crawlable_variant(self, response): """ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e74f80fd7..b3736ce9e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,7 +18,7 @@ import sys from importlib import import_module from os.path import join, abspath, dirname -AJAXCRAWLABLE_ENABLED = False +AJAXCRAWL_ENABLED = False BOT_NAME = 'scrapybot' @@ -81,7 +81,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, 'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550, - 'scrapy.contrib.downloadermiddleware.ajaxcrawlable.AjaxCrawlableMiddleware': 560, + 'scrapy.contrib.downloadermiddleware.ajaxcrawl.AjaxCrawlMiddleware': 560, 'scrapy.contrib.downloadermiddleware.redirect.MetaRefreshMiddleware': 580, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 590, 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, diff --git a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py index c820f78d3..acbfb5268 100644 --- a/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/scrapy/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -1,17 +1,17 @@ import unittest -from scrapy.contrib.downloadermiddleware.ajaxcrawlable import AjaxCrawlableMiddleware +from scrapy.contrib.downloadermiddleware.ajaxcrawl import AjaxCrawlMiddleware from scrapy.spider import BaseSpider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler -__doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawlable'] +__doctests__ = ['scrapy.contrib.downloadermiddleware.ajaxcrawl'] -class AjaxCrawlableMiddlewareTest(unittest.TestCase): +class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): self.spider = BaseSpider('foo') - crawler = get_crawler({'AJAXCRAWLABLE_ENABLED': True}) - self.mw = AjaxCrawlableMiddleware.from_crawler(crawler) + crawler = get_crawler({'AJAXCRAWL_ENABLED': True}) + self.mw = AjaxCrawlMiddleware.from_crawler(crawler) def _ajaxcrawlable_body(self): return '' @@ -32,7 +32,7 @@ class AjaxCrawlableMiddlewareTest(unittest.TestCase): resp2 = self.mw.process_response(req, resp, self.spider) self.assertIs(resp, resp2) - def test_ajax_crawlable(self): + def test_ajaxcrawl(self): req, resp = self._req_resp( 'http://example.com/', {'meta': {'foo': 'bar'}}, @@ -42,7 +42,7 @@ class AjaxCrawlableMiddlewareTest(unittest.TestCase): self.assertEqual(req2.url, 'http://example.com/?_escaped_fragment_=') self.assertEqual(req2.meta['foo'], 'bar') - def test_ajax_crawlable_loop(self): + def test_ajaxcrawl_loop(self): req, resp = self._req_resp('http://example.com/', {}, {'body': self._ajaxcrawlable_body()}) req2 = self.mw.process_response(req, resp, self.spider) resp2 = HtmlResponse(req2.url, body=resp.body, request=req2)