From 27018fced7ada188bfeb3057f193de678524ed90 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 23 Mar 2012 13:45:21 -0300 Subject: [PATCH] changed default user agent to Scrapy/0.15 (+http://scrapy.org) and removed no longer needed BOT_VERSION setting --- docs/topics/settings.rst | 12 +----------- scrapy/settings/default_settings.py | 3 +-- scrapy/templates/project/module/settings.py.tmpl | 4 ++-- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f69442ab8..682206db8 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -222,16 +222,6 @@ also for logging. It's automatically populated with your project name when you create your project with the :command:`startproject` command. -.. setting:: BOT_VERSION - -BOT_VERSION ------------ - -Default: ``1.0`` - -The version of the bot implemented by this Scrapy project. This will be used to -construct the User-Agent by default. - .. setting:: CONCURRENT_ITEMS CONCURRENT_ITEMS @@ -973,7 +963,7 @@ the default value for this setting see: http://www.boutell.com/newfaq/misc/urlle USER_AGENT ---------- -Default: ``"%s/%s" % (BOT_NAME, BOT_VERSION)`` +Default: ``"Scrapy/0.15 (+http://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1cd59d9bd..3baf4cdb7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,7 +17,6 @@ import sys, os from os.path import join, abspath, dirname BOT_NAME = 'scrapybot' -BOT_VERSION = '1.0' CLOSESPIDER_TIMEOUT = 0 CLOSESPIDER_PAGECOUNT = 0 @@ -226,7 +225,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) +USER_AGENT = 'Scrapy/0.15 (+http://scrapy.org)' TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 5ae6cbf5a..f00baeaba 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -7,9 +7,9 @@ # BOT_NAME = '$project_name' -BOT_VERSION = '1.0' SPIDER_MODULES = ['$project_name.spiders'] NEWSPIDER_MODULE = '$project_name.spiders' -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = '$project_name (+http://www.yourdomain.com)'