RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SEP12 implementation 

* Rename BaseSpider.domain_name to BaseSpider.name

    This patch implements the domain_name to name change in BaseSpider class and
    change all spider instantiations to use the new attribute.

  * Add allowed_domains to spider

    This patch implements the merging of spider.domain_name and
    spider.extra_domain_names in spider.allowed_domains for offsite checking
    purposes.

    Note that spider.domain_name is not touched by this patch, only not used.

  * Remove spider.domain_name references from scrapy.stats

    * Rename domain_stats to spider_stats in MemoryStatsCollector
    * Use ``spider`` instead of ``domain`` in SimpledbStatsCollector
    * Rename domain_stats_history table to spider_data_history and rename domain
    field to spider in MysqlStatsCollector

  * Refactor genspider command

    The new signature for genspider is: genspider [options] <domain_name>.

    Genspider uses domain_name for spider name and for the module name.

  * Remove spider.domain_name references

  * Update crawl command signature <spider|url>

  * docs: updated references to domain_name

  * examples/experimental: use spider.name

  * genspider: require <name> <domain>

  * spidermanager: renamed crawl_domain to crawl_spider_name

  * spiderctl: updated references of *domain* to spider

  * added backward compatiblity with legacy spider's attributes
    'domain_name' and 'extra_domain_names'
This commit is contained in:
Rolando Espinoza La fuente 2010-04-01 18:27:22 -03:00
parent 35a7059636
commit db5c3df679
41 changed files with 273 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/intro/overview.rst
View File

@ -128,7 +128,8 @@ Finally, here's the spider code::
class MininovaSpider(CrawlSpider):
domain_name = 'mininova.org'
name = 'mininova.org'
allowed_domains = ['mininova.org']
start_urls = ['http://www.mininova.org/today']
rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]

13
docs/intro/tutorial.rst
View File

@ -102,8 +102,8 @@ to parse the contents of those pages to extract :ref:`items <topics-items>`.
To create a Spider, you must subclass :class:`scrapy.spider.BaseSpider`, and
define the three main, mandatory, attributes:
* :attr:`~scrapy.spider.BaseSpider.domain_name`: identifies the Spider. It must
be unique, that is, you can't set the same domain name for different Spiders.
* :attr:`~scrapy.spider.BaseSpider.name`: identifies the Spider. It must be
unique, that is, you can't set the same name for different Spiders.
* :attr:`~scrapy.spider.BaseSpider.start_urls`: is a list of URLs where the
Spider will begin to crawl from. So, the first pages downloaded will be those
@ -128,7 +128,8 @@ This is the code for our first Spider, save it in a file named
from scrapy.spider import BaseSpider
class DmozSpider(BaseSpider):
domain_name = "dmoz.org"
name = "dmoz.org"
allowed_domains = ["dmoz.org"]
start_urls = [
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
@ -354,7 +355,8 @@ Let's add this code to our spider::
from scrapy.selector import HtmlXPathSelector
class DmozSpider(BaseSpider):
domain_name = "dmoz.org"
name = "dmoz.org"
allowed_domains = ["dmoz.org"]
start_urls = [
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
@ -398,7 +400,8 @@ scraped so far, the code for our Spider should be like this::
from dmoz.items import DmozItem
class DmozSpider(BaseSpider):
domain_name = "dmoz.org"
name = "dmoz.org"
allowed_domains = ["dmoz.org"]
start_urls = [
"http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
"http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"

2
docs/topics/downloader-middleware.rst
View File

@ -199,7 +199,7 @@ HttpAuthMiddleware
http_user = 'someuser'
http_pass = 'somepass'
domain_name = 'intranet.example.com'
name = 'intranet.example.com'
# .. rest of the spider code omitted ...

2
docs/topics/exporters.rst
View File

@ -52,7 +52,7 @@ Exporter to export scraped items to different files, one per spider::
self.files = {}
def spider_opened(self, spider):
file = open('%s_products.xml' % spider.domain_name, 'w+b')
file = open('%s_products.xml' % spider.name, 'w+b')
self.files[spider] = file
self.exporter = XmlItemExporter(file)
self.exporter.start_exporting()

4
docs/topics/extensions.rst
View File

@ -105,10 +105,10 @@ everytime a domain/spider is opened and closed::
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
def spider_opened(self, spider):
log.msg("opened spider %s" % spider.domain_name)
log.msg("opened spider %s" % spider.name)
def spider_closed(self, spider):
log.msg("closed spider %s" % spider.domain_name)
log.msg("closed spider %s" % spider.name)
.. _topics-extensions-ref-manager:

3
docs/topics/firebug.rst
View File

@ -79,7 +79,8 @@ This is how the spider would look so far::
from scrapy.contrib.spiders import CrawlSpider, Rule
class GoogleDirectorySpider(CrawlSpider):
domain_name = 'directory.google.com'
name = 'directory.google.com'
allowed_domains = ['directory.google.com']
start_urls = ['http://directory.google.com/']
rules = (

2
docs/topics/request-response.rst
View File

@ -321,7 +321,7 @@ user name and password. You can use the :meth:`FormRequest.from_response`
method for this job. Here's an example spider which uses it::
class LoginSpider(BaseSpider):
domain_name = 'example.com'
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
def parse(self, response):

2
docs/topics/shell.rst
View File

@ -163,7 +163,7 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function.
Here's an example of how you would call it from your spider::
class MySpider(BaseSpider):
domain_name = 'example.com'
...
def parse(self, response):
if response.url == 'http://www.example.com/products.php':

7
docs/topics/spider-middleware.rst
View File

@ -210,11 +210,8 @@ OffsiteMiddleware
Filters out Requests for URLs outside the domains covered by the spider.
This middleware filters out every request whose host names don't match
:attr:`~scrapy.spider.BaseSpider.domain_name`, or the spider
:attr:`~scrapy.spider.BaseSpider.domain_name` prefixed by "www.".
Spider can add more domains to exclude using
:attr:`~scrapy.spider.BaseSpider.extra_domain_names` attribute.
This middleware filters out every request whose host names aren't in the
spider's :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute.
When your spider returns a request for a domain not belonging to those
covered by the spider, this middleware will log a debug message similar to

41
docs/topics/spiders.rst
View File

@ -70,20 +70,22 @@ BaseSpider
requests the given ``start_urls``/``start_requests``, and calls the spider's
method ``parse`` for each of the resulting responses.
.. attribute:: domain_name
.. attribute:: name
A string which defines the domain name for this spider, which will also be
the unique identifier for this spider (which means you can't have two
spider with the same ``domain_name``). This is the most important spider
attribute and it's required, and it's the name by which Scrapy will known
the spider.
A string which defines the name for this spider. The spider name is how
the spider is located (and instantiated) by Scrapy, so it must be
unique. However, nothing prevents you from instantiating more than one
instance of the same spider. This is the most important spider attribute
and it's required.
.. attribute:: extra_domain_names
Is recommended to name your spiders after the domain that their crawl.
An optional list of strings containing additional domains that this
spider is allowed to crawl. Requests for URLs not belonging to the
domain name specified in :attr:`domain_name` or this list won't be
followed.
.. attribute:: allowed_domains
An optional list of strings containing domains that this spider is
allowed to crawl. Requests for URLs not belonging to the domain names
specified in this list won't be followed if
:class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled.
.. attribute:: start_urls
@ -144,7 +146,7 @@ BaseSpider
.. method:: log(message, [level, component])
Log a message using the :func:`scrapy.log.msg` function, automatically
populating the domain argument with the :attr:`domain_name` of this
populating the spider argument with the :attr:`name` of this
spider. For more information see :ref:`topics-logging`.
@ -157,7 +159,8 @@ Let's see an example::
from scrapy.spider import BaseSpider
class MySpider(BaseSpider):
domain_name = 'http://www.example.com'
name = 'example.com'
allowed_domains = ['example.com']
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
@ -177,7 +180,8 @@ Another example returning multiples Requests and Items from a single callback::
from myproject.items import MyItem
class MySpider(BaseSpider):
domain_name = 'http://www.example.com'
name = 'example.com'
allowed_domains = ['example.com']
start_urls = [
'http://www.example.com/1.html',
'http://www.example.com/2.html',
@ -254,7 +258,8 @@ Let's now take a look at an example CrawlSpider with rules::
from scrapy.item import Item
class MySpider(CrawlSpider):
domain_name = 'example.com'
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com']
rules = (
@ -378,7 +383,8 @@ These spiders are pretty easy to use, let's have at one example::
from myproject.items import TestItem
class MySpider(XMLFeedSpider):
domain_name = 'example.com'
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.xml']
iterator = 'iternodes' # This is actually unnecesary, since it's the default value
itertag = 'item'
@ -435,7 +441,8 @@ Let's see an example similar to the previous one, but using a
from myproject.items import TestItem
class MySpider(CSVFeedSpider):
domain_name = 'example.com'
name = 'example.com'
allowed_domains = ['example.com']
start_urls = ['http://www.example.com/feed.csv']
delimiter = ';'
headers = ['id', 'name', 'description']

14
docs/topics/stats.rst
View File

@ -204,15 +204,15 @@ MemoryStatsCollector
A simple stats collector that keeps the stats of the last scraping run (for
each spider) in memory, after they're closed. The stats can be accessed
through the :attr:`domain_stats` attribute, which is a dict keyed by spider
through the :attr:`spider_stats` attribute, which is a dict keyed by spider
domain name.
This is the default Stats Collector used in Scrapy.
.. attribute:: domain_stats
.. attribute:: spider_stats
A dict of dicts (keyed by spider domain name) containing the stats of
the last scraping run for each domain.
A dict of dicts (keyed by spider name) containing the stats of the last
scraping run for each spider.
DummyStatsCollector
-------------------
@ -240,11 +240,11 @@ SimpledbStatsCollector
In addition to the existing stats keys the following keys are added at
persitance time:
* ``domain``: the spider domain (so you can use it later for querying stats
for that domain)
* ``spider``: the spider name (so you can use it later for querying stats
for that spider)
* ``timestamp``: the timestamp when the stats were persisited
Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB
Both the ``spider`` and ``timestamp`` are used for generating the SimpleDB
item name in order to avoid overwriting stats of previous scraping runs.
As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and

3
examples/experimental/googledir/googledir/spiders/google_directory.py
View File

@ -6,7 +6,8 @@ from googledir.items import GoogledirItem
class GoogleDirectorySpider(CrawlSpider):
domain_name = 'directory.google.com'
name = 'google_directory'
allowed_domains = ['directory.google.com']
start_urls = ['http://directory.google.com/']
rules = (

3
examples/experimental/imdb/imdb/spiders/imdb_site.py
View File

@ -29,7 +29,8 @@ class MovieItem(ImdbItem):
class ImdbSiteSpider(CrawlSpider):
domain_name = 'imdb.com'
name = 'imdb.com'
allowed_domains = ['imdb.com']
start_urls = ['http://www.imdb.com/']
# extract requests using this classes from urls matching 'follow' flag

3
examples/googledir/googledir/spiders/google_directory.py
View File

@ -6,7 +6,8 @@ from googledir.items import GoogledirItem
class GoogleDirectorySpider(CrawlSpider):
domain_name = 'directory.google.com'
name = 'directory.google.com'
allow_domains = ['directory.google.com']
start_urls = ['http://directory.google.com/']
rules = (

19
scrapy/command/commands/crawl.py
View File

@ -13,10 +13,10 @@ class Command(ScrapyCommand):
requires_project = True
def syntax(self):
return "[options] <domain|url> ..."
return "[options] <spider|url> ..."
def short_desc(self):
return "Start crawling a domain or URL"
return "Start crawling from a spider or URL"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
@ -31,10 +31,9 @@ class Command(ScrapyCommand):
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def run(self, args, opts):
urls, domains = self._split_urls_and_domains(args)
for dom in domains:
scrapymanager.crawl_domain(dom)
urls, names = self._split_urls_and_names(args)
for name in names:
scrapymanager.crawl_spider_name(name)
if opts.spider:
try:
@ -65,12 +64,12 @@ class Command(ScrapyCommand):
spider_urls[spider_names[0]].append(url)
return spider_urls.items()
def _split_urls_and_domains(self, args):
def _split_urls_and_names(self, args):
urls = []
domains = []
names = []
for arg in args:
if is_url(arg):
urls.append(arg)
else:
domains.append(arg)
return urls, domains
names.append(arg)
return urls, names

28
scrapy/command/commands/genspider.py
View File

@ -15,10 +15,11 @@ SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders')
def sanitize_module_name(module_name):
"""Sanitize the given module name, by replacing dashes with underscores and
prefixing it with a letter if it doesn't start with one
"""Sanitize the given module name, by replacing dashes and points
with underscores and prefixing it with a letter if it doesn't start
with one
"""
module_name = module_name.replace('-', '_')
module_name = module_name.replace('-', '_').replace('.', '_')
if module_name[0] not in string.ascii_letters:
module_name = "a" + module_name
return module_name
@ -28,7 +29,7 @@ class Command(ScrapyCommand):
requires_project = True
def syntax(self):
return "[options] <spider_module_name> <spider_domain_name>"
return "[options] <name> <domain>"
def short_desc(self):
return "Generate new spider based on template passed with -t or --template"
@ -54,34 +55,37 @@ class Command(ScrapyCommand):
print template.read()
return
if len(args) < 2:
if len(args) != 2:
return False
module = sanitize_module_name(args[0])
name = args[0]
domain = args[1]
module = sanitize_module_name(name)
# if spider already exists and not force option then halt
try:
spider = spiders.create(domain)
spider = spiders.create(name)
except KeyError:
pass
else:
if not opts.force:
print "Spider '%s' already exists in module:" % domain
print "Spider '%s' already exists in module:" % name
print " %s" % spider.__module__
sys.exit(1)
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, domain, opts.template, template_file)
self._genspider(module, name, domain, opts.template, template_file)
def _genspider(self, module, domain, template_name, template_file):
def _genspider(self, module, name, domain, template_name, template_file):
"""Generate the spider module, based on the given template"""
tvars = {
'project_name': settings.get('BOT_NAME'),
'ProjectName': string_camelcase(settings.get('BOT_NAME')),
'module': module,
'site': domain,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join([s.capitalize() \
for s in module.split('_')])
}
@ -92,7 +96,7 @@ class Command(ScrapyCommand):
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print "Created spider %r using template %r in module:" % (domain, \
print "Created spider %r using template %r in module:" % (name, \
template_name)
print " %s.%s" % (spiders_module.__name__, module)

4
scrapy/command/commands/parse.py
View File

@ -46,7 +46,7 @@ class Command(ScrapyCommand):
if callback:
callback_fcn = callback if callable(callback) else getattr(spider, callback, None)
if not callback_fcn:
log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name))
log.msg('Cannot find callback %s in %s spider' % (callback, spider.name))
return (), ()
result = iterate_spider_output(callback_fcn(response))
@ -130,7 +130,7 @@ class Command(ScrapyCommand):
else:
log.msg('No rules found for spider "%s", ' \
'please specify a callback for parsing' \
% spider.domain_name, log.ERROR)
% spider.name, log.ERROR)
else:
# default callback 'parse'
items, links = self.run_callback(spider, response,

2
scrapy/contrib/aws.py
View File

@ -20,7 +20,7 @@ class AWSMiddleware(object):
def process_request(self, request, spider):
hostname = urlparse_cached(request).hostname
if spider.domain_name == 's3.amazonaws.com' \
if spider.name == 's3.amazonaws.com' \
or (hostname and hostname.endswith('s3.amazonaws.com')):
request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \
time.gmtime())

2
scrapy/contrib/downloadermiddleware/httpcache.py
View File

@ -108,7 +108,7 @@ class FilesystemCacheStorage(object):
def _get_request_path(self, spider, request):
key = request_fingerprint(request)
return join(self.cachedir, spider.domain_name, key[0:2], key)
return join(self.cachedir, spider.name, key[0:2], key)
def _read_meta(self, spider, request):
rpath = self._get_request_path(spider, request)

20
scrapy/contrib/itemsampler.py
View File

@ -1,6 +1,6 @@
"""
This module provides a mechanism for collecting one (or more) sample items per
domain.
spider.
The items are collected in a dict of guid->item and persisted by pickling that
dict into a file.
@ -8,7 +8,7 @@ dict into a file.
This can be useful for testing changes made to the framework or other common
code that affects several spiders.
It uses the scrapy stats service to keep track of which domains are already
It uses the scrapy stats service to keep track of which spiders are already
sampled.
Settings that affect this module:
@ -48,7 +48,7 @@ class ItemSamplerPipeline(object):
raise NotConfigured
self.items = {}
self.spiders_count = 0
self.empty_domains = set()
self.empty_spiders = set()
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
@ -66,21 +66,21 @@ class ItemSamplerPipeline(object):
def engine_stopped(self):
with open(self.filename, 'w') as f:
pickle.dump(self.items, f)
if self.empty_domains:
log.msg("No products sampled for: %s" % " ".join(self.empty_domains), \
if self.empty_spiders:
log.msg("No products sampled for: %s" % " ".join(self.empty_spiders), \
level=log.WARNING)
def spider_closed(self, spider, reason):
if reason == 'finished' and not stats.get_value("items_sampled", spider=spider):
self.empty_domains.add(spider.domain_name)
self.empty_spiders.add(spider.name)
self.spiders_count += 1
log.msg("Sampled %d domains so far (%d empty)" % (self.spiders_count, \
len(self.empty_domains)), level=log.INFO)
log.msg("Sampled %d spiders so far (%d empty)" % (self.spiders_count, \
len(self.empty_spiders)), level=log.INFO)
class ItemSamplerMiddleware(object):
"""This middleware drops items and requests (when domain sampling has been
completed) to accelerate the processing of remaining domains"""
"""This middleware drops items and requests (when spider sampling has been
completed) to accelerate the processing of remaining spiders"""
def __init__(self):
if not settings['ITEMSAMPLER_FILE']:

4
scrapy/contrib/pipeline/images.py
View File

@ -47,7 +47,7 @@ class FSImagesStore(object):
dispatcher.connect(self.spider_closed, signals.spider_closed)
def spider_closed(self, spider):
self.created_directories.pop(spider.domain_name, None)
self.created_directories.pop(spider.name, None)
def persist_image(self, key, image, buf, info):
absolute_path = self._get_filesystem_path(key)
@ -92,7 +92,7 @@ class _S3AmazonAWSSpider(BaseSpider):
It means that a spider that uses download_delay or alike is not going to be
delayed even more because it is uploading images to s3.
"""
domain_name = "s3.amazonaws.com"
name = "s3.amazonaws.com"
start_urls = ['http://s3.amazonaws.com/']
max_concurrent_requests = 100

10
scrapy/contrib/spidermanager.py
View File

@ -53,7 +53,7 @@ class TwistedPluginSpiderManager(object):
for module in modules:
for spider in self._getspiders(ISpider, module):
ISpider.validateInvariants(spider)
self._spiders[spider.domain_name] = spider
self._spiders[spider.name] = spider
self.loaded = True
def _getspiders(self, interface, package):
@ -76,14 +76,14 @@ class TwistedPluginSpiderManager(object):
"""Reload spider module to release any resources held on to by the
spider
"""
domain = spider.domain_name
if domain not in self._spiders:
name = spider.name
if name not in self._spiders:
return
spider = self._spiders[domain]
spider = self._spiders[name]
module_name = spider.__module__
module = sys.modules[module_name]
if hasattr(module, 'SPIDER'):
log.msg("Reloading module %s" % module_name, spider=spider, \
level=log.DEBUG)
new_module = rebuild(module, doLog=0)
self._spiders[domain] = new_module.SPIDER
self._spiders[name] = new_module.SPIDER

3
scrapy/contrib/spidermiddleware/offsite.py
View File

@ -47,8 +47,7 @@ class OffsiteMiddleware(object):
return re.compile(regex)
def spider_opened(self, spider):
domains = [spider.domain_name] + spider.extra_domain_names
self.host_regexes[spider] = self.get_host_regex(domains)
self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains)
self.domains_seen[spider] = set()
def spider_closed(self, spider):

4
scrapy/contrib/statsmailer.py
View File

@ -23,6 +23,6 @@ class StatsMailer(object):
mail = MailSender()
body = "Global stats\n\n"
body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items())
body += "\n\n%s stats\n\n" % spider.domain_name
body += "\n\n%s stats\n\n" % spider.name
body += "\n".join("%-50s : %s" % i for i in spider_stats.items())
mail.send(self.recipients, "Scrapy stats for: %s" % spider.domain_name, body)
mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body)

2
scrapy/contrib/webconsole/livestats.py
View File

@ -60,7 +60,7 @@ class LiveStats(object):
runtime = datetime.now() - stats.started
s += '<tr><td>%s</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td align="right">%d</td><td>%s</td><td>%s</td></tr>\n' % \
(spider.domain_name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime))
(spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime))
totdomains += 1
totscraped += stats.scraped

82
scrapy/contrib/webconsole/spiderctl.py
View File

@ -25,18 +25,18 @@ class Spiderctl(object):
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
def spider_opened(self, spider):
self.running[spider.domain_name] = spider
self.running[spider.name] = spider
def spider_closed(self, spider):
del self.running[spider.domain_name]
self.finished.add(spider.domain_name)
del self.running[spider.name]
self.finished.add(spider.name)
def webconsole_render(self, wc_request):
if wc_request.args:
changes = self.webconsole_control(wc_request)
self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders]
self.idle = [d for d in self.enabled_domains if d not in self.scheduled
self.scheduled = [s.name for s in scrapyengine.spider_scheduler._pending_spiders]
self.idle = [d for d in self.enabled_spiders if d not in self.scheduled
and d not in self.running
and d not in self.finished]
@ -53,9 +53,9 @@ class Spiderctl(object):
# idle
s += "<td valign='top'>\n"
s += '<form method="post" action=".">\n'
s += '<select name="add_pending_domains" multiple="multiple">\n'
for domain in sorted(self.idle):
s += "<option>%s</option>\n" % domain
s += '<select name="add_pending_spiders" multiple="multiple">\n'
for name in sorted(self.idle):
s += "<option>%s</option>\n" % name
s += '</select><br>\n'
s += '<br />'
s += '<input type="submit" value="Schedule selected">\n'
@ -65,9 +65,9 @@ class Spiderctl(object):
# scheduled
s += "<td valign='top'>\n"
s += '<form method="post" action=".">\n'
s += '<select name="remove_pending_domains" multiple="multiple">\n'
for domain in self.scheduled:
s += "<option>%s</option>\n" % domain
s += '<select name="remove_pending_spiders" multiple="multiple">\n'
for name in self.scheduled:
s += "<option>%s</option>\n" % name
s += '</select><br>\n'
s += '<br />'
s += '<input type="submit" value="Remove selected">\n'
@ -78,9 +78,9 @@ class Spiderctl(object):
# running
s += "<td valign='top'>\n"
s += '<form method="post" action=".">\n'
s += '<select name="stop_running_domains" multiple="multiple">\n'
for domain in sorted(self.running):
s += "<option>%s</option>\n" % domain
s += '<select name="stop_running_spiders" multiple="multiple">\n'
for name in sorted(self.running):
s += "<option>%s</option>\n" % name
s += '</select><br>\n'
s += '<br />'
s += '<input type="submit" value="Stop selected">\n'
@ -90,9 +90,9 @@ class Spiderctl(object):
# finished
s += "<td valign='top'>\n"
s += '<form method="post" action=".">\n'
s += '<select name="rerun_finished_domains" multiple="multiple">\n'
for domain in sorted(self.finished):
s += "<option>%s</option>\n" % domain
s += '<select name="rerun_finished_spiders" multiple="multiple">\n'
for name in sorted(self.finished):
s += "<option>%s</option>\n" % name
s += '</select><br>\n'
s += '<br />'
s += '<input type="submit" value="Re-schedule selected">\n'
@ -114,42 +114,42 @@ class Spiderctl(object):
args = wc_request.args
s = "<hr />\n"
if "stop_running_domains" in args:
if "stop_running_spiders" in args:
s += "<p>"
stopped_domains = []
for domain in args["stop_running_domains"]:
if domain in self.running:
scrapyengine.close_spider(self.running[domain])
stopped_domains.append(domain)
s += "Stopped spiders: <ul><li>%s</li></ul>" % "</li><li>".join(stopped_domains)
stopped_spiders = []
for name in args["stop_running_spiders"]:
if name in self.running:
scrapyengine.close_spider(self.running[name])
stopped_spiders.append(name)
s += "Stopped spiders: <ul><li>%s</li></ul>" % "</li><li>".join(stopped_spiders)
s += "</p>"
if "remove_pending_domains" in args:
if "remove_pending_spiders" in args:
removed = []
for domain in args["remove_pending_domains"]:
if scrapyengine.spider_scheduler.remove_pending_domain(domain):
removed.append(domain)
for name in args["remove_pending_spiders"]:
if scrapyengine.spider_scheduler.remove_pending_spider(name):
removed.append(name)
if removed:
s += "<p>"
s += "Removed scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["remove_pending_domains"])
s += "Removed scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["remove_pending_spiders"])
s += "</p>"
if "add_pending_domains" in args:
for domain in args["add_pending_domains"]:
if domain not in scrapyengine.scheduler.pending_requests:
scrapymanager.crawl_domain(domain)
if "add_pending_spiders" in args:
for name in args["add_pending_spiders"]:
if name not in scrapyengine.scheduler.pending_requests:
scrapymanager.crawl_spider_name(name)
s += "<p>"
s += "Scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["add_pending_domains"])
s += "Scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["add_pending_spiders"])
s += "</p>"
if "rerun_finished_domains" in args:
for domain in args["rerun_finished_domains"]:
if domain not in scrapyengine.scheduler.pending_requests:
scrapymanager.crawl_domain(domain)
self.finished.remove(domain)
if "rerun_finished_spiders" in args:
for name in args["rerun_finished_spiders"]:
if name not in scrapyengine.scheduler.pending_requests:
scrapymanager.crawl_spider_name(name)
self.finished.remove(name)
s += "<p>"
s += "Re-scheduled finished spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["rerun_finished_domains"])
s += "Re-scheduled finished spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["rerun_finished_spiders"])
s += "</p>"
return s
def webconsole_discover_module(self):
self.enabled_domains = spiders.list()
self.enabled_spiders = spiders.list()
return self

2
scrapy/contrib/webconsole/stats.py
View File

@ -23,7 +23,7 @@ class StatsDump(object):
s += "<h3>Global stats</h3>\n"
s += stats_html_table(stats.get_stats())
for spider, spider_stats in stats.iter_spider_stats():
s += "<h3>%s</h3>\n" % spider.domain_name
s += "<h3>%s</h3>\n" % spider.name
s += stats_html_table(spider_stats)
s += "</body>\n"
s += "</html>\n"

8
scrapy/core/manager.py
View File

@ -54,12 +54,12 @@ class ExecutionManager(object):
if spider:
scrapyengine.crawl(request, spider)
def crawl_domain(self, domain):
"""Schedule given domain for crawling."""
def crawl_spider_name(self, name):
"""Schedule given spider by name for crawling."""
try:
spider = spiders.create(domain)
spider = spiders.create(name)
except KeyError:
log.msg('Could not find spider for domain: %s' % domain, log.ERROR)
log.msg('Could not find spider: %s' % name, log.ERROR)
else:
self.crawl_spider(spider)

4
scrapy/log.py
View File

@ -75,7 +75,7 @@ def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None):
"use 'spider' argument instead", DeprecationWarning, stacklevel=2)
dispatcher.send(signal=logmessage_received, message=message, level=level, \
spider=spider)
system = domain or (spider.domain_name if spider else component)
system = domain or (spider.name if spider else component)
msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding)
log.msg(msg_txt, system=system)
@ -93,7 +93,7 @@ def err(_stuff=None, _why=None, **kwargs):
import warnings
warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \
"use 'spider' argument instead", DeprecationWarning, stacklevel=2)
kwargs['system'] = domain or (spider.domain_name if spider else component)
kwargs['system'] = domain or (spider.name if spider else component)
if _why:
_why = unicode_to_str("ERROR: %s" % _why, log_encoding)
log.err(_stuff, _why, **kwargs)

45
scrapy/spider/models.py
View File

@ -3,6 +3,9 @@ Base class for Scrapy spiders
See documentation in docs/topics/spiders.rst
"""
import warnings
from zope.interface import Interface, Attribute, invariant, implements
from twisted.plugin import IPlugin
@ -11,17 +14,9 @@ from scrapy.http import Request
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.trackref import object_ref
def _valid_domain_name(obj):
"""Check the domain name specified is valid"""
if not obj.domain_name:
raise ValueError("Spider 'domain_name' attribute is required")
class ISpider(Interface, IPlugin) :
"""Interface to be implemented by site-specific web spiders"""
domain_name = Attribute("The domain name of the site to be scraped.")
invariant(_valid_domain_name)
"""Interface used by TwistedPluginSpiderManager to discover spiders"""
pass
class BaseSpider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this
@ -31,19 +26,33 @@ class BaseSpider(object_ref):
implements(ISpider)
# XXX: class attributes kept for backwards compatibility
domain_name = None
name = None
start_urls = []
extra_domain_names = []
allowed_domains = []
def __init__(self, domain_name=None):
if domain_name is not None:
self.domain_name = domain_name
def __init__(self, name=None):
# XXX: SEP-12 backward compatibility (remove for 0.10)
if hasattr(self, 'domain_name'):
warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead", \
DeprecationWarning, stacklevel=4)
self.name = self.domain_name
if hasattr(self, 'extra_domain_names'):
warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \
DeprecationWarning, stacklevel=4)
self.allowed_domains = [self.name] + list(self.extra_domain_names)
if name is not None:
self.name = name
# XXX: create instance attributes (class attributes were kept for
# backwards compatibility)
if not self.start_urls:
self.start_urls = []
if not self.extra_domain_names:
self.extra_domain_names = []
if not self.allowed_domains:
self.allowed_domains = []
if not getattr(self, 'domain_name', None):
self.domain_name = self.name
if not getattr(self, 'extra_domain_names', None):
self.extra_domain_names = self.allowed_domains
def log(self, message, level=log.DEBUG):
"""Log the given messages at the given log level. Always use this
@ -67,6 +76,6 @@ class BaseSpider(object_ref):
pass
def __str__(self):
return "<%s %r>" % (type(self).__name__, self.domain_name)
return "<%s %r>" % (type(self).__name__, self.name)
__repr__ = __str__

4
scrapy/stats/collector/__init__.py
View File

@ -76,11 +76,11 @@ class MemoryStatsCollector(StatsCollector):
def __init__(self):
super(MemoryStatsCollector, self).__init__()
self.domain_stats = {}
self.spider_stats = {}
def _persist_stats(self, stats, spider=None):
if spider is not None:
self.domain_stats[spider.domain_name] = stats
self.spider_stats[spider.name] = stats
class DummyStatsCollector(StatsCollector):

4
scrapy/stats/collector/simpledb.py
View File

@ -36,9 +36,9 @@ class SimpledbStatsCollector(StatsCollector):
def _persist_to_sdb(self, spider, stats):
ts = self._get_timestamp(spider).isoformat()
sdb_item_id = "%s_%s" % (spider.domain_name, ts)
sdb_item_id = "%s_%s" % (spider.name, ts)
sdb_item = dict((k, self._to_sdb_value(v, k)) for k, v in stats.iteritems())
sdb_item['domain'] = spider.domain_name
sdb_item['spider'] = spider.name
sdb_item['timestamp'] = self._to_sdb_value(ts)
connect_sdb().put_attributes(self._sdbdomain, sdb_item_id, sdb_item)

5
scrapy/templates/spiders/basic.tmpl
View File

@ -1,9 +1,10 @@
from scrapy.spider import BaseSpider
class $classname(BaseSpider):
domain_name = "$site"
name = "$name"
allowed_domains = ["$domain"]
start_urls = (
'http://www.$site/',
'http://www.$domain/',
)
def parse(self, response):

7
scrapy/templates/spiders/crawl.tmpl
View File

@ -6,8 +6,9 @@ from scrapy.contrib.spiders import CrawlSpider, Rule
from $project_name.items import ${ProjectName}Item
class $classname(CrawlSpider):
domain_name = '$site'
start_urls = ['http://www.$site/']
name = '$name'
allowed_domains = ['$domain']
start_urls = ['http://www.$domain/']
rules = (
Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
@ -16,7 +17,7 @@ class $classname(CrawlSpider):
def parse_item(self, response):
hxs = HtmlXPathSelector(response)
i = ${ProjectName}Item()
#i['site_id'] = hxs.select('//input[@id="sid"]/@value').extract()
#i['domain_id'] = hxs.select('//input[@id="sid"]/@value').extract()
#i['name'] = hxs.select('//div[@id="name"]').extract()
#i['description'] = hxs.select('//div[@id="description"]').extract()
return i

5
scrapy/templates/spiders/csvfeed.tmpl
View File

@ -2,8 +2,9 @@ from scrapy.contrib.spiders import CSVFeedSpider
from $project_name.items import ${ProjectName}Item
class $classname(CSVFeedSpider):
domain_name = '$site'
start_urls = ['http://www.$site/feed.csv']
name = '$name'
allowed_domains = ['$domain']
start_urls = ['http://www.$domain/feed.csv']
# headers = ['id', 'name', 'description', 'image_link']
# delimiter = '\t'

5
scrapy/templates/spiders/xmlfeed.tmpl
View File

@ -2,8 +2,9 @@ from scrapy.contrib.spiders import XMLFeedSpider
from $project_name.items import ${ProjectName}Item
class $classname(XMLFeedSpider):
domain_name = '$site'
start_urls = ['http://www.$site/feed.xml']
name = '$name'
allowed_domains = ['$domain']
start_urls = ['http://www.$domain/feed.xml']
def parse_item(self, response, selector):
i = ${ProjectName}Item()

14
scrapy/tests/test_commands.py
View File

@ -59,10 +59,18 @@ class CommandTest(ProjectTest):
class GenspiderCommandTest(CommandTest):
def test_arguments(self):
# only pass one argument. spider script shouldn't be created
self.assertEqual(0, self.call('genspider', 'test_name'))
assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
# pass two arguments <name> <domain>. spider script should be created
self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
def test_template_default(self, *args):
self.assertEqual(0, self.call('genspider', 'testspider', 'test.com', *args))
assert exists(join(self.proj_mod_path, 'spiders', 'testspider.py'))
self.assertEqual(1, self.call('genspider', 'otherspider', 'test.com'))
self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args))
assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com'))
def test_template_basic(self):
self.test_template_default('--template=basic')

8
scrapy/tests/test_engine.py
View File

@ -22,8 +22,8 @@ class TestItem(Item):
price = Field()
class TestSpider(BaseSpider):
domain_name = "scrapytest.org"
extra_domain_names = ["localhost"]
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
start_urls = ['http://localhost']
itemurl_re = re.compile("item\d+.html")
@ -68,7 +68,7 @@ def start_test_site():
class CrawlingSession(object):
def __init__(self):
self.domain = 'scrapytest.org'
self.name = 'scrapytest.org'
self.spider = None
self.respplug = []
self.reqplug = []
@ -139,7 +139,7 @@ class EngineTest(unittest.TestCase):
Check the spider is loaded and located properly via the SpiderLocator
"""
assert session.spider is not None
self.assertEqual(session.spider.domain_name, session.domain)
self.assertEqual(session.spider.name, session.name)
def test_visited_urls(self):
"""

56
scrapy/tests/test_spider.py Normal file
View File

@ -0,0 +1,56 @@
from __future__ import with_statement
import sys
import warnings
from twisted.trial import unittest
from scrapy.spider import BaseSpider
from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter
class OldSpider(BaseSpider):
domain_name = 'example.com'
extra_domain_names = ('example.org', 'example.net')
class NewSpider(BaseSpider):
name = 'example.com'
allowed_domains = ('example.org', 'example.net')
class SpiderTest(unittest.TestCase):
def setUp(self):
warnings.simplefilter("always")
def tearDown(self):
warnings.resetwarnings()
def test_sep12_deprecation_warnings(self):
if sys.version_info[:2] < (2, 6):
# warnings.catch_warnings() was added in Python 2.6
raise unittest.SkipTest("This test requires Python 2.6+")
with warnings.catch_warnings(record=True) as w:
spider = OldSpider()
self.assertEqual(len(w), 2) # one for domain_name & one for extra_domain_names
self.assert_(issubclass(w[-1].category, DeprecationWarning))
def test_sep12_backwards_compatibility(self):
spider = OldSpider()
self.assertEqual(spider.name, 'example.com')
self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains)
self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains)
self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains)
spider = NewSpider()
self.assertEqual(spider.domain_name, 'example.com')
self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names)
self.assert_('example.net' in spider.extra_domain_names, spider.extra_domain_names)
def test_base_spider(self):
spider = BaseSpider("example.com")
self.assertEqual(spider.name, 'example.com')
self.assertEqual(spider.start_urls, [])
self.assertEqual(spider.allowed_domains, [])

4
scrapy/tests/test_spidermiddleware_offsite.py
View File

@ -9,8 +9,8 @@ class TestOffsiteMiddleware(TestCase):
def setUp(self):
self.spider = BaseSpider()
self.spider.domain_name = 'scrapytest.org'
self.spider.extra_domain_names = ['scrapy.org']
self.spider.name = 'scrapytest.org'
self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org']
self.mw = OffsiteMiddleware()
self.mw.spider_opened(self.spider)

4
scrapy/utils/url.py
View File

@ -22,9 +22,7 @@ def url_is_from_any_domain(url, domains):
def url_is_from_spider(url, spider):
"""Return True if the url belongs to the given spider"""
domains = [spider.domain_name]
domains.extend(spider.extra_domain_names)
return url_is_from_any_domain(url, domains)
return url_is_from_any_domain(url, spider.allowed_domains)
def urljoin_rfc(base, ref, encoding='utf-8'):
"""Same as urlparse.urljoin but supports unicode values in base and ref