diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7eec036cf..5dd61100e 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -128,7 +128,8 @@ Finally, here's the spider code:: class MininovaSpider(CrawlSpider): - domain_name = 'mininova.org' + name = 'mininova.org' + allowed_domains = ['mininova.org'] start_urls = ['http://www.mininova.org/today'] rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 4358a7ae0..8fb6c7db5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -102,8 +102,8 @@ to parse the contents of those pages to extract :ref:`items `. To create a Spider, you must subclass :class:`scrapy.spider.BaseSpider`, and define the three main, mandatory, attributes: -* :attr:`~scrapy.spider.BaseSpider.domain_name`: identifies the Spider. It must - be unique, that is, you can't set the same domain name for different Spiders. +* :attr:`~scrapy.spider.BaseSpider.name`: identifies the Spider. It must be + unique, that is, you can't set the same name for different Spiders. * :attr:`~scrapy.spider.BaseSpider.start_urls`: is a list of URLs where the Spider will begin to crawl from. So, the first pages downloaded will be those @@ -128,7 +128,8 @@ This is the code for our first Spider, save it in a file named from scrapy.spider import BaseSpider class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -354,7 +355,8 @@ Let's add this code to our spider:: from scrapy.selector import HtmlXPathSelector class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" @@ -398,7 +400,8 @@ scraped so far, the code for our Spider should be like this:: from dmoz.items import DmozItem class DmozSpider(BaseSpider): - domain_name = "dmoz.org" + name = "dmoz.org" + allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15ead63be..d8629849f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -199,7 +199,7 @@ HttpAuthMiddleware http_user = 'someuser' http_pass = 'somepass' - domain_name = 'intranet.example.com' + name = 'intranet.example.com' # .. rest of the spider code omitted ... diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 711df19ce..a4312cb90 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -52,7 +52,7 @@ Exporter to export scraped items to different files, one per spider:: self.files = {} def spider_opened(self, spider): - file = open('%s_products.xml' % spider.domain_name, 'w+b') + file = open('%s_products.xml' % spider.name, 'w+b') self.files[spider] = file self.exporter = XmlItemExporter(file) self.exporter.start_exporting() diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 1bde87bb7..1ff1f068f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -105,10 +105,10 @@ everytime a domain/spider is opened and closed:: dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def spider_opened(self, spider): - log.msg("opened spider %s" % spider.domain_name) + log.msg("opened spider %s" % spider.name) def spider_closed(self, spider): - log.msg("closed spider %s" % spider.domain_name) + log.msg("closed spider %s" % spider.name) .. _topics-extensions-ref-manager: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 9f198a410..f42e6a77e 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -79,7 +79,8 @@ This is how the spider would look so far:: from scrapy.contrib.spiders import CrawlSpider, Rule class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allowed_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 63d289d09..48ec81ccf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -321,7 +321,7 @@ user name and password. You can use the :meth:`FormRequest.from_response` method for this job. Here's an example spider which uses it:: class LoginSpider(BaseSpider): - domain_name = 'example.com' + name = 'example.com' start_urls = ['http://www.example.com/users/login.php'] def parse(self, response): diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 6e3de9bc1..45d9cf0c3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -163,7 +163,7 @@ This can be achieved by using the ``scrapy.shell.inspect_response`` function. Here's an example of how you would call it from your spider:: class MySpider(BaseSpider): - domain_name = 'example.com' + ... def parse(self, response): if response.url == 'http://www.example.com/products.php': diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c0e57ed06..0c5dcc675 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -210,11 +210,8 @@ OffsiteMiddleware Filters out Requests for URLs outside the domains covered by the spider. - This middleware filters out every request whose host names don't match - :attr:`~scrapy.spider.BaseSpider.domain_name`, or the spider - :attr:`~scrapy.spider.BaseSpider.domain_name` prefixed by "www.". - Spider can add more domains to exclude using - :attr:`~scrapy.spider.BaseSpider.extra_domain_names` attribute. + This middleware filters out every request whose host names aren't in the + spider's :attr:`~scrapy.spider.BaseSpider.allowed_domains` attribute. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e60549098..b69f9a445 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -70,20 +70,22 @@ BaseSpider requests the given ``start_urls``/``start_requests``, and calls the spider's method ``parse`` for each of the resulting responses. - .. attribute:: domain_name + .. attribute:: name - A string which defines the domain name for this spider, which will also be - the unique identifier for this spider (which means you can't have two - spider with the same ``domain_name``). This is the most important spider - attribute and it's required, and it's the name by which Scrapy will known - the spider. + A string which defines the name for this spider. The spider name is how + the spider is located (and instantiated) by Scrapy, so it must be + unique. However, nothing prevents you from instantiating more than one + instance of the same spider. This is the most important spider attribute + and it's required. - .. attribute:: extra_domain_names + Is recommended to name your spiders after the domain that their crawl. - An optional list of strings containing additional domains that this - spider is allowed to crawl. Requests for URLs not belonging to the - domain name specified in :attr:`domain_name` or this list won't be - followed. + .. attribute:: allowed_domains + + An optional list of strings containing domains that this spider is + allowed to crawl. Requests for URLs not belonging to the domain names + specified in this list won't be followed if + :class:`~scrapy.contrib.spidermiddleware.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls @@ -144,7 +146,7 @@ BaseSpider .. method:: log(message, [level, component]) Log a message using the :func:`scrapy.log.msg` function, automatically - populating the domain argument with the :attr:`domain_name` of this + populating the spider argument with the :attr:`name` of this spider. For more information see :ref:`topics-logging`. @@ -157,7 +159,8 @@ Let's see an example:: from scrapy.spider import BaseSpider class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -177,7 +180,8 @@ Another example returning multiples Requests and Items from a single callback:: from myproject.items import MyItem class MySpider(BaseSpider): - domain_name = 'http://www.example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = [ 'http://www.example.com/1.html', 'http://www.example.com/2.html', @@ -254,7 +258,8 @@ Let's now take a look at an example CrawlSpider with rules:: from scrapy.item import Item class MySpider(CrawlSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com'] rules = ( @@ -378,7 +383,8 @@ These spiders are pretty easy to use, let's have at one example:: from myproject.items import TestItem class MySpider(XMLFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.xml'] iterator = 'iternodes' # This is actually unnecesary, since it's the default value itertag = 'item' @@ -435,7 +441,8 @@ Let's see an example similar to the previous one, but using a from myproject.items import TestItem class MySpider(CSVFeedSpider): - domain_name = 'example.com' + name = 'example.com' + allowed_domains = ['example.com'] start_urls = ['http://www.example.com/feed.csv'] delimiter = ';' headers = ['id', 'name', 'description'] diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 971a33364..34a3c8b7c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -204,15 +204,15 @@ MemoryStatsCollector A simple stats collector that keeps the stats of the last scraping run (for each spider) in memory, after they're closed. The stats can be accessed - through the :attr:`domain_stats` attribute, which is a dict keyed by spider + through the :attr:`spider_stats` attribute, which is a dict keyed by spider domain name. This is the default Stats Collector used in Scrapy. - .. attribute:: domain_stats + .. attribute:: spider_stats - A dict of dicts (keyed by spider domain name) containing the stats of - the last scraping run for each domain. + A dict of dicts (keyed by spider name) containing the stats of the last + scraping run for each spider. DummyStatsCollector ------------------- @@ -240,11 +240,11 @@ SimpledbStatsCollector In addition to the existing stats keys the following keys are added at persitance time: - * ``domain``: the spider domain (so you can use it later for querying stats - for that domain) + * ``spider``: the spider name (so you can use it later for querying stats + for that spider) * ``timestamp``: the timestamp when the stats were persisited - Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB + Both the ``spider`` and ``timestamp`` are used for generating the SimpleDB item name in order to avoid overwriting stats of previous scraping runs. As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and diff --git a/examples/experimental/googledir/googledir/spiders/google_directory.py b/examples/experimental/googledir/googledir/spiders/google_directory.py index fdbf3f24a..2ed7c52a6 100644 --- a/examples/experimental/googledir/googledir/spiders/google_directory.py +++ b/examples/experimental/googledir/googledir/spiders/google_directory.py @@ -6,7 +6,8 @@ from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'google_directory' + allowed_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/examples/experimental/imdb/imdb/spiders/imdb_site.py b/examples/experimental/imdb/imdb/spiders/imdb_site.py index 5c3f3c06b..8c2ebcd01 100644 --- a/examples/experimental/imdb/imdb/spiders/imdb_site.py +++ b/examples/experimental/imdb/imdb/spiders/imdb_site.py @@ -29,7 +29,8 @@ class MovieItem(ImdbItem): class ImdbSiteSpider(CrawlSpider): - domain_name = 'imdb.com' + name = 'imdb.com' + allowed_domains = ['imdb.com'] start_urls = ['http://www.imdb.com/'] # extract requests using this classes from urls matching 'follow' flag diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index 054cef022..2af52bf1f 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -6,7 +6,8 @@ from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): - domain_name = 'directory.google.com' + name = 'directory.google.com' + allow_domains = ['directory.google.com'] start_urls = ['http://directory.google.com/'] rules = ( diff --git a/scrapy/command/commands/crawl.py b/scrapy/command/commands/crawl.py index 00f375df2..9c02b08e3 100644 --- a/scrapy/command/commands/crawl.py +++ b/scrapy/command/commands/crawl.py @@ -13,10 +13,10 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] ..." + return "[options] ..." def short_desc(self): - return "Start crawling a domain or URL" + return "Start crawling from a spider or URL" def add_options(self, parser): ScrapyCommand.add_options(self, parser) @@ -31,10 +31,9 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - - urls, domains = self._split_urls_and_domains(args) - for dom in domains: - scrapymanager.crawl_domain(dom) + urls, names = self._split_urls_and_names(args) + for name in names: + scrapymanager.crawl_spider_name(name) if opts.spider: try: @@ -65,12 +64,12 @@ class Command(ScrapyCommand): spider_urls[spider_names[0]].append(url) return spider_urls.items() - def _split_urls_and_domains(self, args): + def _split_urls_and_names(self, args): urls = [] - domains = [] + names = [] for arg in args: if is_url(arg): urls.append(arg) else: - domains.append(arg) - return urls, domains + names.append(arg) + return urls, names diff --git a/scrapy/command/commands/genspider.py b/scrapy/command/commands/genspider.py index b2e0a5fb6..2289fee1a 100644 --- a/scrapy/command/commands/genspider.py +++ b/scrapy/command/commands/genspider.py @@ -15,10 +15,11 @@ SPIDER_TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'spiders') def sanitize_module_name(module_name): - """Sanitize the given module name, by replacing dashes with underscores and - prefixing it with a letter if it doesn't start with one + """Sanitize the given module name, by replacing dashes and points + with underscores and prefixing it with a letter if it doesn't start + with one """ - module_name = module_name.replace('-', '_') + module_name = module_name.replace('-', '_').replace('.', '_') if module_name[0] not in string.ascii_letters: module_name = "a" + module_name return module_name @@ -28,7 +29,7 @@ class Command(ScrapyCommand): requires_project = True def syntax(self): - return "[options] " + return "[options] " def short_desc(self): return "Generate new spider based on template passed with -t or --template" @@ -54,34 +55,37 @@ class Command(ScrapyCommand): print template.read() return - if len(args) < 2: + if len(args) != 2: return False - module = sanitize_module_name(args[0]) + name = args[0] domain = args[1] + module = sanitize_module_name(name) + # if spider already exists and not force option then halt try: - spider = spiders.create(domain) + spider = spiders.create(name) except KeyError: pass else: if not opts.force: - print "Spider '%s' already exists in module:" % domain + print "Spider '%s' already exists in module:" % name print " %s" % spider.__module__ sys.exit(1) template_file = self._find_template(opts.template) if template_file: - self._genspider(module, domain, opts.template, template_file) + self._genspider(module, name, domain, opts.template, template_file) - def _genspider(self, module, domain, template_name, template_file): + def _genspider(self, module, name, domain, template_name, template_file): """Generate the spider module, based on the given template""" tvars = { 'project_name': settings.get('BOT_NAME'), 'ProjectName': string_camelcase(settings.get('BOT_NAME')), 'module': module, - 'site': domain, + 'name': name, + 'domain': domain, 'classname': '%sSpider' % ''.join([s.capitalize() \ for s in module.split('_')]) } @@ -92,7 +96,7 @@ class Command(ScrapyCommand): shutil.copyfile(template_file, spider_file) render_templatefile(spider_file, **tvars) - print "Created spider %r using template %r in module:" % (domain, \ + print "Created spider %r using template %r in module:" % (name, \ template_name) print " %s.%s" % (spiders_module.__name__, module) diff --git a/scrapy/command/commands/parse.py b/scrapy/command/commands/parse.py index d67a26d79..9aabe94ff 100644 --- a/scrapy/command/commands/parse.py +++ b/scrapy/command/commands/parse.py @@ -46,7 +46,7 @@ class Command(ScrapyCommand): if callback: callback_fcn = callback if callable(callback) else getattr(spider, callback, None) if not callback_fcn: - log.msg('Cannot find callback %s in %s spider' % (callback, spider.domain_name)) + log.msg('Cannot find callback %s in %s spider' % (callback, spider.name)) return (), () result = iterate_spider_output(callback_fcn(response)) @@ -130,7 +130,7 @@ class Command(ScrapyCommand): else: log.msg('No rules found for spider "%s", ' \ 'please specify a callback for parsing' \ - % spider.domain_name, log.ERROR) + % spider.name, log.ERROR) else: # default callback 'parse' items, links = self.run_callback(spider, response, diff --git a/scrapy/contrib/aws.py b/scrapy/contrib/aws.py index 1f54b7634..91b134a0e 100644 --- a/scrapy/contrib/aws.py +++ b/scrapy/contrib/aws.py @@ -20,7 +20,7 @@ class AWSMiddleware(object): def process_request(self, request, spider): hostname = urlparse_cached(request).hostname - if spider.domain_name == 's3.amazonaws.com' \ + if spider.name == 's3.amazonaws.com' \ or (hostname and hostname.endswith('s3.amazonaws.com')): request.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", \ time.gmtime()) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 239c05bc7..341e9b5a1 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -108,7 +108,7 @@ class FilesystemCacheStorage(object): def _get_request_path(self, spider, request): key = request_fingerprint(request) - return join(self.cachedir, spider.domain_name, key[0:2], key) + return join(self.cachedir, spider.name, key[0:2], key) def _read_meta(self, spider, request): rpath = self._get_request_path(spider, request) diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index 6021bbf5a..9f8666467 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -1,6 +1,6 @@ """ This module provides a mechanism for collecting one (or more) sample items per -domain. +spider. The items are collected in a dict of guid->item and persisted by pickling that dict into a file. @@ -8,7 +8,7 @@ dict into a file. This can be useful for testing changes made to the framework or other common code that affects several spiders. -It uses the scrapy stats service to keep track of which domains are already +It uses the scrapy stats service to keep track of which spiders are already sampled. Settings that affect this module: @@ -48,7 +48,7 @@ class ItemSamplerPipeline(object): raise NotConfigured self.items = {} self.spiders_count = 0 - self.empty_domains = set() + self.empty_spiders = set() dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) @@ -66,21 +66,21 @@ class ItemSamplerPipeline(object): def engine_stopped(self): with open(self.filename, 'w') as f: pickle.dump(self.items, f) - if self.empty_domains: - log.msg("No products sampled for: %s" % " ".join(self.empty_domains), \ + if self.empty_spiders: + log.msg("No products sampled for: %s" % " ".join(self.empty_spiders), \ level=log.WARNING) def spider_closed(self, spider, reason): if reason == 'finished' and not stats.get_value("items_sampled", spider=spider): - self.empty_domains.add(spider.domain_name) + self.empty_spiders.add(spider.name) self.spiders_count += 1 - log.msg("Sampled %d domains so far (%d empty)" % (self.spiders_count, \ - len(self.empty_domains)), level=log.INFO) + log.msg("Sampled %d spiders so far (%d empty)" % (self.spiders_count, \ + len(self.empty_spiders)), level=log.INFO) class ItemSamplerMiddleware(object): - """This middleware drops items and requests (when domain sampling has been - completed) to accelerate the processing of remaining domains""" + """This middleware drops items and requests (when spider sampling has been + completed) to accelerate the processing of remaining spiders""" def __init__(self): if not settings['ITEMSAMPLER_FILE']: diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 3a93553dd..a56b3da0b 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -47,7 +47,7 @@ class FSImagesStore(object): dispatcher.connect(self.spider_closed, signals.spider_closed) def spider_closed(self, spider): - self.created_directories.pop(spider.domain_name, None) + self.created_directories.pop(spider.name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) @@ -92,7 +92,7 @@ class _S3AmazonAWSSpider(BaseSpider): It means that a spider that uses download_delay or alike is not going to be delayed even more because it is uploading images to s3. """ - domain_name = "s3.amazonaws.com" + name = "s3.amazonaws.com" start_urls = ['http://s3.amazonaws.com/'] max_concurrent_requests = 100 diff --git a/scrapy/contrib/spidermanager.py b/scrapy/contrib/spidermanager.py index e37e861a5..331f55aa5 100644 --- a/scrapy/contrib/spidermanager.py +++ b/scrapy/contrib/spidermanager.py @@ -53,7 +53,7 @@ class TwistedPluginSpiderManager(object): for module in modules: for spider in self._getspiders(ISpider, module): ISpider.validateInvariants(spider) - self._spiders[spider.domain_name] = spider + self._spiders[spider.name] = spider self.loaded = True def _getspiders(self, interface, package): @@ -76,14 +76,14 @@ class TwistedPluginSpiderManager(object): """Reload spider module to release any resources held on to by the spider """ - domain = spider.domain_name - if domain not in self._spiders: + name = spider.name + if name not in self._spiders: return - spider = self._spiders[domain] + spider = self._spiders[name] module_name = spider.__module__ module = sys.modules[module_name] if hasattr(module, 'SPIDER'): log.msg("Reloading module %s" % module_name, spider=spider, \ level=log.DEBUG) new_module = rebuild(module, doLog=0) - self._spiders[domain] = new_module.SPIDER + self._spiders[name] = new_module.SPIDER diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 5e40d9915..f28b0d53d 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -47,8 +47,7 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - domains = [spider.domain_name] + spider.extra_domain_names - self.host_regexes[spider] = self.get_host_regex(domains) + self.host_regexes[spider] = self.get_host_regex(spider.allowed_domains) self.domains_seen[spider] = set() def spider_closed(self, spider): diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index c2185e0a7..fc76ccfbf 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -23,6 +23,6 @@ class StatsMailer(object): mail = MailSender() body = "Global stats\n\n" body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) - body += "\n\n%s stats\n\n" % spider.domain_name + body += "\n\n%s stats\n\n" % spider.name body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) - mail.send(self.recipients, "Scrapy stats for: %s" % spider.domain_name, body) + mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 22afc20c1..2e61e8bc3 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -60,7 +60,7 @@ class LiveStats(object): runtime = datetime.now() - stats.started s += '%s%d%d%d%d%d%d%s%s\n' % \ - (spider.domain_name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) + (spider.name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(runtime)) totdomains += 1 totscraped += stats.scraped diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 933801854..ce3fdffd6 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -25,18 +25,18 @@ class Spiderctl(object): dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) def spider_opened(self, spider): - self.running[spider.domain_name] = spider + self.running[spider.name] = spider def spider_closed(self, spider): - del self.running[spider.domain_name] - self.finished.add(spider.domain_name) + del self.running[spider.name] + self.finished.add(spider.name) def webconsole_render(self, wc_request): if wc_request.args: changes = self.webconsole_control(wc_request) - self.scheduled = [s.domain_name for s in scrapyengine.spider_scheduler._pending_spiders] - self.idle = [d for d in self.enabled_domains if d not in self.scheduled + self.scheduled = [s.name for s in scrapyengine.spider_scheduler._pending_spiders] + self.idle = [d for d in self.enabled_spiders if d not in self.scheduled and d not in self.running and d not in self.finished] @@ -53,9 +53,9 @@ class Spiderctl(object): # idle s += "\n" s += '
\n' - s += '\n' + for name in sorted(self.idle): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -65,9 +65,9 @@ class Spiderctl(object): # scheduled s += "\n" s += '\n' - s += '\n' + for name in self.scheduled: + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -78,9 +78,9 @@ class Spiderctl(object): # running s += "\n" s += '\n' - s += '\n' + for name in sorted(self.running): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -90,9 +90,9 @@ class Spiderctl(object): # finished s += "\n" s += '\n' - s += '\n' + for name in sorted(self.finished): + s += "\n" % name s += '
\n' s += '
' s += '\n' @@ -114,42 +114,42 @@ class Spiderctl(object): args = wc_request.args s = "
\n" - if "stop_running_domains" in args: + if "stop_running_spiders" in args: s += "

" - stopped_domains = [] - for domain in args["stop_running_domains"]: - if domain in self.running: - scrapyengine.close_spider(self.running[domain]) - stopped_domains.append(domain) - s += "Stopped spiders:

  • %s
" % "
  • ".join(stopped_domains) + stopped_spiders = [] + for name in args["stop_running_spiders"]: + if name in self.running: + scrapyengine.close_spider(self.running[name]) + stopped_spiders.append(name) + s += "Stopped spiders:
    • %s
    " % "
  • ".join(stopped_spiders) s += "

    " - if "remove_pending_domains" in args: + if "remove_pending_spiders" in args: removed = [] - for domain in args["remove_pending_domains"]: - if scrapyengine.spider_scheduler.remove_pending_domain(domain): - removed.append(domain) + for name in args["remove_pending_spiders"]: + if scrapyengine.spider_scheduler.remove_pending_spider(name): + removed.append(name) if removed: s += "

    " - s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_domains"]) + s += "Removed scheduled spiders:
    • %s
    " % "
  • ".join(args["remove_pending_spiders"]) s += "

    " - if "add_pending_domains" in args: - for domain in args["add_pending_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl_domain(domain) + if "add_pending_spiders" in args: + for name in args["add_pending_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) s += "

    " - s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_domains"]) + s += "Scheduled spiders:
    • %s
    " % "
  • ".join(args["add_pending_spiders"]) s += "

    " - if "rerun_finished_domains" in args: - for domain in args["rerun_finished_domains"]: - if domain not in scrapyengine.scheduler.pending_requests: - scrapymanager.crawl_domain(domain) - self.finished.remove(domain) + if "rerun_finished_spiders" in args: + for name in args["rerun_finished_spiders"]: + if name not in scrapyengine.scheduler.pending_requests: + scrapymanager.crawl_spider_name(name) + self.finished.remove(name) s += "

    " - s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) + s += "Re-scheduled finished spiders:
    • %s
    " % "
  • ".join(args["rerun_finished_spiders"]) s += "

    " return s def webconsole_discover_module(self): - self.enabled_domains = spiders.list() + self.enabled_spiders = spiders.list() return self diff --git a/scrapy/contrib/webconsole/stats.py b/scrapy/contrib/webconsole/stats.py index 837cbcf81..e8c707f64 100644 --- a/scrapy/contrib/webconsole/stats.py +++ b/scrapy/contrib/webconsole/stats.py @@ -23,7 +23,7 @@ class StatsDump(object): s += "

    Global stats

    \n" s += stats_html_table(stats.get_stats()) for spider, spider_stats in stats.iter_spider_stats(): - s += "

    %s

    \n" % spider.domain_name + s += "

    %s

    \n" % spider.name s += stats_html_table(spider_stats) s += "\n" s += "\n" diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index dc6bba782..b1dc3754f 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -54,12 +54,12 @@ class ExecutionManager(object): if spider: scrapyengine.crawl(request, spider) - def crawl_domain(self, domain): - """Schedule given domain for crawling.""" + def crawl_spider_name(self, name): + """Schedule given spider by name for crawling.""" try: - spider = spiders.create(domain) + spider = spiders.create(name) except KeyError: - log.msg('Could not find spider for domain: %s' % domain, log.ERROR) + log.msg('Could not find spider: %s' % name, log.ERROR) else: self.crawl_spider(spider) diff --git a/scrapy/log.py b/scrapy/log.py index c4f9287e5..c8e969928 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -75,7 +75,7 @@ def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): "use 'spider' argument instead", DeprecationWarning, stacklevel=2) dispatcher.send(signal=logmessage_received, message=message, level=level, \ spider=spider) - system = domain or (spider.domain_name if spider else component) + system = domain or (spider.name if spider else component) msg_txt = unicode_to_str("%s: %s" % (level_names[level], message), log_encoding) log.msg(msg_txt, system=system) @@ -93,7 +93,7 @@ def err(_stuff=None, _why=None, **kwargs): import warnings warnings.warn("'domain' argument of scrapy.log.err() is deprecated, " \ "use 'spider' argument instead", DeprecationWarning, stacklevel=2) - kwargs['system'] = domain or (spider.domain_name if spider else component) + kwargs['system'] = domain or (spider.name if spider else component) if _why: _why = unicode_to_str("ERROR: %s" % _why, log_encoding) log.err(_stuff, _why, **kwargs) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index a9b64995e..757aaff55 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -3,6 +3,9 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ + +import warnings + from zope.interface import Interface, Attribute, invariant, implements from twisted.plugin import IPlugin @@ -11,17 +14,9 @@ from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy.utils.trackref import object_ref -def _valid_domain_name(obj): - """Check the domain name specified is valid""" - if not obj.domain_name: - raise ValueError("Spider 'domain_name' attribute is required") - class ISpider(Interface, IPlugin) : - """Interface to be implemented by site-specific web spiders""" - - domain_name = Attribute("The domain name of the site to be scraped.") - - invariant(_valid_domain_name) + """Interface used by TwistedPluginSpiderManager to discover spiders""" + pass class BaseSpider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this @@ -31,19 +26,33 @@ class BaseSpider(object_ref): implements(ISpider) # XXX: class attributes kept for backwards compatibility - domain_name = None + name = None start_urls = [] - extra_domain_names = [] + allowed_domains = [] - def __init__(self, domain_name=None): - if domain_name is not None: - self.domain_name = domain_name + def __init__(self, name=None): + # XXX: SEP-12 backward compatibility (remove for 0.10) + if hasattr(self, 'domain_name'): + warnings.warn("Spider.domain_name attribute is deprecated, use Spider.name instead", \ + DeprecationWarning, stacklevel=4) + self.name = self.domain_name + if hasattr(self, 'extra_domain_names'): + warnings.warn("Spider.extra_domain_names attribute is deprecated - user Spider.allowed_domains instead", \ + DeprecationWarning, stacklevel=4) + self.allowed_domains = [self.name] + list(self.extra_domain_names) + + if name is not None: + self.name = name # XXX: create instance attributes (class attributes were kept for # backwards compatibility) if not self.start_urls: self.start_urls = [] - if not self.extra_domain_names: - self.extra_domain_names = [] + if not self.allowed_domains: + self.allowed_domains = [] + if not getattr(self, 'domain_name', None): + self.domain_name = self.name + if not getattr(self, 'extra_domain_names', None): + self.extra_domain_names = self.allowed_domains def log(self, message, level=log.DEBUG): """Log the given messages at the given log level. Always use this @@ -67,6 +76,6 @@ class BaseSpider(object_ref): pass def __str__(self): - return "<%s %r>" % (type(self).__name__, self.domain_name) + return "<%s %r>" % (type(self).__name__, self.name) __repr__ = __str__ diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index f41916818..0acf36073 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -76,11 +76,11 @@ class MemoryStatsCollector(StatsCollector): def __init__(self): super(MemoryStatsCollector, self).__init__() - self.domain_stats = {} + self.spider_stats = {} def _persist_stats(self, stats, spider=None): if spider is not None: - self.domain_stats[spider.domain_name] = stats + self.spider_stats[spider.name] = stats class DummyStatsCollector(StatsCollector): diff --git a/scrapy/stats/collector/simpledb.py b/scrapy/stats/collector/simpledb.py index 850558442..d521c5e40 100644 --- a/scrapy/stats/collector/simpledb.py +++ b/scrapy/stats/collector/simpledb.py @@ -36,9 +36,9 @@ class SimpledbStatsCollector(StatsCollector): def _persist_to_sdb(self, spider, stats): ts = self._get_timestamp(spider).isoformat() - sdb_item_id = "%s_%s" % (spider.domain_name, ts) + sdb_item_id = "%s_%s" % (spider.name, ts) sdb_item = dict((k, self._to_sdb_value(v, k)) for k, v in stats.iteritems()) - sdb_item['domain'] = spider.domain_name + sdb_item['spider'] = spider.name sdb_item['timestamp'] = self._to_sdb_value(ts) connect_sdb().put_attributes(self._sdbdomain, sdb_item_id, sdb_item) diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 2e3baf992..246015466 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,9 +1,10 @@ from scrapy.spider import BaseSpider class $classname(BaseSpider): - domain_name = "$site" + name = "$name" + allowed_domains = ["$domain"] start_urls = ( - 'http://www.$site/', + 'http://www.$domain/', ) def parse(self, response): diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 18fd0ec42..578779c06 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -6,8 +6,9 @@ from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item class $classname(CrawlSpider): - domain_name = '$site' - start_urls = ['http://www.$site/'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/'] rules = ( Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), @@ -16,7 +17,7 @@ class $classname(CrawlSpider): def parse_item(self, response): hxs = HtmlXPathSelector(response) i = ${ProjectName}Item() - #i['site_id'] = hxs.select('//input[@id="sid"]/@value').extract() + #i['domain_id'] = hxs.select('//input[@id="sid"]/@value').extract() #i['name'] = hxs.select('//div[@id="name"]').extract() #i['description'] = hxs.select('//div[@id="description"]').extract() return i diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 794288570..c9a723000 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item class $classname(CSVFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.csv'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.csv'] # headers = ['id', 'name', 'description', 'image_link'] # delimiter = '\t' diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index f249b4537..f5ecbd707 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -2,8 +2,9 @@ from scrapy.contrib.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item class $classname(XMLFeedSpider): - domain_name = '$site' - start_urls = ['http://www.$site/feed.xml'] + name = '$name' + allowed_domains = ['$domain'] + start_urls = ['http://www.$domain/feed.xml'] def parse_item(self, response, selector): i = ${ProjectName}Item() diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index cb4e5ced1..6eed6d22a 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -59,10 +59,18 @@ class CommandTest(ProjectTest): class GenspiderCommandTest(CommandTest): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + self.assertEqual(0, self.call('genspider', 'test_name')) + assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + # pass two arguments . spider script should be created + self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) + assert exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) + def test_template_default(self, *args): - self.assertEqual(0, self.call('genspider', 'testspider', 'test.com', *args)) - assert exists(join(self.proj_mod_path, 'spiders', 'testspider.py')) - self.assertEqual(1, self.call('genspider', 'otherspider', 'test.com')) + self.assertEqual(0, self.call('genspider', 'test_spider', 'test.com', *args)) + assert exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')) + self.assertEqual(1, self.call('genspider', 'test_spider', 'test.com')) def test_template_basic(self): self.test_template_default('--template=basic') diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 873e0eb30..2d7b5950c 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -22,8 +22,8 @@ class TestItem(Item): price = Field() class TestSpider(BaseSpider): - domain_name = "scrapytest.org" - extra_domain_names = ["localhost"] + name = "scrapytest.org" + allowed_domains = ["scrapytest.org", "localhost"] start_urls = ['http://localhost'] itemurl_re = re.compile("item\d+.html") @@ -68,7 +68,7 @@ def start_test_site(): class CrawlingSession(object): def __init__(self): - self.domain = 'scrapytest.org' + self.name = 'scrapytest.org' self.spider = None self.respplug = [] self.reqplug = [] @@ -139,7 +139,7 @@ class EngineTest(unittest.TestCase): Check the spider is loaded and located properly via the SpiderLocator """ assert session.spider is not None - self.assertEqual(session.spider.domain_name, session.domain) + self.assertEqual(session.spider.name, session.name) def test_visited_urls(self): """ diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py new file mode 100644 index 000000000..507de4abb --- /dev/null +++ b/scrapy/tests/test_spider.py @@ -0,0 +1,56 @@ +from __future__ import with_statement + +import sys +import warnings + +from twisted.trial import unittest + +from scrapy.spider import BaseSpider +from scrapy.contrib.dupefilter import RequestFingerprintDupeFilter, NullDupeFilter + +class OldSpider(BaseSpider): + + domain_name = 'example.com' + extra_domain_names = ('example.org', 'example.net') + +class NewSpider(BaseSpider): + + name = 'example.com' + allowed_domains = ('example.org', 'example.net') + + +class SpiderTest(unittest.TestCase): + + def setUp(self): + warnings.simplefilter("always") + + def tearDown(self): + warnings.resetwarnings() + + def test_sep12_deprecation_warnings(self): + if sys.version_info[:2] < (2, 6): + # warnings.catch_warnings() was added in Python 2.6 + raise unittest.SkipTest("This test requires Python 2.6+") + with warnings.catch_warnings(record=True) as w: + spider = OldSpider() + self.assertEqual(len(w), 2) # one for domain_name & one for extra_domain_names + self.assert_(issubclass(w[-1].category, DeprecationWarning)) + + def test_sep12_backwards_compatibility(self): + spider = OldSpider() + self.assertEqual(spider.name, 'example.com') + self.assert_('example.com' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.org' in spider.allowed_domains, spider.allowed_domains) + self.assert_('example.net' in spider.allowed_domains, spider.allowed_domains) + + spider = NewSpider() + self.assertEqual(spider.domain_name, 'example.com') + self.assert_('example.org' in spider.extra_domain_names, spider.extra_domain_names) + self.assert_('example.net' in spider.extra_domain_names, spider.extra_domain_names) + + def test_base_spider(self): + spider = BaseSpider("example.com") + self.assertEqual(spider.name, 'example.com') + self.assertEqual(spider.start_urls, []) + self.assertEqual(spider.allowed_domains, []) + diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 6c71213aa..cc40c8b20 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -9,8 +9,8 @@ class TestOffsiteMiddleware(TestCase): def setUp(self): self.spider = BaseSpider() - self.spider.domain_name = 'scrapytest.org' - self.spider.extra_domain_names = ['scrapy.org'] + self.spider.name = 'scrapytest.org' + self.spider.allowed_domains = ['scrapytest.org', 'scrapy.org'] self.mw = OffsiteMiddleware() self.mw.spider_opened(self.spider) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 7496328fc..7ce27636d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -22,9 +22,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - domains = [spider.domain_name] - domains.extend(spider.extra_domain_names) - return url_is_from_any_domain(url, domains) + return url_is_from_any_domain(url, spider.allowed_domains) def urljoin_rfc(base, ref, encoding='utf-8'): """Same as urlparse.urljoin but supports unicode values in base and ref