From dd778892d09f4b504bfc65aae83dd70d87f0d93e Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Wed, 5 Oct 2016 12:16:45 -0500 Subject: [PATCH] genspider: removing www. from starturl templates --- scrapy/templates/spiders/basic.tmpl | 2 +- scrapy/templates/spiders/crawl.tmpl | 2 +- scrapy/templates/spiders/csvfeed.tmpl | 2 +- scrapy/templates/spiders/xmlfeed.tmpl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 2d3b96e38..abaa34ef5 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -6,7 +6,7 @@ class $classname(scrapy.Spider): name = "$name" allowed_domains = ["$domain"] start_urls = ( - 'http://www.$domain/', + 'http://$domain/', ) def parse(self, response): diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 154237d9c..802cb88a1 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule class $classname(CrawlSpider): name = '$name' allowed_domains = ['$domain'] - start_urls = ['http://www.$domain/'] + start_urls = ['http://$domain/'] rules = ( Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 0544e0ae7..c2e4bacfe 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider class $classname(CSVFeedSpider): name = '$name' allowed_domains = ['$domain'] - start_urls = ['http://www.$domain/feed.csv'] + start_urls = ['http://$domain/feed.csv'] # headers = ['id', 'name', 'description', 'image_link'] # delimiter = '\t' diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index d8ff61f6e..7c2ff8850 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider class $classname(XMLFeedSpider): name = '$name' allowed_domains = ['$domain'] - start_urls = ['http://www.$domain/feed.xml'] + start_urls = ['http://$domain/feed.xml'] iterator = 'iternodes' # you can change this; see the docs itertag = 'item' # change it accordingly