genspider: removing www. from starturl templates

This commit is contained in:
Raul Gallegos 2016-10-05 12:16:45 -05:00
parent 3235bfeb1e
commit dd778892d0
4 changed files with 4 additions and 4 deletions

View File

@ -6,7 +6,7 @@ class $classname(scrapy.Spider):
name = "$name"
allowed_domains = ["$domain"]
start_urls = (
'http://www.$domain/',
'http://$domain/',
)
def parse(self, response):

View File

@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule
class $classname(CrawlSpider):
name = '$name'
allowed_domains = ['$domain']
start_urls = ['http://www.$domain/']
start_urls = ['http://$domain/']
rules = (
Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),

View File

@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider
class $classname(CSVFeedSpider):
name = '$name'
allowed_domains = ['$domain']
start_urls = ['http://www.$domain/feed.csv']
start_urls = ['http://$domain/feed.csv']
# headers = ['id', 'name', 'description', 'image_link']
# delimiter = '\t'

View File

@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider
class $classname(XMLFeedSpider):
name = '$name'
allowed_domains = ['$domain']
start_urls = ['http://www.$domain/feed.xml']
start_urls = ['http://$domain/feed.xml']
iterator = 'iternodes' # you can change this; see the docs
itertag = 'item' # change it accordingly