mirror of https://github.com/scrapy/scrapy.git
genspider: removing www. from starturl templates
This commit is contained in:
parent
3235bfeb1e
commit
dd778892d0
|
|
@ -6,7 +6,7 @@ class $classname(scrapy.Spider):
|
|||
name = "$name"
|
||||
allowed_domains = ["$domain"]
|
||||
start_urls = (
|
||||
'http://www.$domain/',
|
||||
'http://$domain/',
|
||||
)
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from scrapy.spiders import CrawlSpider, Rule
|
|||
class $classname(CrawlSpider):
|
||||
name = '$name'
|
||||
allowed_domains = ['$domain']
|
||||
start_urls = ['http://www.$domain/']
|
||||
start_urls = ['http://$domain/']
|
||||
|
||||
rules = (
|
||||
Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from scrapy.spiders import CSVFeedSpider
|
|||
class $classname(CSVFeedSpider):
|
||||
name = '$name'
|
||||
allowed_domains = ['$domain']
|
||||
start_urls = ['http://www.$domain/feed.csv']
|
||||
start_urls = ['http://$domain/feed.csv']
|
||||
# headers = ['id', 'name', 'description', 'image_link']
|
||||
# delimiter = '\t'
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from scrapy.spiders import XMLFeedSpider
|
|||
class $classname(XMLFeedSpider):
|
||||
name = '$name'
|
||||
allowed_domains = ['$domain']
|
||||
start_urls = ['http://www.$domain/feed.xml']
|
||||
start_urls = ['http://$domain/feed.xml']
|
||||
iterator = 'iternodes' # you can change this; see the docs
|
||||
itertag = 'item' # change it accordingly
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue