remove decobot reference and fix test_plugin

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40262
This commit is contained in:
Daniel Grana 2008-09-19 16:57:57 +00:00
parent baabdb004d
commit 62975817af
1 changed files with 7 additions and 2 deletions

View File

@ -7,7 +7,8 @@ import re
from scrapy.spider import BaseSpider
from scrapy.item import ScrapedItem
from decobot.utils.link_extraction import follow_link_pattern
from scrapy.link import LinkExtractor
from scrapy.http import Request
class TestSpider(BaseSpider):
domain_name = "scrapytest.org"
@ -19,7 +20,11 @@ class TestSpider(BaseSpider):
price_re = re.compile(">Price: \$(.*?)<", re.M)
def parse(self, response):
return follow_link_pattern(response.body.to_string(), self.parse_item, response.url, self.itemurl_re)
xlink = LinkExtractor()
itemre = re.compile(self.itemurl_re)
for url in xlink.extract_urls(response):
if itemre.search(url):
yield Request(url=url, callback=self.parse_item)
def parse_item(self, response):
item = ScrapedItem()