mirror of https://github.com/scrapy/scrapy.git
96 lines
3.3 KiB
Python
96 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from scrapy.http import Response, TextResponse, XmlResponse
|
|
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
|
|
from tests import get_testdata
|
|
from tests.utils.bases.spider import TestSpiderBase
|
|
|
|
|
|
class TestSpider(TestSpiderBase):
|
|
spider_class = Spider
|
|
|
|
|
|
class TestXMLFeedSpider(TestSpiderBase):
|
|
spider_class = XMLFeedSpider
|
|
|
|
def test_register_namespace(self):
|
|
body = b"""<?xml version="1.0" encoding="UTF-8"?>
|
|
<urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"
|
|
xmlns:y="http://www.example.com/schemas/extras/1.0">
|
|
<url><x:loc>http://www.example.com/Special-Offers.html</x:loc><y:updated>2009-08-16</y:updated>
|
|
<other value="bar" y:custom="fuu"/>
|
|
</url>
|
|
<url><loc>http://www.example.com/</loc><y:updated>2009-08-16</y:updated><other value="foo"/></url>
|
|
</urlset>"""
|
|
response = XmlResponse(url="http://example.com/sitemap.xml", body=body)
|
|
|
|
class _XMLSpider(self.spider_class):
|
|
itertag = "url"
|
|
namespaces = (
|
|
("a", "http://www.google.com/schemas/sitemap/0.84"),
|
|
("b", "http://www.example.com/schemas/extras/1.0"),
|
|
)
|
|
|
|
def parse_node(self, response, selector):
|
|
yield {
|
|
"loc": selector.xpath("a:loc/text()").getall(),
|
|
"updated": selector.xpath("b:updated/text()").getall(),
|
|
"other": selector.xpath("other/@value").getall(),
|
|
"custom": selector.xpath("other/@b:custom").getall(),
|
|
}
|
|
|
|
for iterator in ("iternodes", "xml"):
|
|
spider = _XMLSpider("example", iterator=iterator)
|
|
output = list(spider._parse(response))
|
|
assert len(output) == 2, iterator
|
|
assert output == [
|
|
{
|
|
"loc": ["http://www.example.com/Special-Offers.html"],
|
|
"updated": ["2009-08-16"],
|
|
"custom": ["fuu"],
|
|
"other": ["bar"],
|
|
},
|
|
{
|
|
"loc": [],
|
|
"updated": ["2009-08-16"],
|
|
"other": ["foo"],
|
|
"custom": [],
|
|
},
|
|
], iterator
|
|
|
|
|
|
class TestCSVFeedSpider(TestSpiderBase):
|
|
spider_class = CSVFeedSpider
|
|
|
|
def test_parse_rows(self):
|
|
body = get_testdata("feeds", "feed-sample6.csv")
|
|
response = Response("http://example.org/dummy.csv", body=body)
|
|
|
|
class _CrawlSpider(self.spider_class):
|
|
name = "test"
|
|
delimiter = ","
|
|
quotechar = "'"
|
|
|
|
def parse_row(self, response, row):
|
|
return row
|
|
|
|
spider = _CrawlSpider()
|
|
rows = list(spider.parse_rows(response))
|
|
assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"}
|
|
assert len(rows) == 4
|
|
|
|
|
|
class TestNoParseMethodSpider:
|
|
spider_class = Spider
|
|
|
|
def test_undefined_parse_method(self):
|
|
spider = self.spider_class("example.com")
|
|
text = b"Random text"
|
|
resp = TextResponse(url="http://www.example.com/random_url", body=text)
|
|
|
|
exc_msg = "Spider.parse callback is not defined"
|
|
with pytest.raises(NotImplementedError, match=exc_msg):
|
|
spider.parse(resp)
|