From d5984bbea99f81765596e1aa57d03aff51612576 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 28 Aug 2015 02:12:36 +0500 Subject: [PATCH] PY3 port scrapy.spiders --- scrapy/spiders/crawl.py | 9 +++++++-- scrapy/spiders/init.py | 1 + scrapy/spiders/sitemap.py | 14 +++++++++----- scrapy/utils/gz.py | 6 ++++-- tests/py3-ignores.txt | 1 - tests/test_spider.py | 22 ++++++++++++++-------- tests/test_utils_gz.py | 2 +- 7 files changed, 36 insertions(+), 19 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 77551753e..031f649d6 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,14 +6,17 @@ See documentation in docs/topics/spiders.rst """ import copy +import six from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output from scrapy.spiders import Spider + def identity(x): return x + class Rule(object): def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity): @@ -27,6 +30,7 @@ class Rule(object): else: self.follow = follow + class CrawlSpider(Spider): rules = () @@ -49,7 +53,8 @@ class CrawlSpider(Spider): return seen = set() for n, rule in enumerate(self._rules): - links = [l for l in rule.link_extractor.extract_links(response) if l not in seen] + links = [lnk for lnk in rule.link_extractor.extract_links(response) + if lnk not in seen] if links and rule.process_links: links = rule.process_links(links) for link in links: @@ -77,7 +82,7 @@ class CrawlSpider(Spider): def get_method(method): if callable(method): return method - elif isinstance(method, basestring): + elif isinstance(method, six.string_types): return getattr(self, method, None) self._rules = [copy.copy(r) for r in self.rules] diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 7717c8819..2efb1a869 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,6 +1,7 @@ from scrapy.spiders import Spider from scrapy.utils.spider import iterate_spider_output + class InitSpider(Spider): """Base Spider with initialization facilities""" diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 5aa0b944d..eede467a8 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,5 +1,6 @@ import re import logging +import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse @@ -20,13 +21,14 @@ class SitemapSpider(Spider): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: - if isinstance(c, basestring): + if isinstance(c, six.string_types): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow] def start_requests(self): - return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls) + for url in self.sitemap_urls: + yield Request(url, self._parse_sitemap) def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): @@ -52,8 +54,8 @@ class SitemapSpider(Spider): break def _get_sitemap_body(self, response): - """Return the sitemap body contained in the given response, or None if the - response is not a sitemap. + """Return the sitemap body contained in the given response, + or None if the response is not a sitemap. """ if isinstance(response, XmlResponse): return response.body @@ -64,11 +66,13 @@ class SitemapSpider(Spider): elif response.url.endswith('.xml.gz'): return gunzip(response.body) + def regex(x): - if isinstance(x, basestring): + if isinstance(x, six.string_types): return re.compile(x) return x + def iterloc(it, alt=False): for d in it: yield d['loc'] diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 741948359..7fa4bba57 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -7,6 +7,7 @@ except ImportError: from gzip import GzipFile + def gunzip(data): """Gunzip the given data and return as much data as possible. @@ -31,7 +32,8 @@ def gunzip(data): raise return output + def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" - ctype = response.headers.get('Content-Type', '') - return ctype in ('application/x-gzip', 'application/gzip') + ctype = response.headers.get('Content-Type', b'') + return ctype in (b'application/x-gzip', b'application/gzip') diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index b40293f57..2eb22f149 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -28,7 +28,6 @@ tests/test_spidermiddleware_depth.py tests/test_spidermiddleware_httperror.py tests/test_spidermiddleware_offsite.py tests/test_spidermiddleware_referer.py -tests/test_spider.py tests/test_utils_iterators.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_spider.py b/tests/test_spider.py index f2dfd2dce..4d5d4b07e 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -301,26 +301,32 @@ class SitemapSpiderTest(SpiderTest): g.close() GZBODY = f.getvalue() - def test_get_sitemap_body(self): + def assertSitemapBody(self, response, body): spider = self.spider_class("example.com") + self.assertEqual(spider._get_sitemap_body(response), body) + def test_get_sitemap_body(self): r = XmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + self.assertSitemapBody(r, self.BODY) r = HtmlResponse(url="http://www.example.com/", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), None) + self.assertSitemapBody(r, None) r = Response(url="http://www.example.com/favicon.ico", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), None) + self.assertSitemapBody(r, None) - r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"}) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + def test_get_sitemap_body_gzip_headers(self): + r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, + headers={"content-type": "application/gzip"}) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url(self): r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_body_xml_url_compressed(self): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) - self.assertEqual(spider._get_sitemap_body(r), self.BODY) + self.assertSitemapBody(r, self.BODY) class BaseSpiderDeprecationTest(unittest.TestCase): diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 94e7b71be..8fb1e414d 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -7,7 +7,7 @@ from tests import tests_datadir SAMPLEDIR = join(tests_datadir, 'compressed') -class GzTest(unittest.TestCase): +class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: