PY3 port scrapy.spiders

scrapy/spiders/crawl.py

@@ -6,14 +6,17 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
+import six
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.spiders import Spider
 
+
 def identity(x):
     return x
 
+
 class Rule(object):
 
     def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
@@ -27,6 +30,7 @@ class Rule(object):
         else:
             self.follow = follow
 
+
 class CrawlSpider(Spider):
 
     rules = ()
@@ -49,7 +53,8 @@ class CrawlSpider(Spider):
             return
         seen = set()
         for n, rule in enumerate(self._rules):
-            links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
+            links = [lnk for lnk in rule.link_extractor.extract_links(response)
+                     if lnk not in seen]
             if links and rule.process_links:
                 links = rule.process_links(links)
             for link in links:
@@ -77,7 +82,7 @@ class CrawlSpider(Spider):
         def get_method(method):
             if callable(method):
                 return method
-            elif isinstance(method, basestring):
+            elif isinstance(method, six.string_types):
                 return getattr(self, method, None)
 
         self._rules = [copy.copy(r) for r in self.rules]

scrapy/spiders/init.py

@@ -1,6 +1,7 @@
 from scrapy.spiders import Spider
 from scrapy.utils.spider import iterate_spider_output
 
+
 class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 

scrapy/spiders/sitemap.py

@@ -1,5 +1,6 @@
 import re
 import logging
+import six
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
@@ -20,13 +21,14 @@ class SitemapSpider(Spider):
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
-            if isinstance(c, basestring):
+            if isinstance(c, six.string_types):
                 c = getattr(self, c)
             self._cbs.append((regex(r), c))
         self._follow = [regex(x) for x in self.sitemap_follow]
 
     def start_requests(self):
-        return (Request(x, callback=self._parse_sitemap) for x in self.sitemap_urls)
+        for url in self.sitemap_urls:
+            yield Request(url, self._parse_sitemap)
 
     def _parse_sitemap(self, response):
         if response.url.endswith('/robots.txt'):
@@ -52,8 +54,8 @@ class SitemapSpider(Spider):
                             break
 
     def _get_sitemap_body(self, response):
-        """Return the sitemap body contained in the given response, or None if the
-        response is not a sitemap.
+        """Return the sitemap body contained in the given response,
+        or None if the response is not a sitemap.
         """
         if isinstance(response, XmlResponse):
             return response.body
@@ -64,11 +66,13 @@ class SitemapSpider(Spider):
         elif response.url.endswith('.xml.gz'):
             return gunzip(response.body)
 
+
 def regex(x):
-    if isinstance(x, basestring):
+    if isinstance(x, six.string_types):
         return re.compile(x)
     return x
 
+
 def iterloc(it, alt=False):
     for d in it:
         yield d['loc']

scrapy/utils/gz.py

@@ -7,6 +7,7 @@ except ImportError:
 
 from gzip import GzipFile
 
+
 def gunzip(data):
     """Gunzip the given data and return as much data as possible.
 
@@ -31,7 +32,8 @@ def gunzip(data):
                 raise
     return output
 
+
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
-    ctype = response.headers.get('Content-Type', '')
-    return ctype in ('application/x-gzip', 'application/gzip')
+    ctype = response.headers.get('Content-Type', b'')
+    return ctype in (b'application/x-gzip', b'application/gzip')

tests/py3-ignores.txt

@@ -28,7 +28,6 @@ tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
 tests/test_spidermiddleware_offsite.py
 tests/test_spidermiddleware_referer.py
-tests/test_spider.py
 tests/test_utils_iterators.py
 tests/test_utils_template.py
 tests/test_webclient.py

tests/test_spider.py

@@ -301,26 +301,32 @@ class SitemapSpiderTest(SpiderTest):
     g.close()
     GZBODY = f.getvalue()
 
-    def test_get_sitemap_body(self):
+    def assertSitemapBody(self, response, body):
         spider = self.spider_class("example.com")
+        self.assertEqual(spider._get_sitemap_body(response), body)
 
+    def test_get_sitemap_body(self):
         r = XmlResponse(url="http://www.example.com/", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+        self.assertSitemapBody(r, self.BODY)
 
         r = HtmlResponse(url="http://www.example.com/", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), None)
+        self.assertSitemapBody(r, None)
 
         r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), None)
+        self.assertSitemapBody(r, None)
 
-        r = Response(url="http://www.example.com/sitemap", body=self.GZBODY, headers={"content-type": "application/gzip"})
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+    def test_get_sitemap_body_gzip_headers(self):
+        r = Response(url="http://www.example.com/sitemap", body=self.GZBODY,
+                     headers={"content-type": "application/gzip"})
+        self.assertSitemapBody(r, self.BODY)
 
+    def test_get_sitemap_body_xml_url(self):
         r = TextResponse(url="http://www.example.com/sitemap.xml", body=self.BODY)
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+        self.assertSitemapBody(r, self.BODY)
 
+    def test_get_sitemap_body_xml_url_compressed(self):
         r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)
-        self.assertEqual(spider._get_sitemap_body(r), self.BODY)
+        self.assertSitemapBody(r, self.BODY)
 
 
 class BaseSpiderDeprecationTest(unittest.TestCase):

tests/test_utils_gz.py

@@ -7,7 +7,7 @@ from tests import tests_datadir
 SAMPLEDIR = join(tests_datadir, 'compressed')
 
 
-class GzTest(unittest.TestCase):
+class GunzipTest(unittest.TestCase):
 
     def test_gunzip_basic(self):
         with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: