PY3 depth, offsite and referer spider middlewares; Crawler

scrapy/spidermiddlewares/depth.py

@@ -35,14 +35,18 @@ class DepthMiddleware(object):
                 if self.prio:
                     request.priority -= depth * self.prio
                 if self.maxdepth and depth > self.maxdepth:
-                    logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                                 {'maxdepth': self.maxdepth, 'requrl': request.url},
-                                 extra={'spider': spider})
+                    logger.debug(
+                        "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                        {'maxdepth': self.maxdepth, 'requrl': request.url},
+                        extra={'spider': spider}
+                    )
                     return False
                 elif self.stats:
                     if self.verbose_stats:
-                        self.stats.inc_value('request_depth_count/%s' % depth, spider=spider)
-                    self.stats.max_value('request_depth_max', depth, spider=spider)
+                        self.stats.inc_value('request_depth_count/%s' % depth,
+                                             spider=spider)
+                    self.stats.max_value('request_depth_max', depth,
+                                         spider=spider)
             return True
 
         # base case (depth=0)

scrapy/spidermiddlewares/offsite.py

@@ -13,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
 
 logger = logging.getLogger(__name__)
 
+
 class OffsiteMiddleware(object):
 
     def __init__(self, stats):

tests/py3-ignores.txt

@@ -5,7 +5,6 @@ tests/test_commands.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_crawler.py
 tests/test_downloader_handlers.py
 tests/test_downloadermiddleware_ajaxcrawlable.py
 tests/test_downloadermiddleware_defaultheaders.py
@@ -24,10 +23,7 @@ tests/test_mail.py
 tests/test_pipeline_files.py
 tests/test_pipeline_images.py
 tests/test_proxy_connect.py
-tests/test_spidermiddleware_depth.py
 tests/test_spidermiddleware_httperror.py
-tests/test_spidermiddleware_offsite.py
-tests/test_spidermiddleware_referer.py
 tests/test_utils_iterators.py
 tests/test_utils_template.py
 tests/test_webclient.py

tests/test_spidermiddleware_referer.py

@@ -17,5 +17,5 @@ class TestRefererMiddleware(TestCase):
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))
         self.assertEquals(out[0].headers.get('Referer'),
-                          'http://scrapytest.org')
+                          b'http://scrapytest.org')