fix get_meta_refresh bug raised for TextResponses without encoding

scrapy/tests/test_utils_response.py

@@ -93,6 +93,12 @@ class ResponseUtilsTest(unittest.TestCase):
         response = Response(url='http://example.com', body=body)
         self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT'))
 
+        # responses without refresh tag should return None None
+        response = Response(url='http://example.org')
+        self.assertEqual(get_meta_refresh(response), (None, None))
+        response = TextResponse(url='http://example.org')
+        self.assertEqual(get_meta_refresh(response), (None, None))
+
     def test_response_httprepr(self):
         r1 = Response("http://www.example.com")
         self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n')

scrapy/utils/response.py

@@ -47,7 +47,7 @@ def get_meta_refresh(response):
     If no meta redirect is found, (None, None) is returned.
     """
     if response not in _metaref_cache:
-        encoding = getattr(response, 'encoding', 'utf-8')
+        encoding = getattr(response, 'encoding', None) or 'utf-8'
         body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \
             errors='ignore'))
         match = META_REFRESH_RE.search(body_chunk)