From 5360ba34bc345667f77a4d4256f15fd648e42e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Wed, 21 Jun 2023 11:08:53 -0700 Subject: [PATCH] IOError and other cleanup (#4716) --- docs/utils/linkfix.py | 2 +- scrapy/downloadermiddlewares/decompression.py | 4 ++-- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/settings/default_settings.py | 4 ++-- scrapy/utils/gz.py | 2 +- scrapy/utils/python.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_downloadermiddleware.py | 4 ++-- tests/test_mail.py | 2 -- tests/test_robotstxt_interface.py | 1 - tests/test_utils_gz.py | 2 +- tests/test_utils_iterators.py | 4 ++-- 12 files changed, 14 insertions(+), 17 deletions(-) diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 1f270837c..c17b9d511 100644 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -30,7 +30,7 @@ def main(): try: with Path("build/linkcheck/output.txt").open(encoding="utf-8") as out: output_lines = out.readlines() - except IOError: + except OSError: print("linkcheck output not found; please run linkcheck first.") sys.exit(1) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 5839dc243..3b8702419 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -63,7 +63,7 @@ class DecompressionMiddleware: archive = BytesIO(response.body) try: body = gzip.GzipFile(fileobj=archive).read() - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) @@ -72,7 +72,7 @@ class DecompressionMiddleware: def _is_bzip2(self, response): try: body = bz2.decompress(response.body) - except IOError: + except OSError: return respcls = responsetypes.from_args(body=body) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index b9316c43a..ac87d4a4e 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -37,7 +37,7 @@ class HttpCacheMiddleware: ConnectionLost, TCPTimedOutError, ResponseFailed, - IOError, + OSError, ) def __init__(self, settings: Settings, stats: StatsCollector) -> None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 89837b4ab..a4cb555bd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -268,9 +268,9 @@ RETRY_EXCEPTIONS = [ "twisted.internet.error.ConnectionLost", "twisted.internet.error.TCPTimedOutError", "twisted.web.client.ResponseFailed", - # IOError is raised by the HttpCompression middleware when trying to + # OSError is raised by the HttpCompression middleware when trying to # decompress an empty response - IOError, + OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e5df34d2e..77e0197d8 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -15,7 +15,7 @@ def gunzip(data): try: chunk = f.read1(8196) output_list.append(chunk) - except (IOError, EOFError, struct.error): + except (OSError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error # some pages are quite small so output_list is empty and f.extrabuf diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 27816c0df..bb5dbebbc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -291,7 +291,7 @@ def without_none_values(iterable): try: return {k: v for k, v in iterable.items() if v is not None} except AttributeError: - return type(iterable)((v for v in iterable if v is not None)) + return type(iterable)(v for v in iterable if v is not None) def global_object_name(obj): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fd4176e2f..9731b62c4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -129,7 +129,7 @@ class FileTestCase(unittest.TestCase): def test_non_existent(self): request = Request(f"file://{self.mktemp()}") d = self.download_request(request, Spider("foo")) - return self.assertFailure(d, IOError) + return self.assertFailure(d, OSError) class ContentLengthHeaderResource(resource.Resource): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 060cfe08b..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -70,7 +70,7 @@ class DefaultsTest(ManagerTestCase): In particular when some website returns a 30x response with header 'Content-Encoding: gzip' giving as result the error below: - exceptions.IOError: Not a gzipped file + BadGzipFile: Not a gzipped file (...) """ req = Request("http://example.com") @@ -108,7 +108,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - self.assertRaises(IOError, self._download, request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): diff --git a/tests/test_mail.py b/tests/test_mail.py index bc7298e9d..504c78486 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,5 +1,3 @@ -# coding=utf-8 - import unittest from email.charset import Charset from io import BytesIO diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 8d87a322a..d7a923085 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,4 +1,3 @@ -# coding=utf-8 from twisted.trial import unittest diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py index 6b2a458bc..7b7a25db8 100644 --- a/tests/test_utils_gz.py +++ b/tests/test_utils_gz.py @@ -28,7 +28,7 @@ class GunzipTest(unittest.TestCase): def test_gunzip_no_gzip_file_raises(self): self.assertRaises( - IOError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() + OSError, gunzip, (SAMPLEDIR / "feed-sample1.xml").read_bytes() ) def test_gunzip_truncated_short(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index faf7d2709..3598fa0bb 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -346,8 +346,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) + self.assertTrue(all(isinstance(k, str) for k in result_row.keys())) + self.assertTrue(all(isinstance(v, str) for v in result_row.values())) def test_csviter_delimiter(self): body = get_testdata("feeds", "feed-sample3.csv").replace(b",", b"\t")