Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info

See also: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53

docs/topics/downloader-middleware.rst

@@ -685,13 +685,13 @@ HttpCompressionMiddleware
    sent/received from web sites.
 
    This middleware also supports decoding `brotli-compressed`_ as well as
-   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is
+   `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
    installed, respectively.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://pypi.org/project/brotlipy/
 .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
-.. _zstd: https://pypi.org/project/zstd/
+.. _zstandard: https://pypi.org/project/zstandard/
 
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,4 +1,5 @@
 import zlib
+import io
 
 from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
@@ -15,7 +16,7 @@ except ImportError:
     pass
 
 try:
-    import zstd
+    import zstandard
     ACCEPTED_ENCODINGS.append(b'zstd')
 except ImportError:
     pass
@@ -74,5 +75,8 @@ class HttpCompressionMiddleware:
         if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
             body = brotli.decompress(body)
         if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
-            body = zstd.decompress(body)
+            # Using its streaming API since its simple API could handle only cases
+            # where there is content size data embedded in the frame
+            reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
+            body = reader.read()
         return body