response_httprepr memory issue fixed (#4972)

* response_httprepr replaced by response.body

* unused import deleted

* get_header_size function added

* response size calculation updated

* flake8 codestyle fix

* added counting status code, line breaks to response size

* get_status size: list to tuple, comments added

* test added: comparing new response size counting method with old `len(response_httprepr)`

* downloader stats : unreachable code deleted

* `get_status_size` optimized

* comment added

* tests.test_downloadermiddleware_stats: statement formatting updated

* scrapy.utils.response: `response_httprepr` -> deprecated

* tests.test_downloadermiddleware_stats: flake8 fix

scrapy/downloadermiddlewares/stats.py

@@ -1,7 +1,22 @@
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.python import global_object_name, to_bytes
 from scrapy.utils.request import request_httprepr
-from scrapy.utils.response import response_httprepr
-from scrapy.utils.python import global_object_name
+
+from twisted.web import http
+
+
+def get_header_size(headers):
+    size = 0
+    for key, value in headers.items():
+        if isinstance(value, (list, tuple)):
+            for v in value:
+                size += len(b": ") + len(key) + len(v)
+    return size + len(b'\r\n') * (len(headers.keys()) - 1)
+
+
+def get_status_size(response_status):
+    return len(to_bytes(http.RESPONSES.get(response_status, b''))) + 15
+    # resp.status + b"\r\n" + b"HTTP/1.1 <100-599> "
 
 
 class DownloaderStats:
@@ -24,7 +39,8 @@ class DownloaderStats:
     def process_response(self, request, response, spider):
         self.stats.inc_value('downloader/response_count', spider=spider)
         self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
-        reslen = len(response_httprepr(response))
+        reslen = len(response.body) + get_header_size(response.headers) + get_status_size(response.status) + 4
+        # response.body + b"\r\n"+ response.header + b"\r\n" + response.status
         self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
         return response
 

scrapy/utils/response.py

@@ -14,6 +14,7 @@ from scrapy.http.response import Response
 
 from twisted.web import http
 from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.decorators import deprecated
 from w3lib import html
 
 
@@ -51,6 +52,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
     return f'{status_int} {to_unicode(message)}'
 
 
+@deprecated
 def response_httprepr(response: Response) -> bytes:
     """Return raw HTTP representation (as bytes) of the given response. This
     is provided only for reference, since it's not the exact stream of bytes

tests/test_downloadermiddleware_stats.py

@@ -1,8 +1,10 @@
+from itertools import product
 from unittest import TestCase
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
+from scrapy.utils.response import response_httprepr
 from scrapy.utils.test import get_crawler
 
 
@@ -37,6 +39,23 @@ class TestDownloaderStats(TestCase):
         self.mw.process_response(self.req, self.res, self.spider)
         self.assertStatsEqual('downloader/response_count', 1)
 
+    def test_response_len(self):
+        body = (b'', b'not_empty')  # empty/notempty body
+        headers = ({}, {'lang': 'en'}, {'lang': 'en', 'User-Agent': 'scrapy'})  # 0 headers, 1h and 2h
+        test_responses = [  # form test responses with all combinations of body/headers
+            Response(
+                url='scrapytest.org',
+                status=200,
+                body=r[0],
+                headers=r[1]
+            )
+            for r in product(body, headers)
+        ]
+        for test_response in test_responses:
+            self.crawler.stats.set_value('downloader/response_bytes', 0)
+            self.mw.process_response(self.req, test_response, self.spider)
+            self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
+
     def test_process_exception(self):
         self.mw.process_exception(self.req, MyException(), self.spider)
         self.assertStatsEqual('downloader/exception_count', 1)