diff --git a/docs/news.rst b/docs/news.rst index 525ddbf40..fab8b6f20 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,7 +19,7 @@ Scrapy 2.11.1 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace with a prefix in the node name, and big files with - highly nested trees. + highly nested trees when using libxml2 2.7+. - Fixed regular expressions in the implementation of the :func:`~scrapy.utils.response.open_in_browser` function. diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 8610e9b77..b6abe2e0c 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -19,7 +19,6 @@ from typing import ( from warnings import warn from lxml import etree -from packaging.version import Version from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -31,12 +30,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_LXML_VERSION = Version(etree.__version__) -_LXML_HUGE_TREE_VERSION = Version("4.2") -_ITERPARSE_KWARGS = {} -if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: - _ITERPARSE_KWARGS["huge_tree"] = True - def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -108,7 +101,7 @@ def xmliter_lxml( reader, encoding=reader.encoding, events=("end", "start-ns"), - **_ITERPARSE_KWARGS, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) needs_namespace_resolution = not namespace and ":" in nodename