mirror of https://github.com/scrapy/scrapy.git
Remove the lxml version check for huge_tree on xmliter_lxml
iterparse supports the option since lxml 2.2.1, it was the HTML parser that only got it in 4.2
This commit is contained in:
parent
a49c8762dd
commit
ce9d290eff
|
|
@ -19,7 +19,7 @@ Scrapy 2.11.1 (unreleased)
|
|||
To minimize the impact of this change on existing code,
|
||||
:func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating
|
||||
the node namespace with a prefix in the node name, and big files with
|
||||
highly nested trees.
|
||||
highly nested trees when using libxml2 2.7+.
|
||||
|
||||
- Fixed regular expressions in the implementation of the
|
||||
:func:`~scrapy.utils.response.open_in_browser` function.
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ from typing import (
|
|||
from warnings import warn
|
||||
|
||||
from lxml import etree
|
||||
from packaging.version import Version
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Response, TextResponse
|
||||
|
|
@ -31,12 +30,6 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LXML_VERSION = Version(etree.__version__)
|
||||
_LXML_HUGE_TREE_VERSION = Version("4.2")
|
||||
_ITERPARSE_KWARGS = {}
|
||||
if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION:
|
||||
_ITERPARSE_KWARGS["huge_tree"] = True
|
||||
|
||||
|
||||
def xmliter(
|
||||
obj: Union[Response, str, bytes], nodename: str
|
||||
|
|
@ -108,7 +101,7 @@ def xmliter_lxml(
|
|||
reader,
|
||||
encoding=reader.encoding,
|
||||
events=("end", "start-ns"),
|
||||
**_ITERPARSE_KWARGS,
|
||||
huge_tree=True,
|
||||
)
|
||||
selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
|
||||
needs_namespace_resolution = not namespace and ":" in nodename
|
||||
|
|
|
|||
Loading…
Reference in New Issue