RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize SitemapSpider memory usage (#7007)

This commit is contained in:
Albert Eduardovich N. 2026-04-08 14:59:16 +03:00 committed by GitHub
parent a4377f9a4f
commit 9fffcc1b82
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 278 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
scrapy/spiders/sitemap.py
View File

@ -72,31 +72,53 @@ class SitemapSpider(Spider):
def _parse_sitemap(self, response: Response) -> Iterable[Request]:
if response.url.endswith("/robots.txt"):
for url in sitemap_urls_from_robots(response.text, base_url=response.url):
yield Request(url, callback=self._parse_sitemap)
else:
body = self._get_sitemap_body(response)
if body is None:
logger.warning(
"Ignoring invalid sitemap: %(response)s",
{"response": response},
extra={"spider": self},
)
return
urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))
return (Request(url, callback=self._parse_sitemap) for url in urls)
s = Sitemap(body)
it = self.sitemap_filter(s)
body = self._get_sitemap_body(response)
if not body:
logger.warning(
"Ignoring invalid sitemap: %(response)s",
{"response": response},
extra={"spider": self},
)
return ()
if s.type == "sitemapindex":
for loc in iterloc(it, self.sitemap_alternate_links):
if any(x.search(loc) for x in self._follow):
yield Request(loc, callback=self._parse_sitemap)
elif s.type == "urlset":
for loc in iterloc(it, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield Request(loc, callback=c)
break
s = Sitemap(body)
if s.type == "sitemapindex":
urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s)))
return (Request(loc, callback=self._parse_sitemap) for loc in urls)
if s.type == "urlset":
url_callback_pairs = list(
self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s))
)
return (Request(loc, callback=c) for loc, c in url_callback_pairs)
logger.warning(
"Ignoring invalid sitemap: %(response)s",
{"response": response},
extra={"spider": self},
)
return ()
def _get_urls_from_sitemapindex(
self, it: Iterable[dict[str, Any]]
) -> Iterable[str]:
for loc in iterloc(it, self.sitemap_alternate_links):
if any(x.search(loc) for x in self._follow):
yield loc
def _get_urls_and_callbacks_from_urlset(
self, it: Iterable[dict[str, Any]]
) -> Iterable[tuple[str, CallbackT]]:
for loc in iterloc(it, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield loc, c
break
def _get_sitemap_body(self, response: Response) -> bytes | None:
"""Return the sitemap body contained in the given response,
@ -140,8 +162,9 @@ def regex(x: re.Pattern[str] | str) -> re.Pattern[str]:
def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]:
for d in it:
yield d["loc"]
if loc := d["loc"]:
yield loc
# Also consider alternate URLs (xhtml:link rel="alternate")
if alt and "alternate" in d:
yield from d["alternate"]
if alt and (alt_list := d.get("alternate")):
yield from alt_list

126
scrapy/utils/sitemap.py
View File

@ -7,11 +7,15 @@ SitemapSpider, its API is subject to change without notice.
from __future__ import annotations
import warnings
from io import BytesIO, StringIO
from typing import TYPE_CHECKING, Any
from urllib.parse import urljoin
import lxml.etree
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
@ -20,40 +24,112 @@ class Sitemap:
"""Class to parse Sitemap (type=urlset) and Sitemap Index
(type=sitemapindex) files"""
__slots__ = ("type", "xmliter")
def __init__(self, xmltext: str | bytes):
xmlp = lxml.etree.XMLParser(
recover=True, remove_comments=True, resolve_entities=False
if isinstance(xmltext, str):
warnings.warn(
"Passing `str` type as `xmltext` is deprecated, use `bytes`",
ScrapyDeprecationWarning,
stacklevel=2,
)
xmltext = xmltext.encode()
self.xmliter = lxml.etree.iterparse(
BytesIO(xmltext),
recover=True,
remove_comments=True,
resolve_entities=False,
remove_blank_text=True,
collect_ids=False,
remove_pis=True,
events=("start", "end"),
)
self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
rt = self._root.tag
assert isinstance(rt, str)
self.type = rt.split("}", 1)[1] if "}" in rt else rt
_, root = next(self.xmliter)
self.type = self._get_tag_name(root)
def __iter__(self) -> Iterator[dict[str, Any]]:
for elem in self._root.getchildren():
d: dict[str, Any] = {}
for el in elem.getchildren():
tag = el.tag
assert isinstance(tag, str)
name = tag.split("}", 1)[1] if "}" in tag else tag
for event, elem in self.xmliter:
if event == "start":
continue
if name == "link":
if "href" in el.attrib:
d.setdefault("alternate", []).append(el.get("href"))
else:
d[name] = el.text.strip() if el.text else ""
if self._get_tag_name(elem) not in {"url", "sitemap"}:
continue
if "loc" in d:
if d := self._process_sitemap_element(elem):
yield d
def _process_sitemap_element(
self, elem: lxml.etree._Element
) -> dict[str, Any] | None:
d: dict[str, Any] = {}
alternate: list[str] = []
has_loc = False
for el in elem:
try:
tag_name = self._get_tag_name(el)
if not tag_name:
continue
if tag_name == "link":
if href := el.get("href"):
alternate.append(href)
else:
d[tag_name] = el.text.strip() if el.text else ""
if not has_loc and tag_name == "loc":
has_loc = True
finally:
el.clear()
elem.clear()
parent = elem.getparent()
if parent is not None:
while elem.getprevious() is not None:
del parent[0]
if not has_loc:
return None
if alternate:
d["alternate"] = alternate
return d
@staticmethod
def _get_tag_name(elem: lxml.etree._Element) -> str:
if TYPE_CHECKING:
assert isinstance(elem.tag, str)
_, _, localname = elem.tag.partition("}")
return localname or elem.tag
def sitemap_urls_from_robots(
robots_text: str, base_url: str | None = None
robots_text: str | bytes,
base_url: str | None = None,
) -> Iterable[str]:
"""Return an iterator over all sitemap urls contained in the given
robots.txt file
"""
for line in robots_text.splitlines():
if line.lstrip().lower().startswith("sitemap:"):
url = line.split(":", 1)[1].strip()
if isinstance(robots_text, bytes):
for line in BytesIO(robots_text):
if line.lstrip()[:8].lower() == b"sitemap:":
try:
url = line.partition(b":")[2].strip().decode()
except UnicodeDecodeError:
continue
yield urljoin(base_url or "", url)
else:
yield from _sitemap_urls_from_robots_str(robots_text, base_url)
def _sitemap_urls_from_robots_str(
robots_text: str,
base_url: str | None = None,
) -> Iterable[str]:
warnings.warn(
"Passing `str` type as `robots_text` is deprecated, use `bytes`",
ScrapyDeprecationWarning,
stacklevel=2,
)
for line in StringIO(robots_text):
if line.lstrip()[:8].lower() == "sitemap:":
url = line.partition(":")[2].strip()
yield urljoin(base_url or "", url)

91
tests/test_spider_sitemap.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import gzip
import re
import warnings
from datetime import datetime
from io import BytesIO
@ -43,6 +44,9 @@ class TestSitemapSpider(TestSpider):
r = Response(url="http://www.example.com/favicon.ico", body=self.BODY)
self.assertSitemapBody(r, None)
r = XmlResponse(url="http://www.example.com/", body=b"")
self.assertSitemapBody(r, b"")
def test_get_sitemap_body_gzip_headers(self):
r = Response(
url="http://www.example.com/sitemap",
@ -85,6 +89,20 @@ Sitemap: /sitemap-relative-url.xml
"http://www.example.com/sitemap-relative-url.xml",
]
def test_get_sitemap_urls_from_robotstxt_skips_invalid_utf8_urls(self):
robots = (
b"User-agent: *\n"
b"Sitemap: http://example.com/\xff.xml\n"
b"Sitemap: http://example.com/ok.xml\n"
)
r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
spider = self.spider_class("example.com")
assert [req.url for req in spider._parse_sitemap(r)] == [
"http://example.com/ok.xml",
]
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
@ -218,6 +236,79 @@ Sitemap: /sitemap-relative-url.xml
"http://www.example.com/sitemap2.xml"
]
@pytest.mark.parametrize(
("rule", "result"),
[(r"english", ["http://www.example.com/english/"]), (r"nonexistent", [])],
)
def test_sitemap_filter_with_rule(self, rule: str, result: list[str]):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url><loc>http://www.example.com/english/</loc></url>
<url><loc>http://www.example.com/portuguese/</loc></url>
</urlset>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
class _RuleSpider(self.spider_class): # type: ignore[name-defined,misc]
sitemap_rules = [(rule, "parse")]
spider = _RuleSpider("example.com")
urls = [req.url for req in spider._parse_sitemap(r)]
assert urls == result
def test_parse_sitemap_empty_body(self):
r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
spider = self.spider_class("example.com")
with LogCapture() as lc:
results = list(spider._parse_sitemap(r))
assert not results
lc.check(
(
"scrapy.spiders.sitemap",
"WARNING",
"Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>",
)
)
def test_parse_sitemap_not_sitemap(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
<some attr="string">
<tag><tag3>sometext</tag3></tag>
<tag2><tag4>sometext2</tag4></tag2>
</some>"""
r = XmlResponse(url="http://www.example.com/random.xml", body=body)
spider = self.spider_class("example.com")
results = list(spider._parse_sitemap(r))
assert not results
@pytest.mark.parametrize(
("follow", "result"),
[
(r"1.xml", ["http://www.example.com/sitemap1.xml"]),
(re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]),
(r"nonexistent", []),
],
)
def test_sitemap_follow(self, follow, result):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<loc>http://www.example.com/sitemap1.xml</loc>
</sitemap>
</sitemapindex>"""
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
class _FollowSpider(self.spider_class):
sitemap_follow = [follow]
spider = _FollowSpider("example.com")
urls = [req.url for req in spider._parse_sitemap(r)]
assert urls == result
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)

38
tests/test_utils_sitemap.py
View File

@ -1,3 +1,5 @@
import warnings
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
@ -156,7 +158,7 @@ def test_sitemap_wrong_ns2():
def test_sitemap_urls_from_robots():
robots = """User-agent: *
robots = b"""User-agent: *
Disallow: /aff/
Disallow: /wl/
@ -182,6 +184,40 @@ Disallow: /forum/active/
]
def test_sitemap_urls_from_robots_str_compat():
robots = """User-agent: *
Disallow: /aff/
Disallow: /wl/
# Search and shopping refining
Disallow: /s*/*facet
Disallow: /s*/*tags
# Sitemap files
Sitemap: http://example.com/sitemap.xml
Sitemap: http://example.com/sitemap-product-index.xml
Sitemap: HTTP://example.com/sitemap-uppercase.xml
Sitemap: /sitemap-relative-url.xml
# Forums
Disallow: /forum/search/
Disallow: /forum/active/
"""
with warnings.catch_warnings(record=True) as w:
assert list(
sitemap_urls_from_robots(robots, base_url="http://example.com")
) == [
"http://example.com/sitemap.xml",
"http://example.com/sitemap-product-index.xml",
"http://example.com/sitemap-uppercase.xml",
"http://example.com/sitemap-relative-url.xml",
]
assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str(
w[0].message
)
def test_sitemap_blanklines():
"""Assert we can deal with starting blank lines before <xml> tag"""
s = Sitemap(