Add try/except to safe_url_string()

Added a try catch condition to the safe_url_string() processing
in the LxmlParserLinkExtractor class to avoid scrapers crashing
unneccessarily
This commit is contained in:
Samuel Bartlett 2023-03-30 13:07:51 +00:00
parent 8045d7eaa5
commit 90ce6589ee
1 changed files with 5 additions and 1 deletions

View File

@ -88,7 +88,11 @@ class LxmlParserLinkExtractor:
url = self.process_attr(attr_val)
if url is None:
continue
url = safe_url_string(url, encoding=response_encoding)
try:
url = safe_url_string(url, encoding=response_encoding)
except ValueError:
continue # Disregard badly formatted urls
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(