Add logging Lines

Adds an error loggign line to the LinkExtractor to detail encountered
bad links
This commit is contained in:
Samuel Bartlett 2023-03-31 08:01:54 +00:00
parent 90ce6589ee
commit 9ef00c5c0b
1 changed files with 4 additions and 0 deletions

View File

@ -2,6 +2,7 @@
Link extractor based on lxml.html
"""
import operator
import logging
from functools import partial
from urllib.parse import urljoin, urlparse
@ -23,6 +24,8 @@ from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
logger = logging.getLogger(__name__)
# from lxml/src/lxml/html/__init__.py
XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
@ -91,6 +94,7 @@ class LxmlParserLinkExtractor:
try:
url = safe_url_string(url, encoding=response_encoding)
except ValueError:
logger.error(f"Skipping extraction of bad link {url}")
continue # Disregard badly formatted urls
# to fix relative links after process_value