diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 0d1b76531..23cbd0116 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,8 +1,8 @@
"""
Link extractor based on lxml.html
"""
-import operator
import logging
+import operator
from functools import partial
from urllib.parse import urljoin, urlparse
@@ -94,8 +94,8 @@ class LxmlParserLinkExtractor:
try:
url = safe_url_string(url, encoding=response_encoding)
except ValueError:
- logger.debug(f"Skipping extraction of bad link {url}")
- continue # Disregard badly formatted urls
+ logger.debug(f"Skipping extraction of link with bad URL {url!r}")
+ continue
# to fix relative links after process_value
url = urljoin(response_url, url)
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 784fdb658..e1ec19601 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -1,7 +1,10 @@
import pickle
import re
import unittest
-import sys
+
+from packaging.version import Version
+from pytest import mark
+from w3lib import __version__ as w3lib_version
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.link import Link
@@ -817,13 +820,16 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
def test_restrict_xpaths_with_html_entities(self):
super().test_restrict_xpaths_with_html_entities()
- @unittest.skipIf(
- sys.version_info < (3, 8),
- reason="some library for python 3.7 so is less strict so bad links like htis don't crash scrapy",
+ @mark.skipif(
+ Version(w3lib_version) < Version("2.0.0"),
+ reason=(
+ "Before w3lib 2.0.0, w3lib.url.safe_url_string would not complain "
+ "about an invalid port value."
+ ),
)
def test_skip_bad_links(self):
html = b"""
- Why would you do this?
+ Why would you do this?
Good Link
Good Link 2
"""
diff --git a/tox.ini b/tox.ini
index 5a9d9cf29..873e7662b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -101,6 +101,7 @@ install_command =
python -I -m pip install {opts} {packages}
[testenv:pinned]
+basepython = python3.7
deps =
{[pinned]deps}
PyDispatcher==2.0.5