fix: return unique_list only when link_extractor.unique is True

2022-03-23 17:28:55 +01:00 · 2022-03-23 17:28:55 +01:00 · 0beed7055c
parent 8c3e5a2421
commit 0beed7055c
1 changed files with 3 additions and 1 deletions
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@ -161,4 +161,6 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
        for doc in docs:
            links = self._extract_links(doc, response.url, response.encoding, base_url)
            all_links.extend(self._process_links(links))
-        return unique_list(all_links)
+        if self.link_extractor.unique:
+            return unique_list(all_links)
+        return all_links