From 0beed7055cdec3683bf67ec01b573ddf87df723f Mon Sep 17 00:00:00 2001
From: Silvio Pavanetto <silvio.pavanetto@gmail.com>
Date: Wed, 23 Mar 2022 17:28:55 +0100
Subject: [PATCH] fix: return unique_list only when link_extractor.unique is
 True

---
 scrapy/linkextractors/lxmlhtml.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index b5d2585a8..caef504a0 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -161,4 +161,6 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
         for doc in docs:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
-        return unique_list(all_links)
+        if self.link_extractor.unique:
+            return unique_list(all_links)
+        return all_links