diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 1e6ab984a..3f90ed84a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -195,7 +195,8 @@ class LxmlLinkExtractor:
Only links that match the settings passed to the ``__init__`` method of
the link extractor are returned.
- Duplicate links are omitted.
+ Duplicate links are omitted if the ``unique`` attribute is set to ``True``,
+ otherwise they are returned.
"""
base_url = get_base_url(response)
if self.restrict_xpaths:
@@ -210,4 +211,6 @@ class LxmlLinkExtractor:
for doc in docs:
links = self._extract_links(doc, response.url, response.encoding, base_url)
all_links.extend(self._process_links(links))
- return unique_list(all_links)
+ if self.link_extractor.unique:
+ return unique_list(all_links)
+ return all_links
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index e3a2a4145..29075602d 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -13,6 +13,7 @@
sample 3 text
sample 3 repetition
+ sample 3 repetition
sample 3 repetition with fragment
inner tag
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index bb16b8904..3875bbcb1 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -541,6 +541,7 @@ class TextResponseTest(BaseResponseTest):
'http://example.com/sample2.html',
'http://example.com/sample3.html',
'http://example.com/sample3.html',
+ 'http://example.com/sample3.html',
'http://example.com/sample3.html#foo',
'http://www.google.com/something',
'http://example.com/innertag.html'
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index e28dc9bdb..6c34a96a0 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -53,6 +53,7 @@ class Base:
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+ Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
])
@@ -64,6 +65,7 @@ class Base:
Link(url='http://example.com/sample2.html', text='sample 2'),
Link(url='http://example.com/sample3.html', text='sample 3 text'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+ Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
])