Merge pull request #5458 from reidemeister94/fix/unique-list-link-extractors

fix: return unique_list only when link_extractor.unique is True

scrapy/linkextractors/lxmlhtml.py

@@ -195,7 +195,8 @@ class LxmlLinkExtractor:
         Only links that match the settings passed to the ``__init__`` method of
         the link extractor are returned.
 
-        Duplicate links are omitted.
+        Duplicate links are omitted if the ``unique`` attribute is set to ``True``,
+        otherwise they are returned.
         """
         base_url = get_base_url(response)
         if self.restrict_xpaths:
@@ -210,4 +211,6 @@ class LxmlLinkExtractor:
         for doc in docs:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
-        return unique_list(all_links)
+        if self.link_extractor.unique:
+            return unique_list(all_links)
+        return all_links

tests/sample_data/link_extractor/linkextractor.html

@@ -13,6 +13,7 @@
       </div>
       <a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
       <a href='sample3.html'>sample 3 repetition</a>
+      <a href='sample3.html'>sample 3 repetition</a>
       <a href='sample3.html#foo'>sample 3 repetition with fragment</a>
       <a href='http://www.google.com/something'></a>
       <a href='http://example.com/innertag.html'><strong>inner</strong> tag</a>

tests/test_http_response.py

@@ -541,6 +541,7 @@ class TextResponseTest(BaseResponseTest):
             'http://example.com/sample2.html',
             'http://example.com/sample3.html',
             'http://example.com/sample3.html',
+            'http://example.com/sample3.html',
             'http://example.com/sample3.html#foo',
             'http://www.google.com/something',
             'http://example.com/innertag.html'

tests/test_linkextractors.py

@@ -53,6 +53,7 @@ class Base:
                 Link(url='http://example.com/sample2.html', text='sample 2'),
                 Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment')
             ])
 
@@ -64,6 +65,7 @@ class Base:
                 Link(url='http://example.com/sample2.html', text='sample 2'),
                 Link(url='http://example.com/sample3.html', text='sample 3 text'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
+                Link(url='http://example.com/sample3.html', text='sample 3 repetition'),
                 Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment')
             ])