From 0beed7055cdec3683bf67ec01b573ddf87df723f Mon Sep 17 00:00:00 2001 From: Silvio Pavanetto Date: Wed, 23 Mar 2022 17:28:55 +0100 Subject: [PATCH 1/5] fix: return unique_list only when link_extractor.unique is True --- scrapy/linkextractors/lxmlhtml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index b5d2585a8..caef504a0 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -161,4 +161,6 @@ class LxmlLinkExtractor(FilteringLinkExtractor): for doc in docs: links = self._extract_links(doc, response.url, response.encoding, base_url) all_links.extend(self._process_links(links)) - return unique_list(all_links) + if self.link_extractor.unique: + return unique_list(all_links) + return all_links From b0ec38876d52febb6811348961b75247ab4fe1c8 Mon Sep 17 00:00:00 2001 From: spav Date: Wed, 11 Jan 2023 19:32:33 +0100 Subject: [PATCH 2/5] Fix docstring for extract_links specifying duplicates case --- scrapy/linkextractors/lxmlhtml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 14eb24862..8a4175d49 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -195,7 +195,8 @@ class LxmlLinkExtractor: Only links that match the settings passed to the ``__init__`` method of the link extractor are returned. - Duplicate links are omitted. + Duplicate links are omitted if the ``unique`` parameter is set to ``True``, + otherwise they are returned. """ base_url = get_base_url(response) if self.restrict_xpaths: From b386c648643529f3f2d9e4d1783632840e6fe270 Mon Sep 17 00:00:00 2001 From: spav Date: Wed, 11 Jan 2023 19:58:42 +0100 Subject: [PATCH 3/5] Fix tests for allowing duplicates in extract_links --- tests/sample_data/link_extractor/linkextractor.html | 1 + tests/test_http_response.py | 1 + tests/test_linkextractors.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html index e3a2a4145..29075602d 100644 --- a/tests/sample_data/link_extractor/linkextractor.html +++ b/tests/sample_data/link_extractor/linkextractor.html @@ -13,6 +13,7 @@ sample 3 text sample 3 repetition + sample 3 repetition sample 3 repetition with fragment inner tag diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 74e170ec0..07aef2ee1 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -541,6 +541,7 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/sample2.html', 'http://example.com/sample3.html', 'http://example.com/sample3.html', + 'http://example.com/sample3.html', 'http://example.com/sample3.html#foo', 'http://www.google.com/something', 'http://example.com/innertag.html' diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e28dc9bdb..6c34a96a0 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -53,6 +53,7 @@ class Base: Link(url='http://example.com/sample2.html', text='sample 2'), Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) @@ -64,6 +65,7 @@ class Base: Link(url='http://example.com/sample2.html', text='sample 2'), Link(url='http://example.com/sample3.html', text='sample 3 text'), Link(url='http://example.com/sample3.html', text='sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition'), Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') ]) From faa5bd0f6b688eaef18a6245f2d703e8fc8ff684 Mon Sep 17 00:00:00 2001 From: silviopavanetto Date: Wed, 11 Jan 2023 20:30:57 +0100 Subject: [PATCH 4/5] Update scrapy/linkextractors/lxmlhtml.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8a4175d49..f772df987 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -195,7 +195,7 @@ class LxmlLinkExtractor: Only links that match the settings passed to the ``__init__`` method of the link extractor are returned. - Duplicate links are omitted if the ``unique`` parameter is set to ``True``, + Duplicate links are omitted if the ``unique`` attribute is set to ``True``, otherwise they are returned. """ base_url = get_base_url(response) From 36b89a4b20f12a84123930664eab888dfdf3dd8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 12 Jan 2023 08:36:31 +0100 Subject: [PATCH 5/5] Remove trailing whitespace --- scrapy/linkextractors/lxmlhtml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f772df987..3f90ed84a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -195,7 +195,7 @@ class LxmlLinkExtractor: Only links that match the settings passed to the ``__init__`` method of the link extractor are returned. - Duplicate links are omitted if the ``unique`` attribute is set to ``True``, + Duplicate links are omitted if the ``unique`` attribute is set to ``True``, otherwise they are returned. """ base_url = get_base_url(response)