diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..cda6ddc7e 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -41,7 +41,8 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \ + 'file', 'ftp'} class FilteringLinkExtractor(object): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 1d7c4f311..903032b52 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -451,6 +451,17 @@ class Base: Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), ]) + def test_ftp_links(self): + body = b""" + +
An Item
+ """ + response = HtmlResponse("http://www.example.com/index.html", body=body, encoding='utf8') + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), + ]) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor @@ -471,4 +482,3 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() -