RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some Link extractor improvements: 

* added support for ignoring common file extensions that are not followed if
  they occur in links
* fixed link extractor documentation issues
* slighly improved performance of applying filters
* added link to link extractors doc from documentation index
This commit is contained in:
Pablo Hoffman 2011-05-18 12:32:34 -03:00
parent 495152bd50
commit cd85c12c33
6 changed files with 80 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/index.rst
View File

@ -64,6 +64,7 @@ Basic concepts
topics/shell
topics/item-pipeline
topics/feed-exports
topics/link-extractors
:doc:`topics/commands`
Learn about the command-line tool used to manage your Scrapy project.
@ -89,6 +90,8 @@ Basic concepts
:doc:`topics/feed-exports`
Output your scraped data using different formats and storages.
:doc:`topics/link-extractors`
Convenient classes to extract links to follow from pages.
Built-in services
=================

15
docs/topics/link-extractors.rst
View File

@ -42,7 +42,7 @@ All available link extractors classes bundled with Scrapy are provided in the
SgmlLinkExtractor
-----------------
.. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None)
.. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None)
The SgmlLinkExtractor extends the base :class:`BaseSgmlLinkExtractor` by
providing additional filters that you can specify to extract links,
@ -59,15 +59,21 @@ SgmlLinkExtractor
that the (absolute) urls must match in order to be excluded (ie. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type allow: a regular expression (or list of)
:type deny: a regular expression (or list of)
:param allow_domains: a single value or a list of string containing
domains which will be considered for extracting the links
:type allow: str or list
:type allow_domains: str or list
:param deny_domains: a single value or a list of strings containing
domains which won't be considered for extracting the links
:type allow: str or list
:type deny_domains: str or list
:param deny_extensions: a list of extensions that should be ignored when
extracting links. If not given, it will default to the
``IGNORED_EXTENSIONS`` list defined in the `scrapy.linkextractor`_
module.
:type deny_extensions: list
:param restrict_xpaths: is a XPath (or list of XPath's) which defines
regions inside the response where links should be extracted from.
@ -145,3 +151,4 @@ BaseSgmlLinkExtractor
:type process_value: callable
.. _scrapy.linkextractor: http://dev.scrapy.org/browser/scrapy/linkextractor.py

43
scrapy/contrib/linkextractors/sgml.py
View File

@ -3,14 +3,16 @@ SGMLParser-based Link extractors
"""
import re
from urlparse import urlparse
from w3lib.url import safe_url_string, urljoin_rfc
from scrapy.selector import HtmlXPathSelector
from scrapy.link import Link
from scrapy.linkextractor import IGNORED_EXTENSIONS
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
from scrapy.utils.response import get_base_url
class BaseSgmlLinkExtractor(FixedSGMLParser):
@ -91,13 +93,17 @@ _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'fil
class SgmlLinkExtractor(BaseSgmlLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None):
tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None,
deny_extensions=None):
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
self.allow_domains = set(arg_to_iter(allow_domains))
self.deny_domains = set(arg_to_iter(deny_domains))
self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
self.canonicalize = canonicalize
if deny_extensions is None:
deny_extensions = IGNORED_EXTENSIONS
self.deny_extensions = set(['.' + e for e in deny_extensions])
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
BaseSgmlLinkExtractor.__init__(self, tag=tag_func, attr=attr_func,
@ -118,24 +124,27 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
return links
def _process_links(self, links):
links = [link for link in links if _is_valid_url(link.url)]
if self.allow_res:
links = [link for link in links if _matches(link.url, self.allow_res)]
if self.deny_res:
links = [link for link in links if not _matches(link.url, self.deny_res)]
if self.allow_domains:
links = [link for link in links if url_is_from_any_domain(link.url, self.allow_domains)]
if self.deny_domains:
links = [link for link in links if not url_is_from_any_domain(link.url, self.deny_domains)]
if self.canonicalize:
for link in links:
link.url = canonicalize_url(link.url)
links = [x for x in links if self._link_allowed(x)]
links = BaseSgmlLinkExtractor._process_links(self, links)
return links
def _link_allowed(self, link):
parsed_url = urlparse(link.url)
allowed = _is_valid_url(link.url)
if self.allow_res:
allowed &= _matches(link.url, self.allow_res)
if self.deny_res:
allowed &= not _matches(link.url, self.deny_res)
if self.allow_domains:
allowed &= url_is_from_any_domain(parsed_url, self.allow_domains)
if self.deny_domains:
allowed &= not url_is_from_any_domain(parsed_url, self.deny_domains)
if self.deny_extensions:
allowed &= not url_has_any_extension(parsed_url, self.deny_extensions)
if allowed and self.canonicalize:
link.url = canonicalize_url(parsed_url)
return allowed
def matches(self, url):
if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
return False

21
scrapy/linkextractor.py Normal file
View File

@ -0,0 +1,21 @@
"""
Common code and definitions used by Link extractors (located in
scrapy.contrib.linkextractor).
"""
# common file extensions that are not followed if they occur in links
IGNORED_EXTENSIONS = [
# images
'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg',
# audio
'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff',
# video
'3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
'm4a',
# other
'css', 'pdf', 'doc', 'exe', 'bin', 'rss', 'zip', 'rar',
]

11
scrapy/tests/test_contrib_linkextractors.py
View File

@ -135,10 +135,6 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
self.assertEqual([link for link in lx.extract_links(self.response)],
[ Link(url='http://www.google.com/something', text=u'') ])
lx = SgmlLinkExtractor(tags=('img', ), attrs=('src', ))
self.assertEqual([link for link in lx.extract_links(self.response)],
[ Link(url='http://example.com/sample2.jpg', text=u'') ])
def test_extraction_using_single_values(self):
'''Test the extractor's behaviour among different situations'''
@ -211,6 +207,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
def test_deny_extensions(self):
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
response = HtmlResponse("http://example.org/", body=html)
lx = SgmlLinkExtractor()
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/page.html', text=u'asd')])
def test_process_value(self):
"""Test restrict_xpaths with encodings"""
html = """

18
scrapy/utils/url.py
View File

@ -6,11 +6,8 @@ Some of the functions that used to be imported from this module have been moved
to the w3lib.url module. Always import those from there instead.
"""
import os
import re
import urlparse
import urllib
import posixpath
import cgi
from w3lib.url import *
@ -18,7 +15,7 @@ from scrapy.utils.python import unicode_to_str
def url_is_from_any_domain(url, domains):
"""Return True if the url belongs to any of the given domains"""
host = urlparse.urlparse(url).hostname
host = parse_url(url).hostname
if host:
return any(((host == d) or (host.endswith('.%s' % d)) for d in domains))
@ -30,6 +27,9 @@ def url_is_from_spider(url, spider):
return url_is_from_any_domain(url, [spider.name] + \
getattr(spider, 'allowed_domains', []))
def url_has_any_extension(url, extensions):
return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \
encoding=None):
"""Canonicalize the given url by applying the following procedures:
@ -48,11 +48,17 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \
For examples see the tests in scrapy.tests.test_utils_url
"""
url = unicode_to_str(url, encoding)
scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
scheme, netloc, path, params, query, fragment = parse_url(url)
keyvals = cgi.parse_qsl(query, keep_blank_values)
keyvals.sort()
query = urllib.urlencode(keyvals)
path = safe_url_string(urllib.unquote(path))
fragment = '' if not keep_fragments else fragment
return urlparse.urlunparse((scheme, netloc.lower(), path, params, query, fragment))
def parse_url(url, encoding=None):
"""Return urlparsed url from the given argument (which could be an already
parsed url)
"""
return url if isinstance(url, urlparse.ParseResult) else \
urlparse.urlparse(unicode_to_str(url, encoding))