diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 02b3992ba..3f9f24efd 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -130,14 +130,14 @@ For more information about XPath see the `XPath reference`_.
 Finally, here's the spider code::
 
     from scrapy.contrib.spiders import CrawlSpider, Rule
-    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+    from scrapy.contrib.linkextractors import LinkExtractor
 
     class MininovaSpider(CrawlSpider):
 
         name = 'mininova'
         allowed_domains = ['mininova.org']
         start_urls = ['http://www.mininova.org/today']
-        rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
+        rules = [Rule(LinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
 
         def parse_torrent(self, response):
             torrent = TorrentItem()
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
index fcfb781d7..7dd5a2b76 100644
--- a/docs/topics/firebug.rst
+++ b/docs/topics/firebug.rst
@@ -69,7 +69,7 @@ those links. For example, the following one::
 
 So, based on that regular expression we can create the first crawling rule::
 
-    Rule(SgmlLinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$', ),
+    Rule(LinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$', ),
         'parse_category',
         follow=True,
     ),
@@ -81,7 +81,7 @@ process and extract data from those pages.
 
 This is how the spider would look so far::
 
-   from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+   from scrapy.contrib.linkextractors import LinkExtractor
    from scrapy.contrib.spiders import CrawlSpider, Rule
 
    class GoogleDirectorySpider(CrawlSpider):
@@ -90,7 +90,7 @@ This is how the spider would look so far::
        start_urls = ['http://directory.google.com/']
 
        rules = (
-           Rule(SgmlLinkExtractor(allow='directory\.google\.com/[A-Z][a-zA-Z_/]+$'),
+           Rule(LinkExtractor(allow='directory\.google\.com/[A-Z][a-zA-Z_/]+$'),
                'parse_category', follow=True,
            ),
        )
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 79af6db35..13e18e574 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -16,7 +16,7 @@ The only public method that every LinkExtractor has is ``extract_links``,
 which receives a :class:`~scrapy.http.Response` object and returns a list
 of :class:`scrapy.link.Link` objects. Link Extractors are meant to be instantiated once and their
 ``extract_links`` method called several times with different responses, to
-extract links to follow. 
+extract links to follow.
 
 Link extractors are used in the :class:`~scrapy.contrib.spiders.CrawlSpider`
 class (available in Scrapy), through a set of rules, but you can also use it in
@@ -36,16 +36,88 @@ Built-in link extractors reference
 All available link extractors classes bundled with Scrapy are provided in the
 :mod:`scrapy.contrib.linkextractors` module.
 
+If you don't know what link extractor to choose, just use the default which is
+the same than LxmlLinkExtractor (see below)::
+
+    from scrapy.contrib.linkextractors import LinkExtractor
+
+
+.. module:: scrapy.contrib.linkextractors.lxmlhtml
+   :synopsis: lxml's HTMLParser-based link extractors
+
 .. module:: scrapy.contrib.linkextractors.sgml
    :synopsis: SGMLParser-based link extractors
 
+LxmlLinkExtractor
+-----------------
+
+.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
+
+    LxmlLinkExtractor provides the same API as :class:`SgmlLinkExtractor`,
+    and therefore has the same handy filtering options as constructor parameters,
+    but the implementation underneath uses lxml's more robust ``HTMLParser``
+    to parse HTML documents:
+
+    :param allow: a single regular expression (or list of regular expressions)
+        that the (absolute) urls must match in order to be extracted. If not
+        given (or empty), it will match all links.
+    :type allow: a regular expression (or list of)
+
+    :param deny: a single regular expression (or list of regular expressions)
+        that the (absolute) urls must match in order to be excluded (ie. not
+        extracted). It has precedence over the ``allow`` parameter. If not
+        given (or empty) it won't exclude any links.
+    :type deny: a regular expression (or list of)
+
+    :param allow_domains: a single value or a list of string containing
+        domains which will be considered for extracting the links
+    :type allow_domains: str or list
+
+    :param deny_domains: a single value or a list of strings containing
+        domains which won't be considered for extracting the links
+    :type deny_domains: str or list
+
+    :param deny_extensions: a single value or list of strings containing
+        extensions that should be ignored when extracting links.
+        If not given, it will default to the
+        ``IGNORED_EXTENSIONS`` list defined in the `scrapy.linkextractor`_
+        module.
+    :type deny_extensions: list
+
+    :param restrict_xpaths: is a XPath (or list of XPath's) which defines
+        regions inside the response where links should be extracted from.
+        If given, only the text selected by those XPath will be scanned for
+        links. See examples below.
+    :type restrict_xpaths: str or list
+
+    :param tags: a tag or a list of tags to consider when extracting links.
+        Defaults to ``('a', 'area')``.
+    :type tags: str or list
+
+    :param attrs: an attribute or list of attributes which should be considered when looking
+        for links to extract (only for those tags specified in the ``tags``
+        parameter). Defaults to ``('href',)``
+    :type attrs: list
+
+    :param canonicalize: canonicalize each extracted url (using
+        scrapy.utils.url.canonicalize_url). Defaults to ``True``.
+    :type canonicalize: boolean
+
+    :param unique: whether duplicate filtering should be applied to extracted
+        links.
+    :type unique: boolean
+
+    :param process_value: see ``process_value`` argument of
+        :class:`BaseSgmlLinkExtractor` class constructor
+    :type process_value: callable
+
 SgmlLinkExtractor
 -----------------
 
 .. class:: SgmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None)
 
-    The SgmlLinkExtractor extends the base :class:`BaseSgmlLinkExtractor` by
-    providing additional filters that you can specify to extract links,
+    The SgmlLinkExtractor is built upon the  base :class:`BaseSgmlLinkExtractor`
+    and provides additional filters that you can specify to extract links,
     including regular expressions patterns that the links must match to be
     extracted. All those filters are configured through these constructor
     parameters:
@@ -70,14 +142,14 @@ SgmlLinkExtractor
     :type deny_domains: str or list
 
     :param deny_extensions: a single value or list of strings containing
-        extensions that should be ignored when extracting links. 
+        extensions that should be ignored when extracting links.
         If not given, it will default to the
         ``IGNORED_EXTENSIONS`` list defined in the `scrapy.linkextractor`_
         module.
     :type deny_extensions: list
 
     :param restrict_xpaths: is a XPath (or list of XPath's) which defines
-        regions inside the response where links should be extracted from. 
+        regions inside the response where links should be extracted from.
         If given, only the text selected by those XPath will be scanned for
         links. See examples below.
     :type restrict_xpaths: str or list
@@ -110,7 +182,7 @@ BaseSgmlLinkExtractor
 
     The purpose of this Link Extractor is only to serve as a base class for the
     :class:`SgmlLinkExtractor`. You should use that one instead.
-    
+
     The constructor arguments are:
 
     :param tag: either a string (with the name of a tag) or a function that
@@ -140,15 +212,15 @@ BaseSgmlLinkExtractor
         For example, to extract links from this code::
 
             <a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
-        
+
         .. highlight:: python
 
         You can use the following function in ``process_value``::
-        
+
             def process_value(value):
                 m = re.search("javascript:goToPage\('(.*?)'", value)
                 if m:
-                    return m.group(1) 
+                    return m.group(1)
 
     :type process_value: callable
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d305ef281..73c34e75f 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -318,7 +318,7 @@ Let's now take a look at an example CrawlSpider with rules::
 
     import scrapy
     from scrapy.contrib.spiders import CrawlSpider, Rule
-    from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+    from scrapy.contrib.linkextractors import LinkExtractor
 
     class MySpider(CrawlSpider):
         name = 'example.com'
@@ -328,10 +328,10 @@ Let's now take a look at an example CrawlSpider with rules::
         rules = (
             # Extract links matching 'category.php' (but not matching 'subsection.php')
             # and follow links from them (since no callback means follow=True by default).
-            Rule(SgmlLinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
+            Rule(LinkExtractor(allow=('category\.php', ), deny=('subsection\.php', ))),
 
             # Extract links matching 'item.php' and parse them with the spider's method parse_item
-            Rule(SgmlLinkExtractor(allow=('item\.php', )), callback='parse_item'),
+            Rule(LinkExtractor(allow=('item\.php', )), callback='parse_item'),
         )
 
         def parse_item(self, response):
diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py
index 47f6e3148..48b9c757a 100644
--- a/scrapy/contrib/linkextractors/__init__.py
+++ b/scrapy/contrib/linkextractors/__init__.py
@@ -1,7 +1,8 @@
 """
 scrapy.contrib.linkextractors
 
-This package contains a collection of Link Extractors. 
+This package contains a collection of Link Extractors.
 
 For more info see docs/topics/link-extractors.rst
 """
+from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
index 826c76227..fb5ac2db3 100644
--- a/scrapy/contrib/linkextractors/lxmlhtml.py
+++ b/scrapy/contrib/linkextractors/lxmlhtml.py
@@ -2,10 +2,30 @@
 Link extractor based on lxml.html
 """
 
-import lxml.html
+import re
+from urlparse import urlparse, urljoin
 
+import lxml.etree as etree
+
+from scrapy.selector import Selector
 from scrapy.link import Link
-from scrapy.utils.python import unique as unique_list
+from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.python import unique as unique_list, str_to_unicode
+from scrapy.linkextractor import FilteringLinkExtractor
+from scrapy.utils.response import get_base_url
+
+
+# from lxml/src/lxml/html/__init__.py
+XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
+
+_collect_string_content = etree.XPath("string()")
+
+def _nons(tag):
+    if isinstance(tag, basestring):
+        if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
+            return tag.split('}')[-1]
+    return tag
+
 
 class LxmlParserLinkExtractor(object):
     def __init__(self, tag="a", attr="href", process=None, unique=False):
@@ -16,14 +36,31 @@ class LxmlParserLinkExtractor(object):
 
         self.links = []
 
-    def _extract_links(self, response_text, response_url):
-        html = lxml.html.fromstring(response_text)
-        html.make_links_absolute(response_url)
-        for e, a, l, p in html.iterlinks():
-            if self.scan_tag(e.tag):
-                if self.scan_attr(a):
-                    link = Link(self.process_attr(l), text=e.text)
-                    self.links.append(link)
+    def _iter_links(self, document):
+        for el in document.iter(etree.Element):
+            tag = _nons(el.tag)
+            if not self.scan_tag(el.tag):
+                continue
+            attribs = el.attrib
+            for attrib in attribs:
+                yield (el, attrib, attribs[attrib])
+
+    def _extract_links(self, selector, response_url, response_encoding, base_url):
+        # hacky way to get the underlying lxml parsed document
+        for el, attr, attr_val in self._iter_links(selector._root):
+            if self.scan_tag(el.tag) and self.scan_attr(attr):
+                # pseudo _root.make_links_absolute(base_url)
+                attr_val = urljoin(base_url, attr_val)
+                url = self.process_attr(attr_val)
+                if url is None:
+                    continue
+                if isinstance(url, unicode):
+                    url = url.encode(response_encoding)
+                # to fix relative links after process_value
+                url = urljoin(response_url, url)
+                link = Link(url, _collect_string_content(el) or u'',
+                    nofollow=True if el.get('rel') == 'nofollow' else False)
+                self.links.append(link)
 
         links = unique_list(self.links, key=lambda link: link.url) \
                 if self.unique else self.links
@@ -31,6 +68,46 @@ class LxmlParserLinkExtractor(object):
         return links
 
     def extract_links(self, response):
-        return self._extract_links(response.body, response.url)
+        html = Selector(response)
+        base_url = get_base_url(response)
+        return self._extract_links(html, response.url, response.encoding, base_url)
+
+    def _process_links(self, links):
+        """ Normalize and filter extracted links
+
+        The subclass should override it if neccessary
+        """
+        links = unique_list(links, key=lambda link: link.url) if self.unique else links
+        return links
 
 
+class LxmlLinkExtractor(FilteringLinkExtractor):
+
+    def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
+                 tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
+                 deny_extensions=None):
+        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
+        tag_func = lambda x: x in tags
+        attr_func = lambda x: x in attrs
+        lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
+            unique=unique, process=process_value)
+
+        super(LxmlLinkExtractor, self).__init__(lx, allow, deny,
+            allow_domains, deny_domains, restrict_xpaths, canonicalize,
+            deny_extensions)
+
+    def extract_links(self, response):
+        html = Selector(response)
+        base_url = get_base_url(response)
+        if self.restrict_xpaths:
+            docs = [subdoc
+                    for x in self.restrict_xpaths
+                    for subdoc in html.xpath(x)]
+        else:
+            docs = [html]
+        all_links = []
+        for doc in docs:
+            links = self._extract_links(doc, response.url, response.encoding, base_url)
+            all_links.extend(self._process_links(links))
+        return unique_list(all_links)
+
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
index a0b51b218..9adaad9e5 100644
--- a/scrapy/contrib/linkextractors/sgml.py
+++ b/scrapy/contrib/linkextractors/sgml.py
@@ -6,10 +6,9 @@ from urlparse import urlparse, urljoin
 from w3lib.url import safe_url_string
 from scrapy.selector import Selector
 from scrapy.link import Link
-from scrapy.linkextractor import IGNORED_EXTENSIONS
+from scrapy.linkextractor import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode
-from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
 from scrapy.utils.response import get_base_url
 
 
@@ -86,33 +85,23 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
         it doesn't contain any patterns"""
         return True
 
-_re_type = type(re.compile("", 0))
 
-_matches = lambda url, regexs: any((r.search(url) for r in regexs))
-_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])
-
-
-class SgmlLinkExtractor(BaseSgmlLinkExtractor):
+class SgmlLinkExtractor(FilteringLinkExtractor):
 
     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
                  tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None,
                  deny_extensions=None):
-        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
-        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
-        self.allow_domains = set(arg_to_iter(allow_domains))
-        self.deny_domains = set(arg_to_iter(deny_domains))
-        self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
-        self.canonicalize = canonicalize
-        if deny_extensions is None:
-            deny_extensions = IGNORED_EXTENSIONS
-        self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
-        tag_func = lambda x: x in arg_to_iter(tags)
-        attr_func = lambda x: x in arg_to_iter(attrs)
-        BaseSgmlLinkExtractor.__init__(self,
-                                       tag=tag_func,
-                                       attr=attr_func,
-                                       unique=unique,
-                                       process_value=process_value)
+        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
+        tag_func = lambda x: x in tags
+        attr_func = lambda x: x in attrs
+        lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
+            unique=unique, process_value=process_value)
+        super(SgmlLinkExtractor, self).__init__(lx, allow, deny,
+            allow_domains, deny_domains, restrict_xpaths, canonicalize,
+            deny_extensions)
+
+        # FIXME: was added to fix a RegexLinkExtractor testcase
+        self.base_url = None
 
     def extract_links(self, response):
         base_url = None
@@ -129,35 +118,3 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
         links = self._extract_links(body, response.url, response.encoding, base_url)
         links = self._process_links(links)
         return links
-
-    def _process_links(self, links):
-        links = [x for x in links if self._link_allowed(x)]
-        links = BaseSgmlLinkExtractor._process_links(self, links)
-        return links
-
-    def _link_allowed(self, link):
-        parsed_url = urlparse(link.url)
-        allowed = _is_valid_url(link.url)
-        if self.allow_res:
-            allowed &= _matches(link.url, self.allow_res)
-        if self.deny_res:
-            allowed &= not _matches(link.url, self.deny_res)
-        if self.allow_domains:
-            allowed &= url_is_from_any_domain(parsed_url, self.allow_domains)
-        if self.deny_domains:
-            allowed &= not url_is_from_any_domain(parsed_url, self.deny_domains)
-        if self.deny_extensions:
-            allowed &= not url_has_any_extension(parsed_url, self.deny_extensions)
-        if allowed and self.canonicalize:
-            link.url = canonicalize_url(parsed_url)
-        return allowed
-
-    def matches(self, url):
-        if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
-            return False
-        if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
-            return False
-
-        allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
-        denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
-        return any(allowed) and not any(denied)
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
index 946c82062..d9c85b50a 100644
--- a/scrapy/linkextractor.py
+++ b/scrapy/linkextractor.py
@@ -2,6 +2,13 @@
 Common code and definitions used by Link extractors (located in
 scrapy.contrib.linkextractor).
 """
+import re
+from urlparse import urlparse
+
+from scrapy.utils.url import url_is_from_any_domain
+from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
+from scrapy.utils.misc import arg_to_iter
+
 
 # common file extensions that are not followed if they occur in links
 IGNORED_EXTENSIONS = [
@@ -22,3 +29,66 @@ IGNORED_EXTENSIONS = [
     # other
     'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar',
 ]
+
+
+_re_type = type(re.compile("", 0))
+_matches = lambda url, regexs: any((r.search(url) for r in regexs))
+_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])
+
+
+class FilteringLinkExtractor(object):
+
+    def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
+                 restrict_xpaths, canonicalize, deny_extensions):
+
+        self.link_extractor = link_extractor
+
+        self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
+        self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)]
+
+        self.allow_domains = set(arg_to_iter(allow_domains))
+        self.deny_domains = set(arg_to_iter(deny_domains))
+
+        self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
+        self.canonicalize = canonicalize
+        if deny_extensions is None:
+            deny_extensions = IGNORED_EXTENSIONS
+        self.deny_extensions = set(['.' + e for e in arg_to_iter(deny_extensions)])
+
+    def _link_allowed(self, link):
+        if not _is_valid_url(link.url):
+            return False
+        if self.allow_res and not _matches(link.url, self.allow_res):
+            return False
+        if self.deny_res and _matches(link.url, self.deny_res):
+            return False
+        parsed_url = urlparse(link.url)
+        if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains):
+            return False
+        if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains):
+            return False
+        if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
+            return False
+        return True
+
+    def matches(self, url):
+
+        if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
+            return False
+        if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
+            return False
+
+        allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
+        denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
+        return any(allowed) and not any(denied)
+
+    def _process_links(self, links):
+        links = [x for x in links if self._link_allowed(x)]
+        if self.canonicalize:
+            for link in links:
+                link.url = canonicalize_url(urlparse(link.url))
+        links = self.link_extractor._process_links(links)
+        return links
+
+    def _extract_links(self, *args, **kwargs):
+        return self.link_extractor._extract_links(*args, **kwargs)
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 4c30a69e8..0702f205f 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -1,5 +1,5 @@
 import scrapy
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.contrib.linkextractors import LinkExtractor
 from scrapy.contrib.spiders import CrawlSpider, Rule
 
 from $project_name.items import ${ProjectName}Item
@@ -11,7 +11,7 @@ class $classname(CrawlSpider):
     start_urls = ['http://www.$domain/']
 
     rules = (
-        Rule(SgmlLinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
+        Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True),
     )
 
     def parse_item(self, response):
diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py
index 3eac95d58..7dc8540d9 100644
--- a/scrapy/tests/spiders.py
+++ b/scrapy/tests/spiders.py
@@ -8,7 +8,7 @@ from urllib import urlencode
 from scrapy.spider import Spider
 from scrapy.http import Request
 from scrapy.item import Item
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.contrib.linkextractors import LinkExtractor
 
 
 class MetaSpider(Spider):
@@ -26,7 +26,7 @@ class MetaSpider(Spider):
 class FollowAllSpider(MetaSpider):
 
     name = 'follow'
-    link_extractor = SgmlLinkExtractor()
+    link_extractor = LinkExtractor()
 
     def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
         super(FollowAllSpider, self).__init__(*args, **kwargs)
diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py
index 72c607096..27d73e271 100644
--- a/scrapy/tests/test_contrib_linkextractors.py
+++ b/scrapy/tests/test_contrib_linkextractors.py
@@ -5,6 +5,7 @@ from scrapy.http import HtmlResponse
 from scrapy.link import Link
 from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
+from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor
 from scrapy.tests import get_testdata
 
 
@@ -105,19 +106,21 @@ class LinkExtractorTestCase(unittest.TestCase):
 
 
 class SgmlLinkExtractorTestCase(unittest.TestCase):
+    extractor_cls = SgmlLinkExtractor
+
     def setUp(self):
         body = get_testdata('link_extractor', 'sgml_linkextractor.html')
         self.response = HtmlResponse(url='http://example.com/index', body=body)
 
     def test_urls_type(self):
         '''Test that the resulting urls are regular strings and not a unicode objects'''
-        lx = SgmlLinkExtractor()
+        lx = self.extractor_cls()
         self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response)))
 
     def test_extraction(self):
         '''Test the extractor's behaviour among different situations'''
 
-        lx = SgmlLinkExtractor()
+        lx = self.extractor_cls()
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -126,14 +129,14 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/innertag.html', text=u'inner tag'),
         ])
 
-        lx = SgmlLinkExtractor(allow=('sample', ))
+        lx = self.extractor_cls(allow=('sample', ))
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
             Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
         ])
 
-        lx = SgmlLinkExtractor(allow=('sample', ), unique=False)
+        lx = self.extractor_cls(allow=('sample', ), unique=False)
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -141,20 +144,20 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
         ])
 
-        lx = SgmlLinkExtractor(allow=('sample', ))
+        lx = self.extractor_cls(allow=('sample', ))
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
             Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
         ])
 
-        lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', ))
+        lx = self.extractor_cls(allow=('sample', ), deny=('3', ))
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
         ])
 
-        lx = SgmlLinkExtractor(allow_domains=('google.com', ))
+        lx = self.extractor_cls(allow_domains=('google.com', ))
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://www.google.com/something', text=u''),
         ])
@@ -162,50 +165,78 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
     def test_extraction_using_single_values(self):
         '''Test the extractor's behaviour among different situations'''
 
-        lx = SgmlLinkExtractor(allow='sample')
+        lx = self.extractor_cls(allow='sample')
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
             Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
         ])
 
-        lx = SgmlLinkExtractor(allow='sample', deny='3')
+        lx = self.extractor_cls(allow='sample', deny='3')
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
         ])
 
-        lx = SgmlLinkExtractor(allow_domains='google.com')
+        lx = self.extractor_cls(allow_domains='google.com')
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://www.google.com/something', text=u''),
         ])
 
-        lx = SgmlLinkExtractor(deny_domains='example.com')
+        lx = self.extractor_cls(deny_domains='example.com')
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://www.google.com/something', text=u''),
         ])
 
+    def test_nofollow(self):
+        '''Test the extractor's behaviour for links with rel="nofollow"'''
+
+        html = """<html><head><title>Page title<title>
+        <body>
+        <div class='links'>
+        <p><a href="/about.html">About us</a></p>
+        </div>
+        <div>
+        <p><a href="/follow.html">Follow this link</a></p>
+        </div>
+        <div>
+        <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p>
+        </div>
+        <div>
+        <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p>
+        </div>
+        </body></html>"""
+        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
+
+        lx = self.extractor_cls()
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.org/about.html', text=u'About us'),
+            Link(url='http://example.org/follow.html', text=u'Follow this link'),
+            Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True),
+            Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'),
+        ])
+
     def test_matches(self):
         url1 = 'http://lotsofstuff.com/stuff1/index'
         url2 = 'http://evenmorestuff.com/uglystuff/index'
 
-        lx = SgmlLinkExtractor(allow=(r'stuff1', ))
+        lx = self.extractor_cls(allow=(r'stuff1', ))
         self.assertEqual(lx.matches(url1), True)
         self.assertEqual(lx.matches(url2), False)
 
-        lx = SgmlLinkExtractor(deny=(r'uglystuff', ))
+        lx = self.extractor_cls(deny=(r'uglystuff', ))
         self.assertEqual(lx.matches(url1), True)
         self.assertEqual(lx.matches(url2), False)
 
-        lx = SgmlLinkExtractor(allow_domains=('evenmorestuff.com', ))
+        lx = self.extractor_cls(allow_domains=('evenmorestuff.com', ))
         self.assertEqual(lx.matches(url1), False)
         self.assertEqual(lx.matches(url2), True)
 
-        lx = SgmlLinkExtractor(deny_domains=('lotsofstuff.com', ))
+        lx = self.extractor_cls(deny_domains=('lotsofstuff.com', ))
         self.assertEqual(lx.matches(url1), False)
         self.assertEqual(lx.matches(url2), True)
 
-        lx = SgmlLinkExtractor(allow=('blah1',), deny=('blah2',),
+        lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
                                allow_domains=('blah1.com',),
                                deny_domains=('blah2.com',))
         self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
@@ -214,7 +245,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         self.assertEqual(lx.matches('http://blah2.com/blah2'), False)
 
     def test_restrict_xpaths(self):
-        lx = SgmlLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', ))
+        lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', ))
         self.assertEqual([link for link in lx.extract_links(self.response)], [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -233,7 +264,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         </body></html>"""
         response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
 
-        lx = SgmlLinkExtractor(restrict_xpaths="//div[@class='links']")
+        lx = self.extractor_cls(restrict_xpaths="//div[@class='links']")
         self.assertEqual(lx.extract_links(response),
                          [Link(url='http://example.org/about.html', text=u'About us\xa3')])
 
@@ -248,7 +279,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         """html entities cause SGMLParser to call handle_data hook twice"""
         body = """<html><body><div><a href="/foo">&gt;\xbe\xa9&lt;\xb6\xab</a></body></html>"""
         response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
-        lx = SgmlLinkExtractor(restrict_xpaths="//div")
+        lx = self.extractor_cls(restrict_xpaths="//div")
         self.assertEqual(lx.extract_links(response),
                          [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
                                fragment='', nofollow=False)])
@@ -256,7 +287,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
     def test_encoded_url(self):
         body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
         response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
-        lx = SgmlLinkExtractor()
+        lx = self.extractor_cls()
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
         ])
@@ -264,7 +295,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
     def test_encoded_url_in_restricted_xpath(self):
         body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
         response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')
-        lx = SgmlLinkExtractor(restrict_xpaths="//div")
+        lx = self.extractor_cls(restrict_xpaths="//div")
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False),
         ])
@@ -272,7 +303,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
     def test_deny_extensions(self):
         html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
         response = HtmlResponse("http://example.org/", body=html)
-        lx = SgmlLinkExtractor()
+        lx = self.extractor_cls()
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://example.org/page.html', text=u'asd'),
         ])
@@ -295,7 +326,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             if m:
                 return m.group(1)
 
-        lx = SgmlLinkExtractor(process_value=process_value)
+        lx = self.extractor_cls(process_value=process_value)
         self.assertEqual(lx.extract_links(response),
                          [Link(url='http://example.org/other/page.html', text='Link text')])
 
@@ -304,13 +335,12 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
         <body><p><a href="item/12.html">Item 12</a></p>
         </body></html>"""
         response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-        lx = SgmlLinkExtractor(restrict_xpaths="//p")
+        lx = self.extractor_cls(restrict_xpaths="//p")
         self.assertEqual(lx.extract_links(response),
                          [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
 
-
     def test_attrs(self):
-        lx = SgmlLinkExtractor(attrs="href")
+        lx = self.extractor_cls(attrs="href")
         self.assertEqual(lx.extract_links(self.response), [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -319,7 +349,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/innertag.html', text=u'inner tag'),
         ])
 
-        lx = SgmlLinkExtractor(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
+        lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
         self.assertEqual(lx.extract_links(self.response), [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
@@ -329,7 +359,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/innertag.html', text=u'inner tag'),
         ])
 
-        lx = SgmlLinkExtractor(attrs=None)
+        lx = self.extractor_cls(attrs=None)
         self.assertEqual(lx.extract_links(self.response), [])
 
         html = """<html><area href="sample1.html"></area><a ref="sample2.html">sample text 2</a></html>"""
@@ -339,36 +369,61 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
             Link(url='http://example.com/sample1.html', text=u''),
         ])
 
-
     def test_tags(self):
         html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>"""
         response = HtmlResponse("http://example.com/index.html", body=html)
 
-        lx = SgmlLinkExtractor(tags=None)
+        lx = self.extractor_cls(tags=None)
         self.assertEqual(lx.extract_links(response), [])
 
-        lx = SgmlLinkExtractor()
+        lx = self.extractor_cls()
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://example.com/sample1.html', text=u''),
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
         ])
 
-        lx = SgmlLinkExtractor(tags="area")
+        lx = self.extractor_cls(tags="area")
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://example.com/sample1.html', text=u''),
         ])
 
-        lx = SgmlLinkExtractor(tags="a")
+        lx = self.extractor_cls(tags="a")
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
         ])
 
-        lx = SgmlLinkExtractor(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
+        lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=())
         self.assertEqual(lx.extract_links(response), [
             Link(url='http://example.com/sample2.html', text=u'sample 2'),
             Link(url='http://example.com/sample2.jpg', text=u''),
         ])
 
+    def test_tags_attrs(self):
+        html = """
+        <html><body>
+        <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div>
+        <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div>
+        </body></html>
+        """
+        response = HtmlResponse("http://example.com/index.html", body=html)
+
+        lx = self.extractor_cls(tags='div', attrs='data-url')
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
+            Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+        ])
+
+        lx = self.extractor_cls(tags=('div',), attrs=('data-url',))
+        self.assertEqual(lx.extract_links(response), [
+            Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False),
+            Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False)
+        ])
+
+
+
+class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase):
+    extractor_cls = LxmlLinkExtractor
+
 
 class HtmlParserLinkExtractorTestCase(unittest.TestCase):
 
diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py
index 49a700073..cabdff9bb 100644
--- a/scrapy/tests/test_engine.py
+++ b/scrapy/tests/test_engine.py
@@ -23,7 +23,7 @@ from scrapy.xlib.pydispatch import dispatcher
 from scrapy.tests import tests_datadir
 from scrapy.spider import Spider
 from scrapy.item import Item, Field
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.contrib.linkextractors import LinkExtractor
 from scrapy.http import Request
 from scrapy.utils.signal import disconnect_all
 
@@ -41,7 +41,7 @@ class TestSpider(Spider):
     price_re = re.compile(">Price: \$(.*?)<", re.M)
 
     def parse(self, response):
-        xlink = SgmlLinkExtractor()
+        xlink = LinkExtractor()
         itemre = re.compile(self.itemurl_re)
         for link in xlink.extract_links(response):
             if itemre.search(link.url):
diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py
index 52b77a2a4..8abc39628 100644
--- a/scrapy/tests/test_spider.py
+++ b/scrapy/tests/test_spider.py
@@ -11,7 +11,7 @@ from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlRespon
 from scrapy.contrib.spiders.init import InitSpider
 from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.contrib.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -124,7 +124,7 @@ class CrawlSpiderTest(SpiderTest):
             name="test"
             allowed_domains=['example.org']
             rules = (
-                Rule(SgmlLinkExtractor(), process_links="dummy_process_links"),
+                Rule(LinkExtractor(), process_links="dummy_process_links"),
             )
 
             def dummy_process_links(self, links):
@@ -150,7 +150,7 @@ class CrawlSpiderTest(SpiderTest):
             name="test"
             allowed_domains=['example.org']
             rules = (
-                Rule(SgmlLinkExtractor(), process_links="filter_process_links"),
+                Rule(LinkExtractor(), process_links="filter_process_links"),
             )
             _test_regex = re.compile('nofollow')
             def filter_process_links(self, links):
@@ -174,7 +174,7 @@ class CrawlSpiderTest(SpiderTest):
             name="test"
             allowed_domains=['example.org']
             rules = (
-                Rule(SgmlLinkExtractor(), process_links="dummy_process_links"),
+                Rule(LinkExtractor(), process_links="dummy_process_links"),
             )
 
             def dummy_process_links(self, links):