diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e90cb0223..64693f25b 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -247,6 +247,18 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
+HttpProxyMiddleware
+-------------------
+
+.. module:: scrapy.contrib.downloadermiddleware.httpproxy
+   :synopsis: Http Proxy Middleware
+
+.. class:: HttpProxyMiddleware
+
+   This middleware sets proxy to use for requests, it obeys enviroment
+   variables 'http_proxy', 'https_proxy', and 'no_proxy'
+
+
 RedirectMiddleware
 -------------------
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 79b679fad..5cd6ff5b2 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -725,6 +725,33 @@ Default: ``+2``
 Adjust redirect request priority relative to original request.
 A negative priority adjust means more priority.
 
+.. setting:: REQUEST_HANDLERS
+
+REQUEST_HANDLERS
+----------------
+
+Default: ``{}``
+
+A dict containing the request downloader handlers enabled in your project.
+See `REQUEST_HANDLERS_BASE` for example format.
+
+.. setting:: REQUEST_HANDLERS_BASE
+
+REQUEST_HANDLERS_BASE
+---------------------
+
+Default:: 
+
+    {
+        'file': 'scrapy.core.downloader.handlers.file.download_file',
+        'http': 'scrapy.core.downloader.handlers.http.download_http',
+        'https': 'scrapy.core.downloader.handlers.http.download_http',
+    }
+
+A dict containing the request download handlers enabled by default in Scrapy.
+You should never modify this setting in your project, modify
+:setting:`REQUEST_HANDLERS` instead. 
+
 .. setting:: REQUESTS_PER_DOMAIN
 
 REQUESTS_PER_DOMAIN
diff --git a/extras/build_release.sh b/extras/build_release.sh
index 57873b12e..2230685e8 100755
--- a/extras/build_release.sh
+++ b/extras/build_release.sh
@@ -6,8 +6,8 @@
 hg purge --all
 
 # build packages
-version=$(python -c "import scrapy; print scrapy.__version__")
-python setup.py sdist
+#version=$(python -c "import scrapy; print scrapy.__version__")
+#python setup.py sdist
 # FIXME: bdist_wininst doesn't work on Unix (it doesn't include the data_files)
 # To build the win32 release you need to use Windows for now.
 #python setup.py bdist_wininst -t "Scrapy $version" -p "win32"
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index bee059b0b..5e206de2f 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -3,8 +3,8 @@ Scrapy - a screen scraping framework written in Python
 """
 
 # IMPORTANT: remember to also update the version in docs/conf.py
-version_info = (0, 7, 0, 'final', 0)
-__version__ = "0.7"
+version_info = (0, 8, 0, '', 0)
+__version__ = "0.8-dev"
 
 import sys, os
 
diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py
index 17b4f7db0..a4d92faa9 100644
--- a/scrapy/conf/default_settings.py
+++ b/scrapy/conf/default_settings.py
@@ -60,6 +60,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     'scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware': 550,
     'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600,
     'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
+    'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
     'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
     'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
     'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
@@ -100,6 +101,7 @@ ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'
 ITEM_PIPELINES = []
 
 LOG_ENABLED = True
+LOG_FORMATTER_CRAWLED = 'scrapy.contrib.logformatter.crawled_logline'
 LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
@@ -124,6 +126,13 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
+REQUEST_HANDLERS = {}
+REQUEST_HANDLERS_BASE = {
+    'file': 'scrapy.core.downloader.handlers.file.download_file',
+    'http': 'scrapy.core.downloader.handlers.http.download_http',
+    'https': 'scrapy.core.downloader.handlers.http.download_http',
+}
+
 REQUESTS_QUEUE_SIZE = 0
 REQUESTS_PER_DOMAIN = 8     # max simultaneous requests per domain
 
diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py
new file mode 100644
index 000000000..f2b531d28
--- /dev/null
+++ b/scrapy/contrib/downloadermiddleware/httpproxy.py
@@ -0,0 +1,51 @@
+import base64
+from urllib import getproxies, unquote, proxy_bypass
+from urllib2 import _parse_proxy
+from urlparse import urlunparse
+
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.core.exceptions import NotConfigured
+
+
+class HttpProxyMiddleware(object):
+
+    def __init__(self):
+        self.proxies = {}
+        for type, url in getproxies().items():
+            self.proxies[type] = self._get_proxy(url, type)
+
+        if not self.proxies:
+            raise NotConfigured
+
+    def _get_proxy(self, url, orig_type):
+        proxy_type, user, password, hostport = _parse_proxy(url)
+        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+        else:
+            creds = None
+
+        return creds, proxy_url
+
+    def process_request(self, request, spider):
+        # ignore if proxy is already seted
+        if 'proxy' in request.meta:
+            return
+
+        parsed = urlparse_cached(request)
+        scheme = parsed.scheme
+
+        # 'no_proxy' is only supported by http schemes
+        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
+            return
+
+        if scheme in self.proxies:
+            self._set_proxy(request, scheme)
+
+    def _set_proxy(self, request, scheme):
+        creds, proxy = self.proxies[scheme]
+        request.meta['proxy'] = proxy
+        if creds:
+            request.headers['Proxy-Authorization'] = 'Basic ' + creds
diff --git a/scrapy/contrib/logformatter.py b/scrapy/contrib/logformatter.py
new file mode 100644
index 000000000..55995f6e8
--- /dev/null
+++ b/scrapy/contrib/logformatter.py
@@ -0,0 +1,7 @@
+"""Functions for logging diferent actions"""
+
+def crawled_logline(request, response):
+    referer = request.headers.get('Referer')
+    flags = ' %s' % str(response.flags) if response.flags else ''
+    return "Crawled (%d) %s (referer: %s)%s" % (response.status, \
+        request, referer, flags)
diff --git a/scrapy/contrib_exp/loader/__init__.py b/scrapy/contrib_exp/loader/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/scrapy/contrib_exp/loader/lxmlloader.py b/scrapy/contrib_exp/loader/lxmlloader.py
new file mode 100644
index 000000000..724e7c43c
--- /dev/null
+++ b/scrapy/contrib_exp/loader/lxmlloader.py
@@ -0,0 +1,34 @@
+from lxml import html, etree
+
+from scrapy.contrib.loader import ItemLoader
+
+
+class LxmlItemLoader(ItemLoader):
+
+    def __init__(self, response, item=None, **context):
+        self.tree = html.fromstring(response.body_as_unicode())
+        context.update(response=response)
+        super(LxmlItemLoader, self).__init__(item, **context)
+
+    def add_xpath(self, field_name, xpath):
+        self.add_value(field_name, self._get_xpath(xpath))
+
+    def replace_xpath(self, field_name, xpath):
+        self.replace_value(field_name, self._get_xpath(xpath))
+
+    def _get_xpath(self, xpath):
+        return self._get_values(self.tree.xpath(xpath))
+
+    def add_css(self, field_name, css):
+        self.add_value(field_name, self._get_css(css))
+
+    def replace_css(self, field_name, css):
+        self.replace_value(field_name, self._get_css(css))
+
+    def _get_css(self, css):
+        return self._get_values(self.tree.cssselect(css))
+
+    def _get_values(self, elems):
+        for e in elems:
+            yield etree.tostring(e) if isinstance(e, etree.ElementBase) else e
+
diff --git a/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py
deleted file mode 100644
index 016e6677d..000000000
--- a/scrapy/core/downloader/handlers.py
+++ /dev/null
@@ -1,90 +0,0 @@
-"""
-Download handlers for different schemes
-"""
-from __future__ import with_statement
-
-import urlparse
-
-from twisted.internet import reactor
-try:
-    from twisted.internet import ssl
-except ImportError:
-    pass
-
-from scrapy import optional_features
-from scrapy.core import signals
-from scrapy.http import Headers
-from scrapy.core.exceptions import NotSupported
-from scrapy.utils.defer import defer_succeed
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.signal import send_catch_log
-from scrapy.utils.misc import load_object
-from scrapy.core.downloader.responsetypes import responsetypes
-from scrapy.conf import settings
-
-
-HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
-ssl_supported = 'ssl' in optional_features
-
-def download_any(request, spider):
-    scheme = urlparse_cached(request).scheme
-    if scheme == 'http':
-        return download_http(request, spider)
-    elif scheme == 'https':
-        if ssl_supported:
-            return download_https(request, spider)
-        else:
-            raise NotSupported("HTTPS not supported: install pyopenssl library")
-    elif scheme == 'file':
-        return download_file(request, spider)
-    else:
-        raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
-
-def create_factory(request, spider):
-    """Return HTTPClientFactory for the given Request"""
-    url = urlparse.urldefrag(request.url)[0]
-    timeout = getattr(spider, "download_timeout", None) or default_timeout
-    factory = HTTPClientFactory.from_request(request, timeout)
-
-    def _create_response(body):
-        body = body or ''
-        status = int(factory.status)
-        headers = Headers(factory.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=url)
-        r = respcls(url=request.url, status=status, headers=headers, body=body)
-        send_catch_log(signal=signals.request_uploaded, sender='download_http', \
-            request=request, spider=spider)
-        send_catch_log(signal=signals.response_downloaded, sender='download_http', \
-            response=r, spider=spider)
-        return r
-
-    factory.deferred.addCallbacks(_create_response)
-    return factory
-
-def download_http(request, spider):
-    """Return a deferred for the HTTP download"""
-    factory = create_factory(request, spider)
-    url = urlparse_cached(request)
-    port = url.port
-    reactor.connectTCP(url.hostname, port or 80, factory)
-    return factory.deferred
-
-def download_https(request, spider):
-    """Return a deferred for the HTTPS download"""
-    factory = create_factory(request, spider)
-    url = urlparse_cached(request)
-    port = url.port
-    contextFactory = ssl.ClientContextFactory()
-    reactor.connectSSL(url.hostname, port or 443, factory, contextFactory)
-    return factory.deferred
-
-def download_file(request, spider) :
-    """Return a deferred for a file download."""
-    filepath = request.url.split("file://")[1]
-    with open(filepath) as f:
-        body = f.read()
-        respcls = responsetypes.from_args(filename=filepath, body=body)
-        response = respcls(url=request.url, body=body)
-
-    return defer_succeed(response)
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
new file mode 100644
index 000000000..94d619202
--- /dev/null
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -0,0 +1,27 @@
+"""Download handlers for different schemes"""
+
+from scrapy.core.exceptions import NotSupported
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.conf import settings
+from scrapy.utils.misc import load_object
+
+
+class RequestHandlers(object):
+
+    def __init__(self):
+        self._handlers = {}
+        handlers = settings.get('REQUEST_HANDLERS_BASE')
+        handlers.update(settings.get('REQUEST_HANDLERS', {}))
+        for scheme, cls in handlers.iteritems():
+            self._handlers[scheme] = load_object(cls)
+
+    def download_request(self, request, spider):
+        scheme = urlparse_cached(request).scheme
+        try:
+            handler = self._handlers[scheme]
+        except KeyError:
+            raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
+        return handler(request, spider)
+
+
+download_any = RequestHandlers().download_request
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
new file mode 100644
index 000000000..8a02d02d6
--- /dev/null
+++ b/scrapy/core/downloader/handlers/file.py
@@ -0,0 +1,18 @@
+"""Download handler for file:// scheme"""
+from __future__ import with_statement
+
+from twisted.internet import defer
+from scrapy.core.downloader.responsetypes import responsetypes
+
+
+def download_file(request, spider):
+    """Return a deferred for a file download."""
+    return defer.maybeDeferred(_all_in_one_read_download_file, request, spider)
+
+def _all_in_one_read_download_file(request, spider):
+    filepath = request.url.split("file://")[1]
+    with open(filepath) as f:
+        body = f.read()
+    respcls = responsetypes.from_args(filename=filepath, body=body)
+    return respcls(url=request.url, body=body)
+
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
new file mode 100644
index 000000000..756bde6e3
--- /dev/null
+++ b/scrapy/core/downloader/handlers/http.py
@@ -0,0 +1,50 @@
+"""Download handlers for http and https schemes"""
+
+from twisted.internet import reactor
+
+from scrapy.core import signals
+from scrapy.core.exceptions import NotSupported
+from scrapy.utils.signal import send_catch_log
+from scrapy.utils.misc import load_object
+from scrapy.conf import settings
+from scrapy import optional_features
+
+ssl_supported = 'ssl' in optional_features
+if ssl_supported:
+    from twisted.internet.ssl import ClientContextFactory
+
+
+HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
+
+def _create_factory(request, spider):
+    def _download_signals(response):
+        send_catch_log(signal=signals.request_uploaded, \
+                sender='download_http', request=request, spider=spider)
+        send_catch_log(signal=signals.response_downloaded, \
+                sender='download_http', response=response, spider=spider)
+        return response
+
+    timeout = getattr(spider, "download_timeout", None) or default_timeout
+    factory = HTTPClientFactory(request, timeout)
+    factory.deferred.addCallbacks(_download_signals)
+    return factory
+
+
+def _connect(factory):
+    host, port = factory.host, factory.port
+    if factory.scheme == 'https':
+        if ssl_supported:
+            return reactor.connectSSL(host, port, factory, ClientContextFactory())
+        raise NotSupported("HTTPS not supported: install pyopenssl library")
+    else:
+        return reactor.connectTCP(host, port, factory)
+
+
+def download_http(request, spider):
+    """Return a deferred for the HTTP download"""
+    factory = _create_factory(request, spider)
+    _connect(factory)
+    return factory.deferred
+
+
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index ead74ae79..552fc6597 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,12 +1,14 @@
-from urlparse import urlparse, urlunparse
+from urlparse import urlparse, urlunparse, urldefrag
 
 from twisted.python import failure
-from twisted.web.client import HTTPClientFactory, PartialDownloadError
+from twisted.web.client import PartialDownloadError, HTTPClientFactory
 from twisted.web.http import HTTPClient
 from twisted.internet import defer
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.core.downloader.responsetypes import responsetypes
+
 
 def _parsed_url_args(parsed):
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
@@ -18,11 +20,13 @@ def _parsed_url_args(parsed):
         port = 443 if scheme == 'https' else 80
     return scheme, netloc, host, port, path
 
+
 def _parse(url):
     url = url.strip()
     parsed = urlparse(url)
     return _parsed_url_args(parsed)
 
+
 class ScrapyHTTPPageGetter(HTTPClient):
 
     def connectionMade(self):
@@ -64,7 +68,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
     def timeout(self):
         self.transport.loseConnection()
-        self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout)))
+        self.factory.noPage(\
+                defer.TimeoutError("Getting %s took longer than %s seconds." % \
+                (self.factory.url, self.factory.timeout)))
 
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
@@ -74,22 +80,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     """
 
     protocol = ScrapyHTTPPageGetter
-    response_headers = None
     waiting = 1
     noisy = False
 
-    def __init__(self, url, method='GET', body=None, headers=None, timeout=0, parsedurl=None):
-        self.url = url
-        self.method = method
-        self.body = body or None
-        if parsedurl:
-            self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsedurl)
-        else:
-            self.scheme, self.netloc, self.host, self.port, self.path = _parse(url)
-
+    def __init__(self, request, timeout=0):
+        self.url = urldefrag(request.url)[0]
+        self.method = request.method
+        self.body = request.body or None
+        self.headers = request.headers
+        self.response_headers = None
         self.timeout = timeout
-        self.headers = Headers(headers or {})
-        self.deferred = defer.Deferred()
+        self.deferred = defer.Deferred().addCallback(self._build_response)
+
+        self._set_connection_attributes(request)
 
         # set Host header based on url
         self.headers.setdefault('Host', self.netloc)
@@ -100,33 +103,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
             # just in case a broken http/1.1 decides to keep connection alive
             self.headers.setdefault("Connection", "close")
 
-    @classmethod
-    def from_request(cls, request, timeout):
-        return cls(request.url,
-            method=request.method,
-            body=request.body or None, # see http://dev.scrapy.org/ticket/60
-            headers=Headers(request.headers or {}),
-            timeout=timeout,
-            parsedurl=urlparse_cached(request),
-            )
+    def _build_response(self, body):
+        status = int(self.status)
+        headers = Headers(self.response_headers)
+        respcls = responsetypes.from_args(headers=headers, url=self.url)
+        return respcls(url=self.url, status=status, headers=headers, body=body)
+
+    def _set_connection_attributes(self, request):
+        parsed = urlparse_cached(request)
+        self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
+        proxy = request.meta.get('proxy')
+        if proxy:
+            self.scheme, _, self.host, self.port, _ = _parse(proxy)
+            self.path = self.url
 
     def gotHeaders(self, headers):
         self.response_headers = headers
-
-
-
-def getPage(url, contextFactory=None, *args, **kwargs):
-    """
-    Download a web page as a string.
-
-    Download a page. Return a deferred, which will callback with a
-    page (as a string) or errback with a description of the error.
-
-    See HTTPClientFactory to see what extra args can be passed.
-    """
-    from twisted.web.client import _makeGetterFactory
-    return _makeGetterFactory(
-        url,
-        ScrapyHTTPClientFactory,
-        contextFactory=contextFactory,
-        *args, **kwargs).deferred
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 5b03d4231..aaaed7fd9 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -34,6 +34,7 @@ class ExecutionEngine(object):
         self.paused = False
         self._next_request_pending = set()
         self._mainloop_task = task.LoopingCall(self._mainloop)
+        self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED'])
 
     def configure(self):
         """
@@ -198,15 +199,14 @@ class ExecutionEngine(object):
 
     def download(self, request, spider):
         domain = spider.domain_name
-        referer = request.headers.get('Referer')
 
         def _on_success(response):
             """handle the result of a page download"""
             assert isinstance(response, (Response, Request))
             if isinstance(response, Response):
                 response.request = request # tie request to response received
-                log.msg("Crawled %s (referer: <%s>)" % (request, referer), \
-                    level=log.DEBUG, domain=domain)
+                log.msg(self._crawled_logline(request, response), \
+                    level=log.DEBUG, domain=spider.domain_name)
                 return response
             elif isinstance(response, Request):
                 newrequest = response
@@ -224,14 +224,17 @@ class ExecutionEngine(object):
                 errmsg = str(_failure)
                 level = log.ERROR
             if errmsg:
-                log.msg("Downloading <%s> (referer: <%s>): %s" % (request.url, \
-                    referer, errmsg), level=level, domain=domain)
+                log.msg("Crawling <%s>: %s" % (request.url, errmsg), \
+                    level=level, domain=domain)
             return Failure(IgnoreRequest(str(exc)))
 
         def _on_complete(_):
             self.next_request(spider)
             return _
 
+        if spider not in self.downloader.sites:
+            return defer.fail(Failure(IgnoreRequest())).addBoth(_on_complete)
+
         dwld = mustbe_deferred(self.downloader.fetch, request, spider)
         dwld.addCallbacks(_on_success, _on_error)
         dwld.addBoth(_on_complete)
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 8cd55fe99..65355b03d 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -85,10 +85,7 @@ class Request(object_ref):
         return self._encoding
 
     def __str__(self):
-        if self.method == 'GET':
-            return "<%s>" % self.url
-        else:
-            return "<%s %s>" % (self.method, self.url)
+        return "<%s %s>" % (self.method, self.url)
 
     def __repr__(self):
         attrs = ['url', 'method', 'body', 'headers', 'cookies', 'meta']
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 11a9bfcde..f632ea117 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -65,9 +65,7 @@ class Response(object_ref):
         return "%s(%s)" % (self.__class__.__name__, args)
 
     def __str__(self):
-        flags = "(%s) " % ",".join(self.flags) if self.flags else ""
-        status = "%d " % self.status + " " if self.status != 200 else ""
-        return "%s<%s%s>" % (flags, status, self.url)
+        return "<%d %s>" % (self.status, self.url)
 
     def copy(self):
         """Return a copy of this Response"""
diff --git a/scrapy/tests/test_contrib_exp_loader_lxmlloader.py b/scrapy/tests/test_contrib_exp_loader_lxmlloader.py
new file mode 100644
index 000000000..1980648c1
--- /dev/null
+++ b/scrapy/tests/test_contrib_exp_loader_lxmlloader.py
@@ -0,0 +1,67 @@
+from twisted.trial import unittest
+
+from scrapy.contrib.loader.processor import MapCompose
+from scrapy.item import Item, Field
+from scrapy.http import HtmlResponse
+
+try:
+    import lxml
+except ImportError:
+    lxml = False
+
+
+class TestItem(Item):
+    name = Field()
+
+
+if lxml:
+    from scrapy.contrib_exp.loader.lxmlloader import LxmlItemLoader
+
+    class TestLxmlItemLoader(LxmlItemLoader):
+        default_item_class = TestItem
+
+
+class LxmlItemLoaderTest(unittest.TestCase):
+    response = HtmlResponse(url="", body='<html><body><div id="id">marta</div><p>paragraph</p></body></html>')
+
+    def setUp(self):
+        if not lxml:
+            raise unittest.SkipTest("lxml is not available")
+
+    def test_constructor_with_response(self):
+        l = TestLxmlItemLoader(response=self.response)
+        self.assert_(l.tree)
+
+    def test_add_xpath(self):
+        l = TestLxmlItemLoader(response=self.response)
+        l.add_xpath('name', '//div')
+        self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
+
+    def test_add_xpath_text(self):
+        l = TestLxmlItemLoader(response=self.response)
+        l.add_xpath('name', '//div/text()')
+        self.assertEqual(l.get_output_value('name'), [u'marta'])
+
+    def test_replace_xpath(self):
+        l = TestLxmlItemLoader(response=self.response)
+        l.add_xpath('name', '//div/text()')
+        self.assertEqual(l.get_output_value('name'), [u'marta'])
+        l.replace_xpath('name', '//p/text()')
+        self.assertEqual(l.get_output_value('name'), [u'paragraph'])
+
+    def test_add_css(self):
+        l = TestLxmlItemLoader(response=self.response)
+        l.add_css('name', '#id')
+        self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
+
+    def test_replace_css(self):
+        l = TestLxmlItemLoader(response=self.response)
+        l.add_css('name', '#id')
+        self.assertEqual(l.get_output_value('name'), [u'<div id="id">marta</div>'])
+        l.replace_css('name', 'p')
+        self.assertEqual(l.get_output_value('name'), [u'<p>paragraph</p>'])
+
+
+if __name__ == "__main__":
+    unittest.main()
+
diff --git a/scrapy/tests/test_contrib_logformatter.py b/scrapy/tests/test_contrib_logformatter.py
new file mode 100644
index 000000000..e096f7359
--- /dev/null
+++ b/scrapy/tests/test_contrib_logformatter.py
@@ -0,0 +1,22 @@
+import unittest
+
+from scrapy.http import Request, Response
+from scrapy.contrib.logformatter import crawled_logline
+
+
+class LoggingContribTest(unittest.TestCase):
+
+    def test_crawled_logline(self):
+        req = Request("http://www.example.com")
+        res = Response("http://www.example.com")
+        self.assertEqual(crawled_logline(req, res),
+            "Crawled (200) <GET http://www.example.com> (referer: None)")
+
+        req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
+        res = Response("http://www.example.com", flags=['cached'])
+        self.assertEqual(crawled_logline(req, res),
+            "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py
new file mode 100644
index 000000000..f5bf63ec2
--- /dev/null
+++ b/scrapy/tests/test_downloader_handlers.py
@@ -0,0 +1,149 @@
+import os
+
+from twisted.trial import unittest
+from twisted.protocols.policies import WrappingFactory
+from twisted.python.filepath import FilePath
+from twisted.internet import reactor, defer
+from twisted.web import server, static, util, resource
+from twisted.web.test.test_webclient import ForeverTakingResource, \
+        NoLengthResource, HostHeaderResource, \
+        PayloadResource, BrokenDownloadResource
+
+from scrapy.core.downloader.webclient import PartialDownloadError
+from scrapy.core.downloader.handlers.file import download_file
+from scrapy.core.downloader.handlers.http import download_http
+from scrapy.spider import BaseSpider
+from scrapy.http import Request
+
+
+class FileTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpname = self.mktemp()
+        fd = open(self.tmpname, 'w')
+        fd.write('0123456789')
+        fd.close()
+
+    def test_download(self):
+        def _test(response):
+            self.assertEquals(response.url, request.url)
+            self.assertEquals(response.status, 200)
+            self.assertEquals(response.body, '0123456789')
+
+        request = Request('file://%s' % self.tmpname)
+        return download_file(request, BaseSpider()).addCallback(_test)
+
+    def test_non_existent(self):
+        request = Request('file://%s' % self.mktemp())
+        d = download_file(request, BaseSpider())
+        return self.assertFailure(d, IOError)
+
+
+class HttpTestCase(unittest.TestCase):
+
+    def setUp(self):
+        name = self.mktemp()
+        os.mkdir(name)
+        FilePath(name).child("file").setContent("0123456789")
+        r = static.File(name)
+        r.putChild("redirect", util.Redirect("/file"))
+        r.putChild("wait", ForeverTakingResource())
+        r.putChild("nolength", NoLengthResource())
+        r.putChild("host", HostHeaderResource())
+        r.putChild("payload", PayloadResource())
+        r.putChild("broken", BrokenDownloadResource())
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
+        self.portno = self.port.getHost().port
+
+    def tearDown(self):
+        return self.port.stopListening()
+
+    def getURL(self, path):
+        return "http://127.0.0.1:%d/%s" % (self.portno, path)
+
+    def test_download(self):
+        request = Request(self.getURL('file'))
+        d = download_http(request, BaseSpider())
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, "0123456789")
+        return d
+
+    def test_redirect_status(self):
+        request = Request(self.getURL('redirect'))
+        d = download_http(request, BaseSpider())
+        d.addCallback(lambda r: r.status)
+        d.addCallback(self.assertEquals, 302)
+        return d
+
+    def test_timeout_download_from_spider(self):
+        spider = BaseSpider()
+        spider.download_timeout = 0.000001
+        request = Request(self.getURL('wait'))
+        d = download_http(request, spider)
+        return self.assertFailure(d, defer.TimeoutError)
+
+    def test_host_header(self):
+        request = Request(self.getURL('host'))
+        d = download_http(request, BaseSpider())
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, '127.0.0.1:%d' % self.portno)
+        return d
+
+    def test_payload(self):
+        body = '1'*100 # PayloadResource requires body length to be 100
+        request = Request(self.getURL('payload'), method='POST', body=body)
+        d = download_http(request, BaseSpider())
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, body)
+        return d
+
+    def test_broken_download(self):
+        request = Request(self.getURL('broken'))
+        d = download_http(request, BaseSpider())
+        return self.assertFailure(d, PartialDownloadError)
+
+
+class UriResource(resource.Resource):
+    """Return the full uri that was requested"""
+
+    def getChild(self, path, request):
+        return self
+
+    def render(self, request):
+        return request.uri
+
+
+class HttpProxyTestCase(unittest.TestCase):
+
+    def setUp(self):
+        site = server.Site(UriResource(), timeout=None)
+        wrapper = WrappingFactory(site)
+        self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
+        self.portno = self.port.getHost().port
+
+    def tearDown(self):
+        return self.port.stopListening()
+
+    def getURL(self, path):
+        return "http://127.0.0.1:%d/%s" % (self.portno, path)
+
+    def test_download_with_proxy(self):
+        def _test(response):
+            self.assertEquals(response.status, 200)
+            self.assertEquals(response.url, request.url)
+            self.assertEquals(response.body, 'https://example.com')
+
+        http_proxy = self.getURL('')
+        request = Request('https://example.com', meta={'proxy': http_proxy})
+        return download_http(request, BaseSpider()).addCallback(_test)
+
+    def test_download_without_proxy(self):
+        def _test(response):
+            self.assertEquals(response.status, 200)
+            self.assertEquals(response.url, request.url)
+            self.assertEquals(response.body, '/path/to/resource')
+
+        request = Request(self.getURL('path/to/resource'))
+        return download_http(request, BaseSpider()).addCallback(_test)
diff --git a/scrapy/tests/test_downloadermiddleware_httpproxy.py b/scrapy/tests/test_downloadermiddleware_httpproxy.py
new file mode 100644
index 000000000..c2fb46364
--- /dev/null
+++ b/scrapy/tests/test_downloadermiddleware_httpproxy.py
@@ -0,0 +1,81 @@
+import os
+from unittest import TestCase
+
+from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
+from scrapy.core.exceptions import NotConfigured
+from scrapy.http import Response, Request
+from scrapy.spider import BaseSpider
+from scrapy.conf import settings
+
+spider = BaseSpider()
+
+class TestDefaultHeadersMiddleware(TestCase):
+
+    def setUp(self):
+        self._oldenv = os.environ.copy()
+
+    def tearDown(self):
+        os.environ = self._oldenv
+
+    def test_no_proxies(self):
+        os.environ = {}
+        self.assertRaises(NotConfigured, HttpProxyMiddleware)
+
+    def test_no_enviroment_proxies(self):
+        os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
+        mw = HttpProxyMiddleware()
+
+        for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'):
+            req = Request(url)
+            assert mw.process_request(req, spider) is None
+            self.assertEquals(req.url, url)
+            self.assertEquals(req.meta, {})
+
+    def test_enviroment_proxies(self):
+        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080'
+        os.environ.pop('file_proxy', None)
+        mw = HttpProxyMiddleware()
+
+        for url, proxy in [('http://e.com', http_proxy),
+                ('https://e.com', https_proxy), ('file://tmp/a', None)]:
+            req = Request(url)
+            assert mw.process_request(req, spider) is None
+            self.assertEquals(req.url, url)
+            self.assertEquals(req.meta.get('proxy'), proxy)
+
+    def test_proxy_auth(self):
+        os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
+        mw = HttpProxyMiddleware()
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz')
+
+    def test_proxy_already_seted(self):
+        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        mw = HttpProxyMiddleware()
+        req = Request('http://noproxy.com', meta={'proxy': None})
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' in req.meta and req.meta['proxy'] is None
+
+
+    def test_no_proxy(self):
+        os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
+        mw = HttpProxyMiddleware()
+
+        os.environ['no_proxy'] = '*'
+        req = Request('http://noproxy.com')
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' not in req.meta
+
+        os.environ['no_proxy'] = 'other.com'
+        req = Request('http://noproxy.com')
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' in req.meta
+
+        os.environ['no_proxy'] = 'other.com,noproxy.com'
+        req = Request('http://noproxy.com')
+        assert mw.process_request(req, spider) is None
+        assert 'proxy' not in req.meta
+
diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py
index fa0799657..260b2bc08 100644
--- a/scrapy/tests/test_webclient.py
+++ b/scrapy/tests/test_webclient.py
@@ -13,14 +13,27 @@ from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 
 from scrapy.core.downloader import webclient as client
-from scrapy.http import Headers
+from scrapy.http import Request, Headers
+
+
+def getPage(url, contextFactory=None, *args, **kwargs):
+    """Adapted version of twisted.web.client.getPage"""
+    def _clientfactory(*args, **kwargs):
+        timeout = kwargs.pop('timeout', 0)
+        f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout)
+        f.deferred.addCallback(lambda r: r.body)
+        return f
+
+    from twisted.web.client import _makeGetterFactory
+    return _makeGetterFactory(url, _clientfactory,
+        contextFactory=contextFactory, *args, **kwargs).deferred
 
 
 class ParseUrlTestCase(unittest.TestCase):
     """Test URL parsing facility and defaults values."""
 
     def _parse(self, url):
-        f = client.ScrapyHTTPClientFactory(url)
+        f = client.ScrapyHTTPClientFactory(Request(url))
         return (f.scheme, f.netloc, f.host, f.port, f.path)
 
     def testParse(self):
@@ -75,7 +88,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
 
     def test_earlyHeaders(self):
         # basic test stolen from twisted HTTPageGetter
-        factory = client.ScrapyHTTPClientFactory(
+        factory = client.ScrapyHTTPClientFactory(Request(
             url='http://foo/bar',
             body="some data",
             headers={
@@ -83,7 +96,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
                 'User-Agent': 'fooble',
                 'Cookie': 'blah blah',
                 'Content-Length': '12981',
-                'Useful': 'value'})
+                'Useful': 'value'}))
 
         self._test(factory,
             "GET /bar HTTP/1.0\r\n"
@@ -97,18 +110,18 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             "some data")
 
         # test minimal sent headers
-        factory = client.ScrapyHTTPClientFactory('http://foo/bar')
+        factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
         self._test(factory,
             "GET /bar HTTP/1.0\r\n"
             "Host: foo\r\n"
             "\r\n")
 
         # test a simple POST with body and content-type
-        factory = client.ScrapyHTTPClientFactory(
+        factory = client.ScrapyHTTPClientFactory(Request(
             method='POST',
             url='http://foo/bar',
             body='name=value',
-            headers={'Content-Type': 'application/x-www-form-urlencoded'})
+            headers={'Content-Type': 'application/x-www-form-urlencoded'}))
 
         self._test(factory,
             "POST /bar HTTP/1.0\r\n"
@@ -120,12 +133,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             "name=value")
 
         # test with single and multivalued headers
-        factory = client.ScrapyHTTPClientFactory(
+        factory = client.ScrapyHTTPClientFactory(Request(
             url='http://foo/bar',
             headers={
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                })
+                }))
 
         self._test(factory,
             "GET /bar HTTP/1.0\r\n"
@@ -136,12 +149,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
             "\r\n")
 
         # same test with single and multivalued headers but using Headers class
-        factory = client.ScrapyHTTPClientFactory(
+        factory = client.ScrapyHTTPClientFactory(Request(
             url='http://foo/bar',
             headers=Headers({
                 'X-Meta-Single': 'single',
                 'X-Meta-Multivalued': ['value1', 'value2'],
-                }))
+                })))
 
         self._test(factory,
             "GET /bar HTTP/1.0\r\n"
@@ -193,11 +206,11 @@ class WebClientTestCase(unittest.TestCase):
 
     def testPayload(self):
         s = "0123456789" * 10
-        return client.getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
+        return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
 
     def testBrokenDownload(self):
         # test what happens when download gets disconnected in the middle
-        d = client.getPage(self.getURL("broken"))
+        d = getPage(self.getURL("broken"))
         d = self.assertFailure(d, client.PartialDownloadError)
         d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
         return d
@@ -206,8 +219,8 @@ class WebClientTestCase(unittest.TestCase):
         # if we pass Host header explicitly, it should be used, otherwise
         # it should extract from url
         return defer.gatherResults([
-            client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
-            client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
+            getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
+            getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
 
 
     def test_getPage(self):
@@ -215,7 +228,7 @@ class WebClientTestCase(unittest.TestCase):
         L{client.getPage} returns a L{Deferred} which is called back with
         the body of the response if the default method B{GET} is used.
         """
-        d = client.getPage(self.getURL("file"))
+        d = getPage(self.getURL("file"))
         d.addCallback(self.assertEquals, "0123456789")
         return d
 
@@ -226,11 +239,11 @@ class WebClientTestCase(unittest.TestCase):
         the empty string if the method is C{HEAD} and there is a successful
         response code.
         """
-        def getPage(method):
-            return client.getPage(self.getURL("file"), method=method)
+        def _getPage(method):
+            return getPage(self.getURL("file"), method=method)
         return defer.gatherResults([
-            getPage("head").addCallback(self.assertEqual, ""),
-            getPage("HEAD").addCallback(self.assertEqual, "")])
+            _getPage("head").addCallback(self.assertEqual, ""),
+            _getPage("HEAD").addCallback(self.assertEqual, "")])
 
 
     def test_timeoutNotTriggering(self):
@@ -239,7 +252,7 @@ class WebClientTestCase(unittest.TestCase):
         retrieved before the timeout period elapses, the L{Deferred} is
         called back with the contents of the page.
         """
-        d = client.getPage(self.getURL("host"), timeout=100)
+        d = getPage(self.getURL("host"), timeout=100)
         d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno)
         return d
 
@@ -251,7 +264,7 @@ class WebClientTestCase(unittest.TestCase):
         L{Deferred} is errbacked with a L{error.TimeoutError}.
         """
         finished = self.assertFailure(
-            client.getPage(self.getURL("wait"), timeout=0.000001),
+            getPage(self.getURL("wait"), timeout=0.000001),
             defer.TimeoutError)
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
@@ -266,7 +279,7 @@ class WebClientTestCase(unittest.TestCase):
         return finished
 
     def testNotFound(self):
-        return client.getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
+        return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
 
     def _cbNoSuchFile(self, pageData):
         self.assert_('404 - No Such Resource' in pageData)
@@ -274,7 +287,7 @@ class WebClientTestCase(unittest.TestCase):
     def testFactoryInfo(self):
         url = self.getURL('file')
         scheme, netloc, host, port, path = client._parse(url)
-        factory = client.ScrapyHTTPClientFactory(url)
+        factory = client.ScrapyHTTPClientFactory(Request(url))
         reactor.connectTCP(host, port, factory)
         return factory.deferred.addCallback(self._cbFactoryInfo, factory)
 
@@ -285,7 +298,7 @@ class WebClientTestCase(unittest.TestCase):
         self.assertEquals(factory.response_headers['content-length'], '10')
 
     def testRedirect(self):
-        return client.getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
+        return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
 
     def _cbRedirect(self, pageData):
         self.assertEquals(pageData,