From 8fdf06ed08c5fc9ccd4c3a49a38599ac927f56e1 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 10 Feb 2009 13:23:38 +0000 Subject: [PATCH] Avoid sending Content-Length header when body is an empty string. Some sites can't handle "Content-Length: 0" header, but twisted HTTPClientFactory adds Content-Length header unless body is None. scrapy enforces request.body usage as string, using None is not possible. thanks Matt for report, See: http://groups.google.com/group/scrapy-users/browse_thread/thread/380ffa111879989e?hl=en --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40841 --- scrapy/trunk/docs/ref/request-response.rst | 6 ++++-- scrapy/trunk/scrapy/core/downloader/handlers.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/trunk/docs/ref/request-response.rst b/scrapy/trunk/docs/ref/request-response.rst index 5cf7fabbb..000dc0e72 100644 --- a/scrapy/trunk/docs/ref/request-response.rst +++ b/scrapy/trunk/docs/ref/request-response.rst @@ -41,8 +41,10 @@ Request objects ``meta`` is a dict containing the initial values for the :attr:`Request.meta` attribute. If passed, the dict will be shallow copied. - ``body`` is a str or unicode containing the request body. If ``body`` is - a `unicode` it's encoded to str using the `encoding` passed. + ``body`` is a str or unicode containing the request body. + If ``body`` is a `unicode` it's encoded to str using the `encoding` passed. + If ``body`` is None, an empty string is stored. + In any case, the final stored value will be a string (never unicode, never None). ``headers`` is a multi-valued dict containing the headers of this request diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py index ae74e1710..23f7a5af3 100644 --- a/scrapy/trunk/scrapy/core/downloader/handlers.py +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -53,7 +53,7 @@ def create_factory(request, spider): agent = request.headers.pop('user-agent', default_agent) factory = HTTPClientFactory(url=url, # never pass unicode urls to twisted method=request.method, - postdata=request.body, + postdata=request.body or None, headers=request.headers, agent=agent, cookies=request.cookies,