diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index e4acbc8cb..52cbd4d9d 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -16,9 +16,7 @@ class RedirectMiddleware(object): def process_response(self, request, response, spider): if response.status in [302, 303] and 'Location' in response.headers: redirected_url = urljoin_rfc(request.url, response.headers['location']) - redirected = request.replace(url=redirected_url, method='GET', body='') - redirected.headers.pop('Content-Type', None) - redirected.headers.pop('Content-Length', None) + redirected = self._redirect_request_using_get(request, redirected_url) return self._redirect(redirected, request, spider, response.status) if response.status in [301, 307] and 'Location' in response.headers: @@ -28,9 +26,7 @@ class RedirectMiddleware(object): interval, url = get_meta_refresh(response) if url and interval < self.max_metarefresh_delay: - redirected = request.replace(url=url, method='GET', body='') - redirected.headers.pop('Content-Type', None) - redirected.headers.pop('Content-Length', None) + redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, 'meta refresh') return response @@ -52,4 +48,10 @@ class RedirectMiddleware(object): domain=spider.domain_name, level=log.DEBUG) raise IgnoreRequest + def _redirect_request_using_get(self, request, redirect_url): + redirected = request.replace(url=redirect_url, method='GET', body='') + redirected.headers.pop('Content-Type', None) + redirected.headers.pop('Content-Length', None) + return redirected +