From 1fa96dbcd39a28efda06d6f327e26647133550f2 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 13 Sep 2010 19:02:15 -0300 Subject: [PATCH 1/6] Bumped version to 0.11 --- scrapy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 7f08bf70d..37cb3a0a0 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,8 +2,8 @@ Scrapy - a screen scraping framework written in Python """ -version_info = (0, 10, 0, '') -__version__ = "0.10" +version_info = (0, 11, 0, 'dev') +__version__ = "0.11" import sys, os, warnings From b76cd426907814843da81c8ea14de4f1010242dd Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 14 Sep 2010 01:44:10 -0300 Subject: [PATCH 2/6] Added tests for Scrapyd components. Closes #237 --- bin/runtests.bat | 2 +- bin/runtests.sh | 2 +- scrapyd/config.py | 16 +++++++----- scrapyd/eggstorage.py | 3 +-- scrapyd/interfaces.py | 4 +-- scrapyd/tests/__init__.py | 0 scrapyd/tests/test_eggstorage.py | 41 +++++++++++++++++++++++++++++ scrapyd/tests/test_envion.py | 28 ++++++++++++++++++++ scrapyd/tests/test_poller.py | 41 +++++++++++++++++++++++++++++ scrapyd/tests/test_scheduler.py | 44 ++++++++++++++++++++++++++++++++ 10 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 scrapyd/tests/__init__.py create mode 100644 scrapyd/tests/test_eggstorage.py create mode 100644 scrapyd/tests/test_envion.py create mode 100644 scrapyd/tests/test_poller.py create mode 100644 scrapyd/tests/test_scheduler.py diff --git a/bin/runtests.bat b/bin/runtests.bat index 8a0869938..8e4247681 100644 --- a/bin/runtests.bat +++ b/bin/runtests.bat @@ -1,6 +1,6 @@ @ECHO off -SET test="scrapy" +SET test="scrapy scrapyd" IF NOT "%1" == "" SET test="%1" IF EXIST c:\python26\scripts\trial.py GOTO py26 diff --git a/bin/runtests.sh b/bin/runtests.sh index 33ddc0565..a714a208d 100755 --- a/bin/runtests.sh +++ b/bin/runtests.sh @@ -40,7 +40,7 @@ fi find -name '*.py[co]' -delete if [ $# -eq 0 ]; then - $trial scrapy + $trial scrapy scrapyd else $trial "$@" fi diff --git a/scrapyd/config.py b/scrapyd/config.py index d91efdad1..a01296fee 100644 --- a/scrapyd/config.py +++ b/scrapyd/config.py @@ -9,12 +9,16 @@ class Config(object): SECTION = 'scrapyd' - def __init__(self): - sources = self._getsources() - default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf') - self.cp = SafeConfigParser() - self.cp.readfp(StringIO(default_config)) - self.cp.read(sources) + def __init__(self, values=None): + if values is None: + sources = self._getsources() + default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf') + self.cp = SafeConfigParser() + self.cp.readfp(StringIO(default_config)) + self.cp.read(sources) + else: + self.cp = SafeConfigParser(values) + self.cp.add_section(self.SECTION) def _getsources(self): sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf'] diff --git a/scrapyd/eggstorage.py b/scrapyd/eggstorage.py index 956b02409..f46da4750 100644 --- a/scrapyd/eggstorage.py +++ b/scrapyd/eggstorage.py @@ -6,11 +6,10 @@ from shutil import copyfileobj, rmtree from distutils.version import LooseVersion from zope.interface import implements -from twisted.application.service import Service from .interfaces import IEggStorage -class FilesystemEggStorage(Service): +class FilesystemEggStorage(object): implements(IEggStorage) diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py index ae33ec75f..e8a1f2cf2 100644 --- a/scrapyd/interfaces.py +++ b/scrapyd/interfaces.py @@ -12,11 +12,11 @@ class IEggStorage(Interface): project and version. If version is None, the latest version is returned.""" - def list(self, project): + def list(project): """Return the list of versions which have eggs stored (for the given project) in order (the latest version is the currently used).""" - def delete(self, project, version=None): + def delete(project, version=None): """Delete the egg stored for the given project and version. If should also delete the project if no versions are left""" diff --git a/scrapyd/tests/__init__.py b/scrapyd/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapyd/tests/test_eggstorage.py b/scrapyd/tests/test_eggstorage.py new file mode 100644 index 000000000..9254f0894 --- /dev/null +++ b/scrapyd/tests/test_eggstorage.py @@ -0,0 +1,41 @@ +from cStringIO import StringIO + +from twisted.trial import unittest + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import IEggStorage +from scrapyd.config import Config +from scrapyd.eggstorage import FilesystemEggStorage + +class EggStorageTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + config = Config(values={'eggs_dir': d}) + self.eggst = FilesystemEggStorage(config) + + def test_interface(self): + verifyObject(IEggStorage, self.eggst) + + def test_put_get_list_delete(self): + self.eggst.put(StringIO("egg01"), 'mybot', '01') + self.eggst.put(StringIO("egg03"), 'mybot', '03') + self.eggst.put(StringIO("egg02"), 'mybot', '02') + + self.assertEqual(self.eggst.list('mybot'), ['01', '02', '03']) + self.assertEqual(self.eggst.list('mybot2'), []) + + v, f = self.eggst.get('mybot') + self.assertEqual(v, "03") + self.assertEqual(f.read(), "egg03") + + v, f = self.eggst.get('mybot', '02') + self.assertEqual(v, "02") + self.assertEqual(f.read(), "egg02") + + self.eggst.delete('mybot', '02') + self.assertEqual(self.eggst.list('mybot'), ['01', '03']) + + self.eggst.delete('mybot') + self.assertEqual(self.eggst.list('mybot'), []) diff --git a/scrapyd/tests/test_envion.py b/scrapyd/tests/test_envion.py new file mode 100644 index 000000000..0d1c657ee --- /dev/null +++ b/scrapyd/tests/test_envion.py @@ -0,0 +1,28 @@ +import os + +from twisted.trial import unittest + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import IEnvironment +from scrapyd.config import Config +from scrapyd.environ import Environment + +class EggStorageTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + os.mkdir(d) + config = Config(values={'eggs_dir': d, 'logs_dir': d}) + self.environ = Environment(config) + + def test_interface(self): + verifyObject(IEnvironment, self.environ) + + def test_get_environment(self): + msg = {'project': 'mybot'} + slot = 3 + env = self.environ.get_environment(msg, slot) + self.assertEqual(env['SCRAPY_PROJECT'], 'mybot') + self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) + self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log')) diff --git a/scrapyd/tests/test_poller.py b/scrapyd/tests/test_poller.py new file mode 100644 index 000000000..d8dae49ac --- /dev/null +++ b/scrapyd/tests/test_poller.py @@ -0,0 +1,41 @@ +import os + +from twisted.trial import unittest +from twisted.internet.defer import Deferred + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import IPoller +from scrapyd.config import Config +from scrapyd.poller import QueuePoller +from scrapyd.utils import get_spider_queues + +class QueuePollerTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + eggs_dir = os.path.join(d, 'eggs') + dbs_dir = os.path.join(d, 'dbs') + os.makedirs(eggs_dir) + os.makedirs(dbs_dir) + os.makedirs(os.path.join(eggs_dir, 'mybot1')) + os.makedirs(os.path.join(eggs_dir, 'mybot2')) + config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir}) + self.queues = get_spider_queues(eggs_dir, dbs_dir) + self.poller = QueuePoller(config) + + def test_interface(self): + verifyObject(IPoller, self.poller) + + def test_poll_next(self): + self.queues['mybot1'].add('spider1') + self.queues['mybot2'].add('spider2') + d1 = self.poller.next() + d2 = self.poller.next() + self.failUnless(isinstance(d1, Deferred)) + self.failIf(hasattr(d1, 'result')) + self.poller.poll() + self.queues['mybot1'].pop() + self.poller.poll() + self.failUnlessEqual(d1.result, {'project': 'mybot1'}) + self.failUnlessEqual(d2.result, {'project': 'mybot2'}) diff --git a/scrapyd/tests/test_scheduler.py b/scrapyd/tests/test_scheduler.py new file mode 100644 index 000000000..a9fb7e6f0 --- /dev/null +++ b/scrapyd/tests/test_scheduler.py @@ -0,0 +1,44 @@ +import os + +from twisted.trial import unittest + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import ISpiderScheduler +from scrapyd.config import Config +from scrapyd.scheduler import SpiderScheduler +from scrapyd.utils import get_spider_queues + +class SpiderSchedulerTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + eggs_dir = self.eggs_dir = os.path.join(d, 'eggs') + dbs_dir = os.path.join(d, 'dbs') + os.mkdir(d) + os.makedirs(eggs_dir) + os.makedirs(dbs_dir) + os.makedirs(os.path.join(eggs_dir, 'mybot1')) + os.makedirs(os.path.join(eggs_dir, 'mybot2')) + config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir}) + self.queues = get_spider_queues(eggs_dir, dbs_dir) + self.sched = SpiderScheduler(config) + + def test_interface(self): + verifyObject(ISpiderScheduler, self.sched) + + def test_list_update_projects(self): + self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2'])) + os.makedirs(os.path.join(self.eggs_dir, 'mybot3')) + self.sched.update_projects() + self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2', 'mybot3'])) + + def test_schedule(self): + q = self.queues['mybot1'] + self.failIf(q.count()) + self.sched.schedule('mybot1', 'myspider1', a='b') + self.sched.schedule('mybot2', 'myspider2', c='d') + self.assertEqual(q.pop(), {'name': 'myspider1', 'a': 'b'}) + q = self.queues['mybot2'] + self.assertEqual(q.pop(), {'name': 'myspider2', 'c': 'd'}) + From 833baa60419b708ee0af18fed5c8e2ad31daf929 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 14 Sep 2010 01:44:25 -0300 Subject: [PATCH 3/6] Support running projects without eggs in Scrapyd. Closes #238 --- scrapyd/eggrunner.py | 28 +++++----------------------- scrapyd/eggstorage.py | 5 ++++- scrapyd/eggutils.py | 3 ++- scrapyd/environ.py | 10 +++++++++- scrapyd/interfaces.py | 9 +++++++-- scrapyd/launcher.py | 7 ++++--- scrapyd/tests/test_envion.py | 18 ++++++++++++++++-- 7 files changed, 47 insertions(+), 33 deletions(-) diff --git a/scrapyd/eggrunner.py b/scrapyd/eggrunner.py index 846ee36cd..830768526 100644 --- a/scrapyd/eggrunner.py +++ b/scrapyd/eggrunner.py @@ -1,27 +1,9 @@ -""" -This module can be used to run a Scrapy project contained in an egg file - -To see all spiders in a project: - - python -m scrapyd.eggrunner myproject.egg list - -To crawl a spider: - - python -m scrapyd.eggrunner myproject.egg crawl somespider -""" - -import sys +import sys, os from scrapyd.eggutils import activate_egg -def main(eggpath, args): - """Run scrapy for the settings module name passed""" +eggpath = os.environ.get('SCRAPY_EGGFILE') +if eggpath: activate_egg(eggpath) - from scrapy.cmdline import execute - execute(['scrapy'] + list(args)) - -if __name__ == '__main__': - if len(sys.argv) < 2: - print "usage: %s [scrapy_command args ...]" % sys.argv[0] - sys.exit(1) - main(sys.argv[1], sys.argv[2:]) +from scrapy.cmdline import execute +execute() diff --git a/scrapyd/eggstorage.py b/scrapyd/eggstorage.py index f46da4750..fbeb9e9c4 100644 --- a/scrapyd/eggstorage.py +++ b/scrapyd/eggstorage.py @@ -26,7 +26,10 @@ class FilesystemEggStorage(object): def get(self, project, version=None): if version is None: - version = self.list(project)[-1] + try: + version = self.list(project)[-1] + except IndexError: + return None, None return version, open(self._eggpath(project, version), 'rb') def list(self, project): diff --git a/scrapyd/eggutils.py b/scrapyd/eggutils.py index 95ab2aca5..1e1975580 100644 --- a/scrapyd/eggutils.py +++ b/scrapyd/eggutils.py @@ -12,9 +12,10 @@ def get_spider_list_from_eggfile(eggfile, project): shutil.copyfileobj(eggfile, f) f.flush() eggfile.seek(0) - pargs = [sys.executable, '-m', 'scrapyd.eggrunner', f.name, 'list'] + pargs = [sys.executable, '-m', 'scrapyd.eggrunner', 'list'] env = os.environ.copy() env['SCRAPY_PROJECT'] = project + env['SCRAPY_EGGFILE'] = f.name proc = Popen(pargs, stdout=PIPE, cwd=tmpdir, env=env) out = proc.communicate()[0] return out.splitlines() diff --git a/scrapyd/environ.py b/scrapyd/environ.py index 0cd184687..0cbf7c30b 100644 --- a/scrapyd/environ.py +++ b/scrapyd/environ.py @@ -11,11 +11,19 @@ class Environment(object): def __init__(self, config): self.dbs_dir = config.get('dbs_dir', 'dbs') self.logs_dir = config.get('logs_dir', 'logs') + if config.cp.has_section('settings'): + self.settings = dict(config.cp.items('settings')) + else: + self.settings = {} - def get_environment(self, message, slot): + def get_environment(self, message, slot, eggpath): project = message['project'] env = os.environ.copy() env['SCRAPY_PROJECT'] = project + if eggpath: + env['SCRAPY_EGGFILE'] = eggpath + elif project in self.settings: + env['SCRAPY_SETTINGS_MODULE'] = self.settings[project] dbpath = os.path.join(self.dbs_dir, '%s.db' % project) env['SCRAPY_SQLITE_DB'] = dbpath logpath = os.path.join(self.logs_dir, 'slot%s.log' % slot) diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py index e8a1f2cf2..30a27e637 100644 --- a/scrapyd/interfaces.py +++ b/scrapyd/interfaces.py @@ -10,7 +10,8 @@ class IEggStorage(Interface): def get(project, version=None): """Return a tuple (version, file) with the the egg for the specified project and version. If version is None, the latest version is - returned.""" + returned. If no egg is found for the given project/version (None, None) + should be returned.""" def list(project): """Return the list of versions which have eggs stored (for the given @@ -61,9 +62,13 @@ class ISpiderScheduler(Interface): class IEnvironment(Interface): """A component to generate the environment of crawler processes""" - def get_environment(message, slot): + def get_environment(message, slot, eggpath): """Return the environment variables to use for running the process. `message` is the message received from the IPoller.next() `slot` is the Launcher slot where the process will be running. + `eggpath` is the path to an eggfile that contains the project code. The + `eggpath` may be `None` if no egg was found for the project, in + which case the project must be on the python path and its settings + defined in scrapyd.conf [settings] section """ diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index 9afcad16f..6742af78b 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -39,15 +39,16 @@ class Launcher(Service): def _spawn_process(self, message, slot): project = message['project'] eggpath = self._get_eggpath(project) - args = [sys.executable, '-m', self.egg_runner, eggpath, 'crawl'] + args = [sys.executable, '-m', self.egg_runner, 'crawl'] e = self.app.getComponent(IEnvironment) - env = e.get_environment(message, slot) + env = e.get_environment(message, slot, eggpath) pp = ScrapyProcessProtocol(eggpath, slot) pp.deferred.addBoth(self._process_finished, eggpath, slot) reactor.spawnProcess(pp, sys.executable, args=args, env=env) def _process_finished(self, _, eggpath, slot): - os.remove(eggpath) + if eggpath: + os.remove(eggpath) self._wait_for_project(slot) diff --git a/scrapyd/tests/test_envion.py b/scrapyd/tests/test_envion.py index 0d1c657ee..9f99da3a6 100644 --- a/scrapyd/tests/test_envion.py +++ b/scrapyd/tests/test_envion.py @@ -14,15 +14,29 @@ class EggStorageTest(unittest.TestCase): d = self.mktemp() os.mkdir(d) config = Config(values={'eggs_dir': d, 'logs_dir': d}) + config.cp.add_section('settings') + config.cp.set('settings', 'newbot', 'newbot.settings') self.environ = Environment(config) def test_interface(self): verifyObject(IEnvironment, self.environ) - def test_get_environment(self): + def test_get_environment_with_eggfile(self): msg = {'project': 'mybot'} slot = 3 - env = self.environ.get_environment(msg, slot) + env = self.environ.get_environment(msg, slot, '/path/to/file.egg') self.assertEqual(env['SCRAPY_PROJECT'], 'mybot') self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log')) + self.assert_(env['SCRAPY_EGGFILE'].endswith('/path/to/file.egg')) + self.failIf('SCRAPY_SETTINGS_MODULE' in env) + + def test_get_environment_without_eggfile(self): + msg = {'project': 'newbot'} + slot = 3 + env = self.environ.get_environment(msg, slot, None) + self.assertEqual(env['SCRAPY_PROJECT'], 'newbot') + self.assert_(env['SCRAPY_SQLITE_DB'].endswith('newbot.db')) + self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log')) + self.assertEqual(env['SCRAPY_SETTINGS_MODULE'], 'newbot.settings') + self.failIf('SCRAPY_EGGFILE' in env) From c559b06a850d35924ea15e2bb7b90cb5cfe36e83 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 14 Sep 2010 01:53:05 -0300 Subject: [PATCH 4/6] Removed unused import --- scrapyd/eggrunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapyd/eggrunner.py b/scrapyd/eggrunner.py index 830768526..481c0a2bc 100644 --- a/scrapyd/eggrunner.py +++ b/scrapyd/eggrunner.py @@ -1,4 +1,4 @@ -import sys, os +import os from scrapyd.eggutils import activate_egg From 9acc99e723295710d9b42525b97fe2246048398b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 14 Sep 2010 14:36:35 -0300 Subject: [PATCH 5/6] Added 'inthread' decorator --- scrapy/utils/decorator.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py index b6a8b83cf..54edebc85 100644 --- a/scrapy/utils/decorator.py +++ b/scrapy/utils/decorator.py @@ -1,7 +1,7 @@ import warnings from functools import wraps -from twisted.internet.defer import maybeDeferred +from twisted.internet import defer, threads def deprecated(use_instead=None): @@ -24,5 +24,14 @@ def defers(func): """Decorator to make sure a function always returns a deferred""" @wraps(func) def wrapped(*a, **kw): - return maybeDeferred(func, *a, **kw) + return defer.maybeDeferred(func, *a, **kw) + return wrapped + +def inthread(func): + """Decorator to call a function in a thread and return a deferred with the + result + """ + @wraps(func) + def wrapped(*a, **kw): + return threads.deferToThread(func, *a, **kw) return wrapped From 39499a24377770e6eeca91101f1be59e738764e0 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 14 Sep 2010 20:22:25 -0300 Subject: [PATCH 6/6] Fixed bug in HTTP Compression middleware which was failing to properly discover the encoding when the encoding was declared inside the response body. Closes #239. Also changed responsetypes to return Response class (instead of HtmlResponse) when the response has a Content-Encoding header --- .../downloadermiddleware/httpcompression.py | 5 +++- .../core/downloader/responsetypes/__init__.py | 7 ++++-- ...st_downloadermiddleware_httpcompression.py | 24 ++++++++++++++++++- scrapy/tests/test_responsetypes.py | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py index a2de8757d..031001f15 100644 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ b/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -3,6 +3,7 @@ from gzip import GzipFile from cStringIO import StringIO from scrapy.http import Response +from scrapy.core.downloader.responsetypes import responsetypes class HttpCompressionMiddleware(object): @@ -18,7 +19,9 @@ class HttpCompressionMiddleware(object): if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - response = response.replace(body=decoded_body) + respcls = responsetypes.from_args(headers=response.headers, \ + url=response.url) + response = response.replace(cls=respcls, body=decoded_body) if not content_encoding: del response.headers['Content-Encoding'] diff --git a/scrapy/core/downloader/responsetypes/__init__.py b/scrapy/core/downloader/responsetypes/__init__.py index 297ec30ac..51631b282 100644 --- a/scrapy/core/downloader/responsetypes/__init__.py +++ b/scrapy/core/downloader/responsetypes/__init__.py @@ -46,9 +46,11 @@ class ResponseTypes(object): basetype = "%s/*" % mimetype.split('/')[0] return self.classes.get(basetype, Response) - def from_content_type(self, content_type): + def from_content_type(self, content_type, content_encoding=None): """Return the most appropiate Response class from an HTTP Content-Type header """ + if content_encoding: + return Response mimetype = content_type.split(';')[0].strip().lower() return self.from_mimetype(mimetype) @@ -65,7 +67,8 @@ class ResponseTypes(object): headers""" cls = Response if 'Content-Type' in headers: - cls = self.from_content_type(headers['Content-type']) + cls = self.from_content_type(headers['Content-type'], \ + headers.get('Content-Encoding')) if cls is Response and 'Content-Disposition' in headers: cls = self.from_content_disposition(headers['Content-Disposition']) return cls diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index c8fb38d22..40988ecd7 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -2,11 +2,14 @@ from __future__ import with_statement from unittest import TestCase from os.path import join, abspath, dirname +from cStringIO import StringIO +from gzip import GzipFile from scrapy.spider import BaseSpider -from scrapy.http import Response, Request +from scrapy.http import Response, Request, HtmlResponse from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware from scrapy.tests import tests_datadir +from scrapy.utils.encoding import resolve_encoding SAMPLEDIR = join(tests_datadir, 'compressed') @@ -96,3 +99,22 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) assert newresponse is not response self.assertEqual(newresponse.headers.getlist('Content-Encoding'), ['uuencode']) + + def test_process_response_encoding_inside_body(self): + headers = { + 'Content-Type': 'text/html', + 'Content-Encoding': 'gzip', + } + f = StringIO() + plainbody = """Some page""" + zf = GzipFile(fileobj=f, mode='wb') + zf.write(plainbody) + zf.close() + response = Response("http;//www.example.com/", headers=headers, body=f.getvalue()) + request = Request("http://www.example.com/") + + newresponse = self.mw.process_response(request, response, self.spider) + assert isinstance(newresponse, HtmlResponse) + self.assertEqual(newresponse.body, plainbody) + self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + diff --git a/scrapy/tests/test_responsetypes.py b/scrapy/tests/test_responsetypes.py index 14f3cf387..8e4e1643a 100644 --- a/scrapy/tests/test_responsetypes.py +++ b/scrapy/tests/test_responsetypes.py @@ -54,6 +54,7 @@ class ResponseTypesTest(unittest.TestCase): mappings = [ ({'Content-Type': ['text/html; charset=utf-8']}, HtmlResponse), ({'Content-Type': ['application/octet-stream'], 'Content-Disposition': ['attachment; filename=data.txt']}, TextResponse), + ({'Content-Type': ['text/html; charset=utf-8'], 'Content-Encoding': ['gzip']}, Response), ] for source, cls in mappings: source = Headers(source)