From b9928558b293be71b02afc4da7ba4e42896b6a87 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 17:50:38 +0500 Subject: [PATCH] TST remove temporary files and folders --- tests/test_downloader_handlers.py | 21 ++++++++++++++------- tests/test_feedexport.py | 22 ++++++++++++++-------- tests/test_spiderstate.py | 28 ++++++++++++++++------------ tests/test_webclient.py | 19 +++++++++---------- 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6333efceb..f9980f7b2 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,6 +1,7 @@ import os import six import contextlib +import shutil try: from unittest import mock except ImportError: @@ -84,11 +85,13 @@ class FileTestCase(unittest.TestCase): def setUp(self): self.tmpname = self.mktemp() - fd = open(self.tmpname + '^', 'w') - fd.write('0123456789') - fd.close() + with open(self.tmpname + '^', 'w') as f: + f.write('0123456789') self.download_request = FileDownloadHandler(Settings()).download_request + def tearDown(self): + os.unlink(self.tmpname + '^') + def test_download(self): def _test(response): self.assertEquals(response.url, request.url) @@ -134,10 +137,10 @@ class HttpTestCase(unittest.TestCase): certfile = 'keys/cert.pem' def setUp(self): - name = self.mktemp() - os.mkdir(name) - FilePath(name).child("file").setContent(b"0123456789") - r = static.File(name) + self.tmpname = self.mktemp() + os.mkdir(self.tmpname) + FilePath(self.tmpname).child("file").setContent(b"0123456789") + r = static.File(self.tmpname) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) @@ -165,6 +168,7 @@ class HttpTestCase(unittest.TestCase): yield self.port.stopListening() if hasattr(self.download_handler, 'close'): yield self.download_handler.close() + shutil.rmtree(self.tmpname) def getURL(self, path): return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) @@ -709,6 +713,9 @@ class FTPTestCase(unittest.TestCase): self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening) + def tearDown(self): + shutil.rmtree(self.directory) + def _add_test_callbacks(self, deferred, callback=None, errback=None): def _clean(data): self.download_handler.client.transport.loseConnection() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e93d2bafb..2d137edf4 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -57,8 +57,11 @@ class FileFeedStorageTest(unittest.TestCase): file.write(b"content") yield storage.store(file) self.assertTrue(os.path.exists(path)) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") + try: + with open(path, 'rb') as fp: + self.assertEqual(fp.read(), b"content") + finally: + os.unlink(path) class FTPFeedStorageTest(unittest.TestCase): @@ -79,12 +82,15 @@ class FTPFeedStorageTest(unittest.TestCase): file.write(b"content") yield storage.store(file) self.assertTrue(os.path.exists(path)) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"content") - # again, to check s3 objects are overwritten - yield storage.store(BytesIO(b"new content")) - with open(path, 'rb') as fp: - self.assertEqual(fp.read(), b"new content") + try: + with open(path, 'rb') as fp: + self.assertEqual(fp.read(), b"content") + # again, to check s3 objects are overwritten + yield storage.store(BytesIO(b"new content")) + with open(path, 'rb') as fp: + self.assertEqual(fp.read(), b"new content") + finally: + os.unlink(path) class BlockingFeedStorageTest(unittest.TestCase): diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py index d1d6debec..383fadfeb 100644 --- a/tests/test_spiderstate.py +++ b/tests/test_spiderstate.py @@ -1,5 +1,6 @@ import os from datetime import datetime +import shutil from twisted.trial import unittest from scrapy.extensions.spiderstate import SpiderState @@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase): def test_store_load(self): jobdir = self.mktemp() os.mkdir(jobdir) - spider = Spider(name='default') - dt = datetime.now() + try: + spider = Spider(name='default') + dt = datetime.now() - ss = SpiderState(jobdir) - ss.spider_opened(spider) - spider.state['one'] = 1 - spider.state['dt'] = dt - ss.spider_closed(spider) + ss = SpiderState(jobdir) + ss.spider_opened(spider) + spider.state['one'] = 1 + spider.state['dt'] = dt + ss.spider_closed(spider) - spider2 = Spider(name='default') - ss2 = SpiderState(jobdir) - ss2.spider_opened(spider2) - self.assertEqual(spider.state, {'one': 1, 'dt': dt}) - ss2.spider_closed(spider2) + spider2 = Spider(name='default') + ss2 = SpiderState(jobdir) + ss2.spider_opened(spider2) + self.assertEqual(spider.state, {'one': 1, 'dt': dt}) + ss2.spider_closed(spider2) + finally: + shutil.rmtree(jobdir) def test_state_attribute(self): # state attribute must be present if jobdir is not set, to provide a diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 9b5beda4c..af531c81f 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -4,7 +4,7 @@ Tests borrowed from the twisted.web.client tests. """ import os import six -from six.moves.urllib.parse import urlparse +import shutil from twisted.trial import unittest from twisted.web import server, static, util, resource @@ -12,6 +12,7 @@ from twisted.internet import reactor, defer from twisted.test.proto_helpers import StringTransport from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory +from twisted.internet.defer import inlineCallbacks from scrapy.core.downloader import webclient as client from scrapy.http import Request, Headers @@ -229,10 +230,10 @@ class WebClientTestCase(unittest.TestCase): return reactor.listenTCP(0, site, interface="127.0.0.1") def setUp(self): - name = self.mktemp() - os.mkdir(name) - FilePath(name).child("file").setContent(b"0123456789") - r = static.File(name) + self.tmpname = self.mktemp() + os.mkdir(self.tmpname) + FilePath(self.tmpname).child("file").setContent(b"0123456789") + r = static.File(self.tmpname) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"error", ErrorResource()) @@ -246,8 +247,10 @@ class WebClientTestCase(unittest.TestCase): self.port = self._listen(self.wrapper) self.portno = self.port.getHost().port + @inlineCallbacks def tearDown(self): - return self.port.stopListening() + yield self.port.stopListening() + shutil.rmtree(self.tmpname) def getURL(self, path): return "http://127.0.0.1:%d/%s" % (self.portno, path) @@ -266,7 +269,6 @@ class WebClientTestCase(unittest.TestCase): getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( self.assertEquals, to_bytes("www.example.com"))]) - def test_getPage(self): """ L{client.getPage} returns a L{Deferred} which is called back with @@ -276,7 +278,6 @@ class WebClientTestCase(unittest.TestCase): d.addCallback(self.assertEquals, b"0123456789") return d - def test_getPageHead(self): """ L{client.getPage} returns a L{Deferred} which is called back with @@ -289,7 +290,6 @@ class WebClientTestCase(unittest.TestCase): _getPage("head").addCallback(self.assertEqual, b""), _getPage("HEAD").addCallback(self.assertEqual, b"")]) - def test_timeoutNotTriggering(self): """ When a non-zero timeout is passed to L{getPage} and the page is @@ -301,7 +301,6 @@ class WebClientTestCase(unittest.TestCase): self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)) return d - def test_timeoutTriggering(self): """ When a non-zero timeout is passed to L{getPage} and that many