TST remove temporary files and folders

tests/test_downloader_handlers.py

@@ -1,6 +1,7 @@
 import os
 import six
 import contextlib
+import shutil
 try:
     from unittest import mock
 except ImportError:
@@ -84,11 +85,13 @@ class FileTestCase(unittest.TestCase):
 
     def setUp(self):
         self.tmpname = self.mktemp()
-        fd = open(self.tmpname + '^', 'w')
-        fd.write('0123456789')
-        fd.close()
+        with open(self.tmpname + '^', 'w') as f:
+            f.write('0123456789')
         self.download_request = FileDownloadHandler(Settings()).download_request
 
+    def tearDown(self):
+        os.unlink(self.tmpname + '^')
+
     def test_download(self):
         def _test(response):
             self.assertEquals(response.url, request.url)
@@ -134,10 +137,10 @@ class HttpTestCase(unittest.TestCase):
     certfile = 'keys/cert.pem'
 
     def setUp(self):
-        name = self.mktemp()
-        os.mkdir(name)
-        FilePath(name).child("file").setContent(b"0123456789")
-        r = static.File(name)
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
         r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@@ -165,6 +168,7 @@ class HttpTestCase(unittest.TestCase):
         yield self.port.stopListening()
         if hasattr(self.download_handler, 'close'):
             yield self.download_handler.close()
+        shutil.rmtree(self.tmpname)
 
     def getURL(self, path):
         return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
@@ -709,6 +713,9 @@ class FTPTestCase(unittest.TestCase):
         self.download_handler = FTPDownloadHandler(Settings())
         self.addCleanup(self.port.stopListening)
 
+    def tearDown(self):
+        shutil.rmtree(self.directory)
+
     def _add_test_callbacks(self, deferred, callback=None, errback=None):
         def _clean(data):
             self.download_handler.client.transport.loseConnection()

tests/test_feedexport.py

@@ -57,8 +57,11 @@ class FileFeedStorageTest(unittest.TestCase):
         file.write(b"content")
         yield storage.store(file)
         self.assertTrue(os.path.exists(path))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"content")
+        try:
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"content")
+        finally:
+            os.unlink(path)
 
 
 class FTPFeedStorageTest(unittest.TestCase):
@@ -79,12 +82,15 @@ class FTPFeedStorageTest(unittest.TestCase):
         file.write(b"content")
         yield storage.store(file)
         self.assertTrue(os.path.exists(path))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"content")
-        # again, to check s3 objects are overwritten
-        yield storage.store(BytesIO(b"new content"))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"new content")
+        try:
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"content")
+            # again, to check s3 objects are overwritten
+            yield storage.store(BytesIO(b"new content"))
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"new content")
+        finally:
+            os.unlink(path)
 
 
 class BlockingFeedStorageTest(unittest.TestCase):

tests/test_spiderstate.py

@@ -1,5 +1,6 @@
 import os
 from datetime import datetime
+import shutil
 from twisted.trial import unittest
 
 from scrapy.extensions.spiderstate import SpiderState
@@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase):
     def test_store_load(self):
         jobdir = self.mktemp()
         os.mkdir(jobdir)
-        spider = Spider(name='default')
-        dt = datetime.now()
+        try:
+            spider = Spider(name='default')
+            dt = datetime.now()
 
-        ss = SpiderState(jobdir)
-        ss.spider_opened(spider)
-        spider.state['one'] = 1
-        spider.state['dt'] = dt
-        ss.spider_closed(spider)
+            ss = SpiderState(jobdir)
+            ss.spider_opened(spider)
+            spider.state['one'] = 1
+            spider.state['dt'] = dt
+            ss.spider_closed(spider)
 
-        spider2 = Spider(name='default')
-        ss2 = SpiderState(jobdir)
-        ss2.spider_opened(spider2)
-        self.assertEqual(spider.state, {'one': 1, 'dt': dt})
-        ss2.spider_closed(spider2)
+            spider2 = Spider(name='default')
+            ss2 = SpiderState(jobdir)
+            ss2.spider_opened(spider2)
+            self.assertEqual(spider.state, {'one': 1, 'dt': dt})
+            ss2.spider_closed(spider2)
+        finally:
+            shutil.rmtree(jobdir)
 
     def test_state_attribute(self):
         # state attribute must be present if jobdir is not set, to provide a

tests/test_webclient.py

@@ -4,7 +4,7 @@ Tests borrowed from the twisted.web.client tests.
 """
 import os
 import six
-from six.moves.urllib.parse import urlparse
+import shutil
 
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
@@ -12,6 +12,7 @@ from twisted.internet import reactor, defer
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
+from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.downloader import webclient as client
 from scrapy.http import Request, Headers
@@ -229,10 +230,10 @@ class WebClientTestCase(unittest.TestCase):
         return reactor.listenTCP(0, site, interface="127.0.0.1")
 
     def setUp(self):
-        name = self.mktemp()
-        os.mkdir(name)
-        FilePath(name).child("file").setContent(b"0123456789")
-        r = static.File(name)
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
         r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"error", ErrorResource())
@@ -246,8 +247,10 @@ class WebClientTestCase(unittest.TestCase):
         self.port = self._listen(self.wrapper)
         self.portno = self.port.getHost().port
 
+    @inlineCallbacks
     def tearDown(self):
-        return self.port.stopListening()
+        yield self.port.stopListening()
+        shutil.rmtree(self.tmpname)
 
     def getURL(self, path):
         return "http://127.0.0.1:%d/%s" % (self.portno, path)
@@ -266,7 +269,6 @@ class WebClientTestCase(unittest.TestCase):
             getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
                 self.assertEquals, to_bytes("www.example.com"))])
 
-
     def test_getPage(self):
         """
         L{client.getPage} returns a L{Deferred} which is called back with
@@ -276,7 +278,6 @@ class WebClientTestCase(unittest.TestCase):
         d.addCallback(self.assertEquals, b"0123456789")
         return d
 
-
     def test_getPageHead(self):
         """
         L{client.getPage} returns a L{Deferred} which is called back with
@@ -289,7 +290,6 @@ class WebClientTestCase(unittest.TestCase):
             _getPage("head").addCallback(self.assertEqual, b""),
             _getPage("HEAD").addCallback(self.assertEqual, b"")])
 
-
     def test_timeoutNotTriggering(self):
         """
         When a non-zero timeout is passed to L{getPage} and the page is
@@ -301,7 +301,6 @@ class WebClientTestCase(unittest.TestCase):
             self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
         return d
 
-
     def test_timeoutTriggering(self):
         """
         When a non-zero timeout is passed to L{getPage} and that many