From a25cf5c82f99f7ae11346a2e565d6255835c3814 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 20 Nov 2018 16:13:09 +0000 Subject: [PATCH] function to get unique file queues for any type of base queue --- scrapy/core/queues.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scrapy/core/queues.py diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py new file mode 100644 index 000000000..96d582fc7 --- /dev/null +++ b/scrapy/core/queues.py @@ -0,0 +1,15 @@ +import uuid +import os.path + + +def unique_files_queue(queue_class): + + class UniqueFilesQueue(queue_class): + def __init__(self, path): + path = path + "-" + uuid.uuid4().hex + while os.path.exists(path): + path = path + "-" + uuid.uuid4().hex + + super().__init__(path) + + return UniqueFilesQueue