From 39dbd890b063af1b153824214fa973993560416c Mon Sep 17 00:00:00 2001 From: pawelmhm Date: Sat, 24 Sep 2016 08:36:09 +0200 Subject: [PATCH] [docs/item_pipeline] process_item returning Deferred docs * quote url * use hash of url as filename --- docs/topics/item-pipeline.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index a31fe7423..6b43fe258 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -178,6 +178,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite :: import scrapy + import hashlib + from urllib.parse import quote class ScreenshotPipeline(object): @@ -187,8 +189,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite SPLASH_URL = "http://localhost:8050/render.png?url={}" def process_item(self, item, spider): - item_url = item["url"] - screenshot_url = self.SPLASH_URL.format(item_url) + encoded_item_url = quote(item["url"]) + screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) dfd = spider.crawler.engine.download(request, spider) dfd.addBoth(self.return_item, item) @@ -199,8 +201,10 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite # Error happened, return item. return item - # Save screenshot to file. - filename = "item_file_name.png" + # Save screenshot to file, filename will be hash of url. + url = item["url"] + url_hash = hashlib.md5(url.encode("utf8")).hexdigest() + filename = "{}.png".format(url_hash) with open(filename, "wb") as f: f.write(response.body) @@ -208,7 +212,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite item["screenshot_filename"] = filename return item - .. _Splash: http://splash.readthedocs.io/en/stable/ .. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html