[docs/item_pipeline] process_item returning Deferred docs

* quote url
* use hash of url as filename
This commit is contained in:
pawelmhm 2016-09-24 08:36:09 +02:00
parent d867268976
commit 39dbd890b0
1 changed files with 8 additions and 5 deletions

View File

@ -178,6 +178,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
::
import scrapy
import hashlib
from urllib.parse import quote
class ScreenshotPipeline(object):
@ -187,8 +189,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
SPLASH_URL = "http://localhost:8050/render.png?url={}"
def process_item(self, item, spider):
item_url = item["url"]
screenshot_url = self.SPLASH_URL.format(item_url)
encoded_item_url = quote(item["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
dfd = spider.crawler.engine.download(request, spider)
dfd.addBoth(self.return_item, item)
@ -199,8 +201,10 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
# Error happened, return item.
return item
# Save screenshot to file.
filename = "item_file_name.png"
# Save screenshot to file, filename will be hash of url.
url = item["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
with open(filename, "wb") as f:
f.write(response.body)
@ -208,7 +212,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
item["screenshot_filename"] = filename
return item
.. _Splash: http://splash.readthedocs.io/en/stable/
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html