mirror of https://github.com/scrapy/scrapy.git
[docs/item_pipeline] process_item returning Deferred docs
* quote url * use hash of url as filename
This commit is contained in:
parent
d867268976
commit
39dbd890b0
|
|
@ -178,6 +178,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
::
|
||||
|
||||
import scrapy
|
||||
import hashlib
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
class ScreenshotPipeline(object):
|
||||
|
|
@ -187,8 +189,8 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
SPLASH_URL = "http://localhost:8050/render.png?url={}"
|
||||
|
||||
def process_item(self, item, spider):
|
||||
item_url = item["url"]
|
||||
screenshot_url = self.SPLASH_URL.format(item_url)
|
||||
encoded_item_url = quote(item["url"])
|
||||
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
|
||||
request = scrapy.Request(screenshot_url)
|
||||
dfd = spider.crawler.engine.download(request, spider)
|
||||
dfd.addBoth(self.return_item, item)
|
||||
|
|
@ -199,8 +201,10 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
# Error happened, return item.
|
||||
return item
|
||||
|
||||
# Save screenshot to file.
|
||||
filename = "item_file_name.png"
|
||||
# Save screenshot to file, filename will be hash of url.
|
||||
url = item["url"]
|
||||
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
|
||||
filename = "{}.png".format(url_hash)
|
||||
with open(filename, "wb") as f:
|
||||
f.write(response.body)
|
||||
|
||||
|
|
@ -208,7 +212,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
|
|||
item["screenshot_filename"] = filename
|
||||
return item
|
||||
|
||||
|
||||
.. _Splash: http://splash.readthedocs.io/en/stable/
|
||||
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue