RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

StopDownload: check partial response contents

This commit is contained in:
Eugenio Lacuesta 2020-05-20 20:20:30 -03:00
parent 9514393bd1
commit fbe915d9cb
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/spiders.py
View File

@ -293,10 +293,12 @@ class BytesReceivedCallbackSpider(MetaSpider):
self.meta["failure"] = failure
def bytes_received(self, data, request, spider):
self.meta["bytes_received"] = data
raise StopDownload(fail=False)
class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
def bytes_received(self, data, request, spider):
self.meta["bytes_received"] = data
raise StopDownload(fail=True)

6
tests/test_crawl.py
View File

@ -478,7 +478,8 @@ with multiples lines
yield crawler.crawl(mockserver=self.mockserver)
self.assertIsNone(crawler.spider.meta.get("failure"))
self.assertIsInstance(crawler.spider.meta["response"], Response)
self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length)
self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received"))
self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
@defer.inlineCallbacks
def test_stop_download_errback(self):
@ -488,4 +489,5 @@ with multiples lines
self.assertIsInstance(crawler.spider.meta["failure"], Failure)
self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
self.assertIsInstance(crawler.spider.meta["failure"].response, Response)
self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length)
self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received"))
self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length)