RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed a couple of bugs caused by adding priority to Requests (thanks Artem for reporting)

This commit is contained in:
Pablo Hoffman 2009-06-12 08:31:30 -03:00
parent 4a1a01354b
commit 7c2476bb25
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -44,8 +44,8 @@ class RobotsTxtMiddleware(object):
else:
self._parsers[urldomain] = None
robotsurl = "%s://%s/robots.txt" % parsedurl[0:2]
robotsreq = Request(robotsurl)
dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain), priority=self.DOWNLOAD_PRIORITY)
robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
dfd = scrapyengine.schedule(robotsreq, spiders.fromdomain(spiderdomain))
dfd.addCallbacks(callback=self._parse_robots, callbackArgs=[urldomain])
self._spiderdomains[spiderdomain].add(urldomain)

3
scrapy/contrib/pipeline/media.py
View File

@ -122,7 +122,8 @@ class MediaPipeline(object):
request fingerprint is used as cache key.
"""
return scrapyengine.schedule(request, info.spider, priority=self.DOWNLOAD_PRIORITY)
request.priority = self.DOWNLOAD_PRIORITY
return scrapyengine.schedule(request, info.spider)
def media_to_download(self, request, info):
""" Ongoing request hook pre-cache