RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Also some typing for Scraper.

This commit is contained in:
Andrey Rakhmatullin 2021-04-04 16:15:33 +05:00
parent a0101361e2
commit 7dc857668f
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/core/scraper.py
View File

@ -3,12 +3,14 @@ extracts information from them"""
import logging
from collections import deque
from collections.abc import Iterable
from typing import Union
from itemadapter import is_item
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy import signals
from scrapy import signals, Spider
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
@ -120,7 +122,7 @@ class Scraper:
response, request, deferred = slot.next_response_request_deferred()
self._scrape(response, request, spider).chainDeferred(deferred)
def _scrape(self, result, request, spider):
def _scrape(self, result: Union[Response, Failure], request: Request, spider: Spider):
"""
Handle the downloaded response or failure through the spider callback/errback
"""
@ -131,7 +133,7 @@ class Scraper:
dfd.addCallback(self.handle_spider_output, request, result, spider)
return dfd
def _scrape2(self, result, request, spider):
def _scrape2(self, result: Union[Response, Failure], request: Request, spider: Spider):
"""
Handle the different cases of request's result been a Response or a Failure
"""
@ -141,7 +143,7 @@ class Scraper:
dfd = self.call_spider(result, request, spider)
return dfd.addErrback(self._log_download_errors, result, request, spider)
def call_spider(self, result, request, spider):
def call_spider(self, result: Union[Response, Failure], request: Request, spider: Spider):
if isinstance(result, Response):
if getattr(result, "request", None) is None:
result.request = request
@ -156,7 +158,7 @@ class Scraper:
dfd.addErrback(request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
def handle_spider_error(self, _failure: Failure, request: Request, response: Response, spider: Spider):
exc = _failure.value
if isinstance(exc, CloseSpider):
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
@ -177,7 +179,7 @@ class Scraper:
spider=spider
)
def handle_spider_output(self, result, request, response, spider):
def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response, spider)