RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow yielding items from start_requests (#6417) 

Co-authored-by: Georgiy Zatserklianyi <george.zatseklyany@gmail.com>
Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
This commit is contained in:
Georgiy Zatserklianyi 2024-08-26 20:53:06 +02:00 committed by GitHub
parent 5794071f96
commit 6ce0342beb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 109 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
docs/topics/signals.rst
View File

@ -159,8 +159,9 @@ item_scraped
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.Spider` object
:param response: the response from where the item was scraped
:type response: :class:`~scrapy.http.Response` object
:param response: the response from where the item was scraped, or ``None``
if it was yielded from :meth:`~scrapy.Spider.start_requests`.
:type response: :class:`~scrapy.http.Response` | ``None``
item_dropped
~~~~~~~~~~~~
@ -179,8 +180,9 @@ item_dropped
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.Spider` object
:param response: the response from where the item was dropped
:type response: :class:`~scrapy.http.Response` object
:param response: the response from where the item was dropped, or ``None``
if it was yielded from :meth:`~scrapy.Spider.start_requests`.
:type response: :class:`~scrapy.http.Response` | ``None``
:param exception: the exception (which must be a
:exc:`~scrapy.exceptions.DropItem` subclass) which caused the item
@ -201,8 +203,10 @@ item_error
:param item: the item that caused the error in the :ref:`topics-item-pipeline`
:type item: :ref:`item object <item-types>`
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
:param response: the response being processed when the exception was
raised, or ``None`` if it was yielded from
:meth:`~scrapy.Spider.start_requests`.
:type response: :class:`~scrapy.http.Response` | ``None``
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object

2
docs/topics/spider-middleware.rst
View File

@ -176,7 +176,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
items).
It receives an iterable (in the ``start_requests`` parameter) and must
return another iterable of :class:`~scrapy.Request` objects.
return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects <topics-items>`.
.. note:: When implementing this method in your spider middleware, you
should always return an iterable (that follows the input one) and

3
docs/topics/spiders.rst
View File

@ -203,7 +203,8 @@ scrapy.Spider
.. method:: start_requests()
This method must return an iterable with the first Requests to crawl for
This method must return an iterable with the first Requests to crawl and/or with :ref:`item objects
<topics-items>` for
this spider. It is called by Scrapy when the spider is opened for
scraping. Scrapy calls it only once, so it is safe to implement
:meth:`start_requests` as a generator.

14
scrapy/core/engine.py
View File

@ -24,6 +24,7 @@ from typing import (
cast,
)
from itemadapter import is_item
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from twisted.internet.task import LoopingCall
from twisted.python.failure import Failure
@ -194,7 +195,7 @@ class ExecutionEngine:
if self.slot.start_requests is not None and not self._needs_backout():
try:
request = next(self.slot.start_requests)
request_or_item = next(self.slot.start_requests)
except StopIteration:
self.slot.start_requests = None
except Exception:
@ -205,7 +206,16 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
else:
self.crawl(request)
if isinstance(request_or_item, Request):
self.crawl(request_or_item)
elif is_item(request_or_item):
self.scraper.start_itemproc(request_or_item, response=None)
else:
logger.error(
f"Got {request_or_item!r} among start requests. Only "
f"requests and items are supported. It will be "
f"ignored."
)
if self.spider_is_idle() and self.slot.close_if_idle:
self._spider_idle()

21
scrapy/core/scraper.py
View File

@ -313,15 +313,11 @@ class Scraper:
"""Process each Request/Item (given in the output parameter) returned
from the given spider
"""
assert self.slot is not None # typing
if isinstance(output, Request):
assert self.crawler.engine is not None # typing
self.crawler.engine.crawl(request=output)
elif is_item(output):
self.slot.itemproc_size += 1
dfd = self.itemproc.process_item(output, spider)
dfd.addBoth(self._itemproc_finished, output, response, spider)
return dfd
return self.start_itemproc(output, response=response)
elif output is None:
pass
else:
@ -333,6 +329,19 @@ class Scraper:
)
return None
def start_itemproc(self, item, *, response: Optional[Response]) -> Deferred[Any]:
"""Send *item* to the item pipelines for processing.
*response* is the source of the item data. If the item does not come
from response data, e.g. it was hard-coded, set it to ``None``.
"""
assert self.slot is not None # typing
assert self.crawler.spider is not None # typing
self.slot.itemproc_size += 1
dfd = self.itemproc.process_item(item, self.crawler.spider)
dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider)
return dfd
def _log_download_errors(
self,
spider_failure: Failure,
@ -373,7 +382,7 @@ class Scraper:
return None
def _itemproc_finished(
self, output: Any, item: Any, response: Response, spider: Spider
self, output: Any, item: Any, response: Optional[Response], spider: Spider
) -> Deferred[Any]:
"""ItemProcessor finished for the given ``item`` and returned ``output``"""
assert self.slot is not None # typing

19
scrapy/logformatter.py
View File

@ -9,6 +9,7 @@ from twisted.python.failure import Failure
# working around https://github.com/sphinx-doc/sphinx/issues/10400
from scrapy import Request, Spider # noqa: TC001
from scrapy.http import Response # noqa: TC001
from scrapy.utils.python import global_object_name
from scrapy.utils.request import referer_str
if TYPE_CHECKING:
@ -92,11 +93,13 @@ class LogFormatter:
}
def scraped(
self, item: Any, response: Union[Response, Failure], spider: Spider
self, item: Any, response: Union[Response, Failure, None], spider: Spider
) -> LogFormatterResult:
"""Logs a message when an item is scraped by a spider."""
src: Any
if isinstance(response, Failure):
if response is None:
src = f"{global_object_name(spider.__class__)}.start_requests"
elif isinstance(response, Failure):
src = response.getErrorMessage()
else:
src = response
@ -110,7 +113,11 @@ class LogFormatter:
}
def dropped(
self, item: Any, exception: BaseException, response: Response, spider: Spider
self,
item: Any,
exception: BaseException,
response: Optional[Response],
spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item is dropped while it is passing through the item pipeline."""
return {
@ -123,7 +130,11 @@ class LogFormatter:
}
def item_error(
self, item: Any, exception: BaseException, response: Response, spider: Spider
self,
item: Any,
exception: BaseException,
response: Optional[Response],
spider: Spider,
) -> LogFormatterResult:
"""Logs a message when an item causes an error while it is passing
through the item pipeline.

13
tests/spiders.py
View File

@ -346,6 +346,19 @@ class BrokenStartRequestsSpider(FollowAllSpider):
yield from super().parse(response)
class StartRequestsItemSpider(FollowAllSpider):
def start_requests(self):
yield {"name": "test item"}
class StartRequestsGoodAndBadOutput(FollowAllSpider):
def start_requests(self):
yield {"a": "a"}
yield Request("data:,a")
yield "data:,b"
yield object()
class SingleRequestSpider(MetaSpider):
seed = None
callback_func = None

36
tests/test_crawl.py
View File

@ -1,5 +1,6 @@
import json
import logging
import re
import unittest
from ipaddress import IPv4Address
from socket import gethostbyname
@ -49,6 +50,8 @@ from tests.spiders import (
HeadersReceivedErrbackSpider,
SimpleSpider,
SingleRequestSpider,
StartRequestsGoodAndBadOutput,
StartRequestsItemSpider,
)
@ -184,6 +187,39 @@ class CrawlTestCase(TestCase):
self.assertIsNotNone(record.exc_info)
self.assertIs(record.exc_info[0], ZeroDivisionError)
@defer.inlineCallbacks
def test_start_requests_items(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(StartRequestsItemSpider)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(log.records), 0)
@defer.inlineCallbacks
def test_start_requests_unsupported_output(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(StartRequestsGoodAndBadOutput)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(log.records), 2)
self.assertEqual(
log.records[0].msg,
(
"Got 'data:,b' among start requests. Only requests and items "
"are supported. It will be ignored."
),
)
self.assertTrue(
re.match(
(
r"^Got <object object at 0x[0-9a-fA-F]+> among start "
r"requests\. Only requests and items are supported\. It "
r"will be ignored\.$"
),
log.records[1].msg,
)
)
@defer.inlineCallbacks
def test_start_requests_laziness(self):
settings = {"CONCURRENT_REQUESTS": 1}

9
tests/test_spidermiddleware.py
View File

@ -1,5 +1,5 @@
import collections.abc
from typing import Optional
from typing import Optional, Union
from unittest import mock
from testfixtures import LogCapture
@ -112,7 +112,7 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
Should work for process_spider_output and, when it's supported, process_start_requests.
"""
ITEM_TYPE: type
ITEM_TYPE: Union[type, tuple]
RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects
@staticmethod
@ -328,12 +328,13 @@ class ProcessStartRequestsSimpleMiddleware:
class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
"""process_start_requests tests for simple start_requests"""
ITEM_TYPE = Request
ITEM_TYPE = (Request, dict)
MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
def _start_requests(self):
for i in range(3):
for i in range(2):
yield Request(f"https://example.com/{i}", dont_filter=True)
yield {"name": "test item"}
@defer.inlineCallbacks
def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):