From 010edfe85caa72b4c366f2dada0f79f1f91e43ef Mon Sep 17 00:00:00 2001 From: Aditi Dutta Date: Wed, 25 Mar 2020 14:38:22 -0400 Subject: [PATCH] [Docs] mention curl2scrapy in Request.from_curl --- docs/topics/developer-tools.rst | 3 +++ docs/topics/dynamic-content.rst | 3 +++ scrapy/http/request/__init__.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index f1b0964c6..4e87a00f2 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -292,6 +292,9 @@ Alternatively, if you want to know the arguments needed to recreate that request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` function to get a dictionary with the equivalent arguments. +Note that to translate a cURL command into a Scrapy request, +you may use `curl2scrapy `_. + As you can see, with a few inspections in the `Network`-tool we were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index b98133676..aa326868b 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be buggy, overloaded, or :ref:`banning ` some of your requests. +Note that to translate a cURL command into a Scrapy request, +you may use `curl2scrapy `_. + .. _topics-handling-response-formats: Handling different response formats diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 6c536cb71..0a6637af8 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -129,6 +129,9 @@ class Request(object_ref): :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, may modify the :class:`~scrapy.http.Request` object. + To translate a cURL command into a Scrapy request, + you may use `curl2scrapy `_. + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs)