Create Request from curl command (#3862)

docs/topics/developer-tools.rst

@@ -252,9 +252,33 @@ If the handy ``has_next`` element is ``true`` (try loading
 `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
 page-number greater than 10), we increment the ``page`` attribute 
 and ``yield`` a new request, inserting the incremented page-number 
-into our ``url``. 
+into our ``url``.
 
-You can see that with a few inspections in the `Network`-tool we 
+.. _requests-from-curl:
+
+In more complex websites, it could be difficult to easily reproduce the
+requests, as we could need to add ``headers`` or ``cookies`` to make it work.
+In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
+format, by right-clicking on each of them in the network tool and using the
+:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
+request::
+
+    from scrapy import Request
+
+    request = Request.from_curl(
+        "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
+        "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
+        "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
+        "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
+        "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
+        "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
+        "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
+
+Alternatively, if you want to know the arguments needed to recreate that
+request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments.
+
+As you can see, with a few inspections in the `Network`-tool we
 were able to easily replicate the dynamic requests of the scrolling 
 functionality of the page. Crawling dynamic pages can be quite
 daunting and pages can be very complex, but it (mostly) boils down
@@ -262,7 +286,7 @@ to identifying the correct request and replicating it in your spider.
 
 .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
 .. _quotes.toscrape.com: http://quotes.toscrape.com
-.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/
+.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
 .. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
 

docs/topics/dynamic-content.rst

@@ -85,6 +85,13 @@ It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
 method and URL. However, you may also need to reproduce the body, headers and
 form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
 
+As all major browsers allow to export the requests in `cURL
+<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
+:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
+:class:`~scrapy.http.Request` from a cURL command. To get more information
+visit :ref:`request from curl <requests-from-curl>` inside the network
+tool section.
+
 Once you get the expected response, you can :ref:`extract the desired data from
 it <topics-handling-response-formats>`.
 

docs/topics/request-response.rst

@@ -194,6 +194,8 @@ Request objects
        copied by default (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
+    .. automethod:: from_curl
+
 .. _topics-request-response-ref-request-callback-arguments:
 
 Passing additional data to callback functions

scrapy/http/request/__init__.py

@@ -12,6 +12,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 from scrapy.http.common import obsolete_setter
+from scrapy.utils.curl import curl_to_request_kwargs
 
 
 class Request(object_ref):
@@ -103,3 +104,34 @@ class Request(object_ref):
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)
+
+    @classmethod
+    def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+        """Create a Request object from a string containing a `cURL
+        <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
+        URL, the headers, the cookies and the body. It accepts the same
+        arguments as the :class:`Request` class, taking preference and
+        overriding the values of the same arguments contained in the cURL
+        command.
+
+        Unrecognized options are ignored by default. To raise an error when
+        finding unknown options call this method by passing
+        ``ignore_unknown_options=False``.
+
+        .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
+                     subclasses, such as :class:`~scrapy.http.JSONRequest`, or
+                     :class:`~scrapy.http.XmlRpcRequest`, as well as having
+                     :ref:`downloader middlewares <topics-downloader-middleware>`
+                     and
+                     :ref:`spider middlewares <topics-spider-middleware>`
+                     enabled, such as
+                     :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`,
+                     :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`,
+                     or
+                     :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
+                     may modify the :class:`~scrapy.http.Request` object.
+
+       """
+        request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
+        request_kwargs.update(kwargs)
+        return cls(**request_kwargs)

scrapy/utils/curl.py

@@ -0,0 +1,95 @@
+import argparse
+import warnings
+from shlex import split
+
+from six.moves.http_cookies import SimpleCookie
+from six.moves.urllib.parse import urlparse
+from six import string_types, iteritems
+from w3lib.http import basic_auth_header
+
+
+class CurlParser(argparse.ArgumentParser):
+    def error(self, message):
+        error_msg = \
+            'There was an error parsing the curl command: {}'.format(message)
+        raise ValueError(error_msg)
+
+
+curl_parser = CurlParser()
+curl_parser.add_argument('url')
+curl_parser.add_argument('-H', '--header', dest='headers', action='append')
+curl_parser.add_argument('-X', '--request', dest='method', default='get')
+curl_parser.add_argument('-d', '--data', dest='data')
+curl_parser.add_argument('-u', '--user', dest='auth')
+
+
+safe_to_ignore_arguments = [
+    ['--compressed'],
+    # `--compressed` argument is not safe to ignore, but it's included here
+    # because the `HttpCompressionMiddleware` is enabled by default
+    ['-s', '--silent'],
+    ['-v', '--verbose'],
+    ['-#', '--progress-bar']
+]
+
+for argument in safe_to_ignore_arguments:
+    curl_parser.add_argument(*argument, action='store_true')
+
+
+def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
+    """Convert a cURL command syntax to Request kwargs.
+
+    :param str curl_command: string containing the curl command
+    :param bool ignore_unknown_options: If true, only a warning is emitted when
+    cURL options are unknown. Otherwise raises an error. (default: True)
+    :return: dictionary of Request kwargs
+    """
+
+    curl_args = split(curl_command)
+
+    if curl_args[0] != 'curl':
+        raise ValueError('A curl command must start with "curl"')
+
+    parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
+
+    if argv:
+        msg = 'Unrecognized options: {}'.format(', '.join(argv))
+        if ignore_unknown_options:
+            warnings.warn(msg)
+        else:
+            raise ValueError(msg)
+
+    url = parsed_args.url
+
+    # curl automatically prepends 'http' if the scheme is missing, but Request
+    # needs the scheme to work
+    parsed_url = urlparse(url)
+    if not parsed_url.scheme:
+        url = 'http://' + url
+
+    result = {'method': parsed_args.method.upper(), 'url': url}
+
+    headers = []
+    cookies = {}
+    for header in parsed_args.headers or ():
+        name, val = header.split(':', 1)
+        name = name.strip()
+        val = val.strip()
+        if name.title() == 'Cookie':
+            for name, morsel in iteritems(SimpleCookie(val)):
+                cookies[name] = morsel.value
+        else:
+            headers.append((name, val))
+
+    if parsed_args.auth:
+        user, password = parsed_args.auth.split(':', 1)
+        headers.append(('Authorization', basic_auth_header(user, password)))
+
+    if headers:
+        result['headers'] = headers
+    if cookies:
+        result['cookies'] = cookies
+    if parsed_args.data:
+        result['body'] = parsed_args.data
+
+    return result

tests/test_http_request.py

@@ -269,6 +269,82 @@ class RequestTest(unittest.TestCase):
         with self.assertRaises(TypeError):
             self.request_class('http://example.com', a_function, errback='a_function')
 
+    def test_from_curl(self):
+        # Note: more curated tests regarding curl conversion are in
+        # `test_utils_curl.py`
+        curl_command = (
+            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
+            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
+            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
+            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
+            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
+            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
+            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
+            "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
+            "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
+            "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
+            "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
+            "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
+            "%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
+            "2%3A15&comments=' --compressed"
+        )
+        r = self.request_class.from_curl(curl_command)
+        self.assertEqual(r.method, "POST")
+        self.assertEqual(r.url, "http://httpbin.org/post")
+        self.assertEqual(r.body,
+                         b"custname=John+Smith&custtel=500&custemail=jsmith%40"
+                         b"example.org&size=small&topping=cheese&topping=onion"
+                         b"&delivery=12%3A15&comments=")
+        self.assertEqual(r.cookies, {
+            '_gauges_unique_year': '1',
+            '_gauges_unique': '1',
+            '_gauges_unique_month': '1',
+            '_gauges_unique_hour': '1',
+            '_gauges_unique_day': '1'
+        })
+        self.assertEqual(r.headers, {
+            b'Origin': [b'http://httpbin.org'],
+            b'Accept-Encoding': [b'gzip, deflate'],
+            b'Accept-Language': [b'en-US,en;q=0.9,ru;q=0.8,es;q=0.7'],
+            b'Upgrade-Insecure-Requests': [b'1'],
+            b'User-Agent': [b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.'
+                            b'36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202'
+                            b'.75 Chrome/62.0.3202.75 Safari/537.36'],
+            b'Content-Type': [b'application /x-www-form-urlencoded'],
+            b'Accept': [b'text/html,application/xhtml+xml,application/xml;q=0.'
+                        b'9,image/webp,image/apng,*/*;q=0.8'],
+            b'Cache-Control': [b'max-age=0'],
+            b'Referer': [b'http://httpbin.org/forms/post'],
+            b'Connection': [b'keep-alive']})
+
+    def test_from_curl_with_kwargs(self):
+        r = self.request_class.from_curl(
+            'curl -X PATCH "http://example.org"',
+            method="POST",
+            meta={'key': 'value'}
+        )
+        self.assertEqual(r.method, "POST")
+        self.assertEqual(r.meta, {"key": "value"})
+
+    def test_from_curl_ignore_unknown_options(self):
+        # By default: it works and ignores the unknown options: --foo and -z
+        with warnings.catch_warnings():  # avoid warning when executing tests
+            warnings.simplefilter('ignore')
+            r = self.request_class.from_curl(
+                'curl -X DELETE "http://example.org" --foo -z',
+            )
+            self.assertEqual(r.method, "DELETE")
+
+        # If `ignore_unknon_options` is set to `False` it raises an error with
+        # the unknown options: --foo and -z
+        self.assertRaises(
+            ValueError,
+            lambda: self.request_class.from_curl(
+                'curl -X PATCH "http://example.org" --foo -z',
+                ignore_unknown_options=False,
+            ),
+        )
+
 
 class FormRequestTest(RequestTest):
 

tests/test_utils_curl.py

@@ -0,0 +1,211 @@
+import unittest
+import warnings
+
+from six import assertRaisesRegex
+from w3lib.http import basic_auth_header
+
+from scrapy import Request
+from scrapy.utils.curl import curl_to_request_kwargs
+
+
+class CurlToRequestKwargsTest(unittest.TestCase):
+    maxDiff = 5000
+
+    def _test_command(self, curl_command, expected_result):
+        result = curl_to_request_kwargs(curl_command)
+        self.assertEqual(result, expected_result)
+        try:
+            Request(**result)
+        except TypeError as e:
+            self.fail("Request kwargs are not correct {}".format(e))
+
+    def test_get(self):
+        curl_command = "curl http://example.org/"
+        expected_result = {"method": "GET", "url": "http://example.org/"}
+        self._test_command(curl_command, expected_result)
+
+    def test_get_without_scheme(self):
+        curl_command = "curl www.example.org"
+        expected_result = {"method": "GET", "url": "http://www.example.org"}
+        self._test_command(curl_command, expected_result)
+
+    def test_get_basic_auth(self):
+        curl_command = 'curl "https://api.test.com/" -u ' \
+                       '"some_username:some_password"'
+        expected_result = {
+            "method": "GET",
+            "url": "https://api.test.com/",
+            "headers": [
+                (
+                    "Authorization",
+                    basic_auth_header("some_username", "some_password")
+                )
+            ],
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_get_complex(self):
+        curl_command = (
+            "curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'"
+            " -H 'Accept-Language: en-US,en;q=0.9,ru;q=0.8,es;q=0.7' -H 'Upgra"
+            "de-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux "
+            "x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/62"
+            ".0.3202.75 Chrome/62.0.3202.75 Safari/537.36' -H 'Accept: text/ht"
+            "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a"
+            "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau"
+            "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau"
+            "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali"
+            "ve' --compressed"
+        )
+        expected_result = {
+            "method": "GET",
+            "url": "http://httpbin.org/get",
+            "headers": [
+                ("Accept-Encoding", "gzip, deflate"),
+                ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
+                ("Upgrade-Insecure-Requests", "1"),
+                (
+                    "User-Agent",
+                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
+                    ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
+                    "02.75 Safari/537.36",
+                ),
+                (
+                    "Accept",
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
+                    "ge/webp,image/apng,*/*;q=0.8",
+                ),
+                ("Referer", "http://httpbin.org/"),
+                ("Connection", "keep-alive"),
+            ],
+            "cookies": {
+                '_gauges_unique_year': '1',
+                '_gauges_unique_hour': '1',
+                '_gauges_unique_day': '1',
+                '_gauges_unique': '1',
+                '_gauges_unique_month': '1'
+            },
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_post(self):
+        curl_command = (
+            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
+            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
+            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
+            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
+            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
+            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
+            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
+            "afari/537.36' -H 'Content-Type: application/x-www-form-urlencoded"
+            "' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0"
+            ".9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0'"
+            " -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: keep"
+            "-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith%"
+            "40example.org&size=small&topping=cheese&topping=onion&delivery=12"
+            "%3A15&comments=' --compressed"
+        )
+        expected_result = {
+            "method": "POST",
+            "url": "http://httpbin.org/post",
+            "body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl"
+                    "e.org&size=small&topping=cheese&topping=onion&delivery=12"
+                    "%3A15&comments=",
+            "cookies": {
+                '_gauges_unique_year': '1',
+                '_gauges_unique_hour': '1',
+                '_gauges_unique_day': '1',
+                '_gauges_unique': '1',
+                '_gauges_unique_month': '1'
+            },
+            "headers": [
+                ("Origin", "http://httpbin.org"),
+                ("Accept-Encoding", "gzip, deflate"),
+                ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"),
+                ("Upgrade-Insecure-Requests", "1"),
+                (
+                    "User-Agent",
+                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML"
+                    ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32"
+                    "02.75 Safari/537.36",
+                ),
+                ("Content-Type", "application/x-www-form-urlencoded"),
+                (
+                    "Accept",
+                    "text/html,application/xhtml+xml,application/xml;q=0.9,ima"
+                    "ge/webp,image/apng,*/*;q=0.8",
+                ),
+                ("Cache-Control", "max-age=0"),
+                ("Referer", "http://httpbin.org/forms/post"),
+                ("Connection", "keep-alive"),
+            ],
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_patch(self):
+        curl_command = (
+            'curl "https://example.com/api/fake" -u "username:password" -H "Ac'
+            'cept: application/vnd.go.cd.v4+json" -H "Content-Type: applicatio'
+            'n/json" -X PATCH -d \'{"hostname": "agent02.example.com",  "agent'
+            '_config_state": "Enabled", "resources": ["Java","Linux"], "enviro'
+            'nments": ["Dev"]}\''
+        )
+        expected_result = {
+            "method": "PATCH",
+            "url": "https://example.com/api/fake",
+            "headers": [
+                ("Accept", "application/vnd.go.cd.v4+json"),
+                ("Content-Type", "application/json"),
+                ("Authorization", basic_auth_header("username", "password")),
+            ],
+            "body": '{"hostname": "agent02.example.com",  "agent_config_state"'
+                    ': "Enabled", "resources": ["Java","Linux"], "environments'
+                    '": ["Dev"]}',
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_delete(self):
+        curl_command = 'curl -X "DELETE" https://www.url.com/page'
+        expected_result = {
+            "method": "DELETE", "url": "https://www.url.com/page"
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_get_silent(self):
+        curl_command = 'curl --silent "www.example.com"'
+        expected_result = {"method": "GET", "url": "http://www.example.com"}
+        self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
+
+    def test_too_few_arguments_error(self):
+        assertRaisesRegex(
+            self,
+            ValueError,
+            r"too few arguments|the following arguments are required:\s*url",
+            lambda: curl_to_request_kwargs("curl"),
+        )
+
+    def test_ignore_unknown_options(self):
+        # case 1: ignore_unknown_options=True:
+        with warnings.catch_warnings():  # avoid warning when executing tests
+            warnings.simplefilter('ignore')
+            curl_command = 'curl --bar --baz http://www.example.com'
+            expected_result = \
+                {"method": "GET", "url": "http://www.example.com"}
+            self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
+
+        # case 2: ignore_unknown_options=False (raise exception):
+        assertRaisesRegex(
+            self,
+            ValueError,
+            "Unrecognized options:.*--bar.*--baz",
+            lambda: curl_to_request_kwargs(
+                "curl --bar --baz http://www.example.com",
+                ignore_unknown_options=False
+            ),
+        )
+
+    def test_must_start_with_curl_error(self):
+        self.assertRaises(
+            ValueError,
+            lambda: curl_to_request_kwargs("carl -X POST http://example.org")
+        )