diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 82857c9da..dcf8af365 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -252,9 +252,33 @@ If the handy ``has_next`` element is ``true`` (try loading `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a page-number greater than 10), we increment the ``page`` attribute and ``yield`` a new request, inserting the incremented page-number -into our ``url``. +into our ``url``. -You can see that with a few inspections in the `Network`-tool we +.. _requests-from-curl: + +In more complex websites, it could be difficult to easily reproduce the +requests, as we could need to add ``headers`` or ``cookies`` to make it work. +In those cases you can export the requests in `cURL `_ +format, by right-clicking on each of them in the network tool and using the +:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent +request:: + + from scrapy import Request + + request = Request.from_curl( + "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil" + "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce" + "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X" + "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" + "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW" + "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http" + "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") + +Alternatively, if you want to know the arguments needed to recreate that +request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs` +function to get a dictionary with the equivalent arguments. + +As you can see, with a few inspections in the `Network`-tool we were able to easily replicate the dynamic requests of the scrolling functionality of the page. Crawling dynamic pages can be quite daunting and pages can be very complex, but it (mostly) boils down @@ -262,7 +286,7 @@ to identifying the correct request and replicating it in your spider. .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools .. _quotes.toscrape.com: http://quotes.toscrape.com -.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/ +.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll .. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 8b5dacf56..8334ddcec 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -85,6 +85,13 @@ It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and form parameters (see :class:`~scrapy.http.FormRequest`) of that request. +As all major browsers allow to export the requests in `cURL +`_ format, Scrapy incorporates the method +:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent +:class:`~scrapy.http.Request` from a cURL command. To get more information +visit :ref:`request from curl ` inside the network +tool section. + Once you get the expected response, you can :ref:`extract the desired data from it `. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4e81ce878..9a5c65b0d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -194,6 +194,8 @@ Request objects copied by default (unless new values are given as arguments). See also :ref:`topics-request-response-ref-request-callback-arguments`. + .. automethod:: from_curl + .. _topics-request-response-ref-request-callback-arguments: Passing additional data to callback functions diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index f5935c4ef..d09eaf849 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,6 +12,7 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax from scrapy.http.common import obsolete_setter +from scrapy.utils.curl import curl_to_request_kwargs class Request(object_ref): @@ -103,3 +104,34 @@ class Request(object_ref): kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) + + @classmethod + def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): + """Create a Request object from a string containing a `cURL + `_ command. It populates the HTTP method, the + URL, the headers, the cookies and the body. It accepts the same + arguments as the :class:`Request` class, taking preference and + overriding the values of the same arguments contained in the cURL + command. + + Unrecognized options are ignored by default. To raise an error when + finding unknown options call this method by passing + ``ignore_unknown_options=False``. + + .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request` + subclasses, such as :class:`~scrapy.http.JSONRequest`, or + :class:`~scrapy.http.XmlRpcRequest`, as well as having + :ref:`downloader middlewares ` + and + :ref:`spider middlewares ` + enabled, such as + :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`, + :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`, + or + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`, + may modify the :class:`~scrapy.http.Request` object. + + """ + request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) + request_kwargs.update(kwargs) + return cls(**request_kwargs) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py new file mode 100644 index 000000000..b3fd0a497 --- /dev/null +++ b/scrapy/utils/curl.py @@ -0,0 +1,95 @@ +import argparse +import warnings +from shlex import split + +from six.moves.http_cookies import SimpleCookie +from six.moves.urllib.parse import urlparse +from six import string_types, iteritems +from w3lib.http import basic_auth_header + + +class CurlParser(argparse.ArgumentParser): + def error(self, message): + error_msg = \ + 'There was an error parsing the curl command: {}'.format(message) + raise ValueError(error_msg) + + +curl_parser = CurlParser() +curl_parser.add_argument('url') +curl_parser.add_argument('-H', '--header', dest='headers', action='append') +curl_parser.add_argument('-X', '--request', dest='method', default='get') +curl_parser.add_argument('-d', '--data', dest='data') +curl_parser.add_argument('-u', '--user', dest='auth') + + +safe_to_ignore_arguments = [ + ['--compressed'], + # `--compressed` argument is not safe to ignore, but it's included here + # because the `HttpCompressionMiddleware` is enabled by default + ['-s', '--silent'], + ['-v', '--verbose'], + ['-#', '--progress-bar'] +] + +for argument in safe_to_ignore_arguments: + curl_parser.add_argument(*argument, action='store_true') + + +def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): + """Convert a cURL command syntax to Request kwargs. + + :param str curl_command: string containing the curl command + :param bool ignore_unknown_options: If true, only a warning is emitted when + cURL options are unknown. Otherwise raises an error. (default: True) + :return: dictionary of Request kwargs + """ + + curl_args = split(curl_command) + + if curl_args[0] != 'curl': + raise ValueError('A curl command must start with "curl"') + + parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) + + if argv: + msg = 'Unrecognized options: {}'.format(', '.join(argv)) + if ignore_unknown_options: + warnings.warn(msg) + else: + raise ValueError(msg) + + url = parsed_args.url + + # curl automatically prepends 'http' if the scheme is missing, but Request + # needs the scheme to work + parsed_url = urlparse(url) + if not parsed_url.scheme: + url = 'http://' + url + + result = {'method': parsed_args.method.upper(), 'url': url} + + headers = [] + cookies = {} + for header in parsed_args.headers or (): + name, val = header.split(':', 1) + name = name.strip() + val = val.strip() + if name.title() == 'Cookie': + for name, morsel in iteritems(SimpleCookie(val)): + cookies[name] = morsel.value + else: + headers.append((name, val)) + + if parsed_args.auth: + user, password = parsed_args.auth.split(':', 1) + headers.append(('Authorization', basic_auth_header(user, password))) + + if headers: + result['headers'] = headers + if cookies: + result['cookies'] = cookies + if parsed_args.data: + result['body'] = parsed_args.data + + return result diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 952e208de..60494d792 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -269,6 +269,82 @@ class RequestTest(unittest.TestCase): with self.assertRaises(TypeError): self.request_class('http://example.com', a_function, errback='a_function') + def test_from_curl(self): + # Note: more curated tests regarding curl conversion are in + # `test_utils_curl.py` + curl_command = ( + "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" + "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" + "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" + " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" + "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" + "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" + "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" + "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode" + "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=" + "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0" + "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee" + "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith" + "%40example.org&size=small&topping=cheese&topping=onion&delivery=1" + "2%3A15&comments=' --compressed" + ) + r = self.request_class.from_curl(curl_command) + self.assertEqual(r.method, "POST") + self.assertEqual(r.url, "http://httpbin.org/post") + self.assertEqual(r.body, + b"custname=John+Smith&custtel=500&custemail=jsmith%40" + b"example.org&size=small&topping=cheese&topping=onion" + b"&delivery=12%3A15&comments=") + self.assertEqual(r.cookies, { + '_gauges_unique_year': '1', + '_gauges_unique': '1', + '_gauges_unique_month': '1', + '_gauges_unique_hour': '1', + '_gauges_unique_day': '1' + }) + self.assertEqual(r.headers, { + b'Origin': [b'http://httpbin.org'], + b'Accept-Encoding': [b'gzip, deflate'], + b'Accept-Language': [b'en-US,en;q=0.9,ru;q=0.8,es;q=0.7'], + b'Upgrade-Insecure-Requests': [b'1'], + b'User-Agent': [b'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.' + b'36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202' + b'.75 Chrome/62.0.3202.75 Safari/537.36'], + b'Content-Type': [b'application /x-www-form-urlencoded'], + b'Accept': [b'text/html,application/xhtml+xml,application/xml;q=0.' + b'9,image/webp,image/apng,*/*;q=0.8'], + b'Cache-Control': [b'max-age=0'], + b'Referer': [b'http://httpbin.org/forms/post'], + b'Connection': [b'keep-alive']}) + + def test_from_curl_with_kwargs(self): + r = self.request_class.from_curl( + 'curl -X PATCH "http://example.org"', + method="POST", + meta={'key': 'value'} + ) + self.assertEqual(r.method, "POST") + self.assertEqual(r.meta, {"key": "value"}) + + def test_from_curl_ignore_unknown_options(self): + # By default: it works and ignores the unknown options: --foo and -z + with warnings.catch_warnings(): # avoid warning when executing tests + warnings.simplefilter('ignore') + r = self.request_class.from_curl( + 'curl -X DELETE "http://example.org" --foo -z', + ) + self.assertEqual(r.method, "DELETE") + + # If `ignore_unknon_options` is set to `False` it raises an error with + # the unknown options: --foo and -z + self.assertRaises( + ValueError, + lambda: self.request_class.from_curl( + 'curl -X PATCH "http://example.org" --foo -z', + ignore_unknown_options=False, + ), + ) + class FormRequestTest(RequestTest): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py new file mode 100644 index 000000000..c5655df7e --- /dev/null +++ b/tests/test_utils_curl.py @@ -0,0 +1,211 @@ +import unittest +import warnings + +from six import assertRaisesRegex +from w3lib.http import basic_auth_header + +from scrapy import Request +from scrapy.utils.curl import curl_to_request_kwargs + + +class CurlToRequestKwargsTest(unittest.TestCase): + maxDiff = 5000 + + def _test_command(self, curl_command, expected_result): + result = curl_to_request_kwargs(curl_command) + self.assertEqual(result, expected_result) + try: + Request(**result) + except TypeError as e: + self.fail("Request kwargs are not correct {}".format(e)) + + def test_get(self): + curl_command = "curl http://example.org/" + expected_result = {"method": "GET", "url": "http://example.org/"} + self._test_command(curl_command, expected_result) + + def test_get_without_scheme(self): + curl_command = "curl www.example.org" + expected_result = {"method": "GET", "url": "http://www.example.org"} + self._test_command(curl_command, expected_result) + + def test_get_basic_auth(self): + curl_command = 'curl "https://api.test.com/" -u ' \ + '"some_username:some_password"' + expected_result = { + "method": "GET", + "url": "https://api.test.com/", + "headers": [ + ( + "Authorization", + basic_auth_header("some_username", "some_password") + ) + ], + } + self._test_command(curl_command, expected_result) + + def test_get_complex(self): + curl_command = ( + "curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'" + " -H 'Accept-Language: en-US,en;q=0.9,ru;q=0.8,es;q=0.7' -H 'Upgra" + "de-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (X11; Linux " + "x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/62" + ".0.3202.75 Chrome/62.0.3202.75 Safari/537.36' -H 'Accept: text/ht" + "ml,application/xhtml+xml,application/xml;q=0.9,image/webp,image/a" + "png,*/*;q=0.8' -H 'Referer: http://httpbin.org/' -H 'Cookie: _gau" + "ges_unique_year=1; _gauges_unique=1; _gauges_unique_month=1; _gau" + "ges_unique_hour=1; _gauges_unique_day=1' -H 'Connection: keep-ali" + "ve' --compressed" + ) + expected_result = { + "method": "GET", + "url": "http://httpbin.org/get", + "headers": [ + ("Accept-Encoding", "gzip, deflate"), + ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"), + ("Upgrade-Insecure-Requests", "1"), + ( + "User-Agent", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML" + ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32" + "02.75 Safari/537.36", + ), + ( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,ima" + "ge/webp,image/apng,*/*;q=0.8", + ), + ("Referer", "http://httpbin.org/"), + ("Connection", "keep-alive"), + ], + "cookies": { + '_gauges_unique_year': '1', + '_gauges_unique_hour': '1', + '_gauges_unique_day': '1', + '_gauges_unique': '1', + '_gauges_unique_month': '1' + }, + } + self._test_command(curl_command, expected_result) + + def test_post(self): + curl_command = ( + "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique" + "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique" + "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H" + " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q" + "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use" + "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM" + "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S" + "afari/537.36' -H 'Content-Type: application/x-www-form-urlencoded" + "' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0" + ".9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0'" + " -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: keep" + "-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith%" + "40example.org&size=small&topping=cheese&topping=onion&delivery=12" + "%3A15&comments=' --compressed" + ) + expected_result = { + "method": "POST", + "url": "http://httpbin.org/post", + "body": "custname=John+Smith&custtel=500&custemail=jsmith%40exampl" + "e.org&size=small&topping=cheese&topping=onion&delivery=12" + "%3A15&comments=", + "cookies": { + '_gauges_unique_year': '1', + '_gauges_unique_hour': '1', + '_gauges_unique_day': '1', + '_gauges_unique': '1', + '_gauges_unique_month': '1' + }, + "headers": [ + ("Origin", "http://httpbin.org"), + ("Accept-Encoding", "gzip, deflate"), + ("Accept-Language", "en-US,en;q=0.9,ru;q=0.8,es;q=0.7"), + ("Upgrade-Insecure-Requests", "1"), + ( + "User-Agent", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML" + ", like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.32" + "02.75 Safari/537.36", + ), + ("Content-Type", "application/x-www-form-urlencoded"), + ( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,ima" + "ge/webp,image/apng,*/*;q=0.8", + ), + ("Cache-Control", "max-age=0"), + ("Referer", "http://httpbin.org/forms/post"), + ("Connection", "keep-alive"), + ], + } + self._test_command(curl_command, expected_result) + + def test_patch(self): + curl_command = ( + 'curl "https://example.com/api/fake" -u "username:password" -H "Ac' + 'cept: application/vnd.go.cd.v4+json" -H "Content-Type: applicatio' + 'n/json" -X PATCH -d \'{"hostname": "agent02.example.com", "agent' + '_config_state": "Enabled", "resources": ["Java","Linux"], "enviro' + 'nments": ["Dev"]}\'' + ) + expected_result = { + "method": "PATCH", + "url": "https://example.com/api/fake", + "headers": [ + ("Accept", "application/vnd.go.cd.v4+json"), + ("Content-Type", "application/json"), + ("Authorization", basic_auth_header("username", "password")), + ], + "body": '{"hostname": "agent02.example.com", "agent_config_state"' + ': "Enabled", "resources": ["Java","Linux"], "environments' + '": ["Dev"]}', + } + self._test_command(curl_command, expected_result) + + def test_delete(self): + curl_command = 'curl -X "DELETE" https://www.url.com/page' + expected_result = { + "method": "DELETE", "url": "https://www.url.com/page" + } + self._test_command(curl_command, expected_result) + + def test_get_silent(self): + curl_command = 'curl --silent "www.example.com"' + expected_result = {"method": "GET", "url": "http://www.example.com"} + self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + + def test_too_few_arguments_error(self): + assertRaisesRegex( + self, + ValueError, + r"too few arguments|the following arguments are required:\s*url", + lambda: curl_to_request_kwargs("curl"), + ) + + def test_ignore_unknown_options(self): + # case 1: ignore_unknown_options=True: + with warnings.catch_warnings(): # avoid warning when executing tests + warnings.simplefilter('ignore') + curl_command = 'curl --bar --baz http://www.example.com' + expected_result = \ + {"method": "GET", "url": "http://www.example.com"} + self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) + + # case 2: ignore_unknown_options=False (raise exception): + assertRaisesRegex( + self, + ValueError, + "Unrecognized options:.*--bar.*--baz", + lambda: curl_to_request_kwargs( + "curl --bar --baz http://www.example.com", + ignore_unknown_options=False + ), + ) + + def test_must_start_with_curl_error(self): + self.assertRaises( + ValueError, + lambda: curl_to_request_kwargs("carl -X POST http://example.org") + )