From 29d6bcf0d10f929b501aac5bfbb33a1eede7a64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 20 Apr 2012 12:24:30 +0000 Subject: [PATCH] Workaround bug in lxml for multiple select options In lxml version pre 2.3.1 there is a bug that returns all options elements instead of those selected for select tag it is mentioned in 2.3.1 release notes http://lxml.de/2.3/changes-2.3.1.html and fixed by https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1R1139 --- scrapy/http/request/form.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index aa23eaec7..9f6897280 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -93,16 +93,25 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): def _value(ele): n = ele.name v = ele.value - # Match browser behaviour on simple select tag without options selected - # Or for select tags wihout options - if v is None and ele.tag == 'select' and not ele.multiple: - o = ele.value_options - if o: - return n, o[0] - else: - return None, None + if ele.tag == 'select': + return _select_value(ele, n, v) return n, v +def _select_value(ele, n, v): + multiple = ele.multiple + if v is None and not multiple: + # Match browser behaviour on simple select tag without options selected + # And for select tags wihout options + o = ele.value_options + return (n, o[0]) if o else (None, None) + elif v is not None and multiple: + # This is a workround to bug in lxml fixed 2.3.1 + # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 + selected_options = ele.xpath('.//option[@selected]') + v = [(o.get('value') or o.text or u'').strip() for o in selected_options] + return n, v + + def _get_clickable(clickdata, form): """ Returns the clickable element specified in clickdata,