make selector re() method use re.UNICODE flag to compile regexes

scrapy/tests/test_selector.py

@@ -174,6 +174,13 @@ class XPathSelectorTestCase(unittest.TestCase):
         self.assertEqual(x.select("//ul/li").re("Age: (\d+)"),
                          ["10", "20"])
 
+    @libxml2debug
+    def test_selector_re_intl(self):
+        body = """<div>Evento: cumplea\xc3\xb1os</div>"""
+        response = HtmlResponse(url="http://example.com", body=body, encoding='utf-8')
+        x = self.hxs_cls(response)
+        self.assertEqual(x.select("//div").re("Evento: (\w+)"), [u'cumplea\xf1os'])
+
     @libxml2debug
     def test_selector_over_text(self):
         hxs = self.hxs_cls(text='<root>lala</root>')

scrapy/utils/misc.py

@@ -75,7 +75,7 @@ def extract_regex(regex, text, encoding='utf-8'):
     """
 
     if isinstance(regex, basestring):
-        regex = re.compile(regex)
+        regex = re.compile(regex, re.UNICODE)
 
     try:
         strings = [regex.search(text).group('extract')]   # named group