diff --git a/docs/reference/filtering.md b/docs/reference/filtering.md index 6d3672580..14f14b46f 100644 --- a/docs/reference/filtering.md +++ b/docs/reference/filtering.md @@ -104,9 +104,12 @@ GET /api/dcim/devices/?name__ic=switch !!! note "Case-insensitive matching depends on the field's collation" Most `name` fields use a database collation which sorts them in natural order, so that `device-2` precedes `device-10`. Case-insensitive matching on those fields follows the - same collation, which treats the German `ß` and `ss` as equivalent: a search for - `Strasse` matches a device named `Straße`, and vice versa. Fields which do not use this - collation, such as `serial` and `description`, match these characters literally. + same collation, which treats a character as equivalent to the sequence it expands to in + upper case. The German `ß` is the common example: a search for `Strasse` matches a + device named `Straße`, and vice versa. Ligatures such as `fi` behave the same way. One + consequence is that a case-insensitive exact match on such a field may return more than + one object. Fields which do not use this collation, such as `serial` and `description`, + match these characters literally. ### Foreign Keys & Other Fields diff --git a/netbox/dcim/tests/test_filtersets.py b/netbox/dcim/tests/test_filtersets.py index 643e8af30..a9cdd9527 100644 --- a/netbox/dcim/tests/test_filtersets.py +++ b/netbox/dcim/tests/test_filtersets.py @@ -2,6 +2,7 @@ from decimal import Decimal from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django.db import DEFAULT_DB_ALIAS, connection from django.test import TestCase from circuits.models import Circuit, CircuitTermination, CircuitType, Provider @@ -3464,6 +3465,28 @@ class DeviceCollatedFilterTestCase(TestCase): qs = Device.objects.annotate(collated=CollateAsChar('name')).filter(collated__icontains='switch') self.assertEqual(qs.count(), 2) + def test_collation_is_applied_to_parameter(self): + # The tests above assert on results, which stay correct for ASCII values even if + # the collation is never applied. This asserts on the lookup's own output instead, + # so that the mechanism failing open is caught rather than passing silently. + for lookup in ('icontains', 'iexact', 'istartswith', 'iendswith'): + with self.subTest(lookup=lookup): + self.assertEqual( + self._compiled_rhs(Device, 'name', lookup), + '%s COLLATE "natural_sort"' + ) + self.assertEqual(self._compiled_rhs(Device, 'serial', lookup), '%s') + + @staticmethod + def _compiled_rhs(model, field_name, lookup): + """ + Compile a single filter's right-hand side and return its SQL. + """ + query = model.objects.filter(**{f'{field_name}__{lookup}': 'x'}).query + compiler = query.get_compiler(using=DEFAULT_DB_ALIAS) + rhs, _ = query.where.children[0].process_rhs(compiler, connection) + return rhs + class ModuleTestCase(TestCase, ChangeLoggedFilterSetTestMixin): queryset = Module.objects.all() diff --git a/netbox/extras/lookups.py b/netbox/extras/lookups.py index 65efcd937..466aad6e8 100644 --- a/netbox/extras/lookups.py +++ b/netbox/extras/lookups.py @@ -144,6 +144,9 @@ class CollatedCaseInsensitiveMixin: The COLLATE clause must sit inside UPPER(), not after the comparison, or it applies to the comparison's result rather than to its operand and has no effect. + + Tested in dcim.tests.test_filtersets.DeviceCollatedFilterTestCase, which is where the + collated fields these lookups act upon are defined. """ def process_rhs(self, compiler, connection): rhs, params = super().process_rhs(compiler, connection) @@ -155,6 +158,11 @@ class CollatedCaseInsensitiveMixin: # one comparison. Requiring a Col also avoids reading a collation from an # annotation's output_field which the annotation itself does not carry, as Concat() # and Coalesce() both do. + # + # The placeholder is compared literally rather than inspected structurally: a field + # declaring its own get_placeholder() compiles to something other than '%s', and + # splicing a COLLATE clause into that is not safe. Any other rhs is a deliberate + # opt-out which leaves the lookup at its previous behaviour. if collation == NATURAL_SORT_COLLATION and rhs == '%s' and isinstance(self.lhs, Col): # The collation name cannot be passed as a query parameter, but it originates # from the field definition rather than from user input.