diff --git a/docs/models/extras/customfield.md b/docs/models/extras/customfield.md index 7aeae7dad..37b9368e8 100644 --- a/docs/models/extras/customfield.md +++ b/docs/models/extras/customfield.md @@ -109,6 +109,10 @@ Choice sets may optionally define colors for individual values. Colored choices If enabled, values from this field will be automatically pre-populated when cloning existing objects. +### Nulls First + +When ordering objects by this custom field, controls whether objects with no value (null) are sorted before or after objects that have a value. This option is enabled by default. + ### Minimum Value For numeric custom fields only. The minimum valid value (optional). diff --git a/netbox/extras/api/serializers_/customfields.py b/netbox/extras/api/serializers_/customfields.py index 175b88646..b2ad94526 100644 --- a/netbox/extras/api/serializers_/customfields.py +++ b/netbox/extras/api/serializers_/customfields.py @@ -68,7 +68,7 @@ class CustomFieldSerializer(OwnerMixin, ChangeLogMessageSerializer, ValidatedMod fields = [ 'id', 'url', 'display_url', 'display', 'object_types', 'type', 'related_object_type', 'data_type', 'name', 'label', 'group_name', 'description', 'required', 'unique', 'search_weight', 'filter_logic', - 'ui_visible', 'ui_editable', 'is_cloneable', 'default', 'related_object_filter', 'weight', + 'ui_visible', 'ui_editable', 'is_cloneable', 'nulls_first', 'default', 'related_object_filter', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'validation_schema', 'choice_set', 'owner', 'comments', 'created', 'last_updated', ] diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index 8bb91ed0e..1d8b7d8ac 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -175,8 +175,8 @@ class CustomFieldFilterSet(OwnerFilterMixin, ChangeLoggedModelFilterSet): model = CustomField fields = ( 'id', 'name', 'label', 'group_name', 'required', 'unique', 'search_weight', 'filter_logic', 'ui_visible', - 'ui_editable', 'weight', 'is_cloneable', 'description', 'validation_minimum', 'validation_maximum', - 'validation_regex', + 'ui_editable', 'weight', 'is_cloneable', 'nulls_first', 'description', 'validation_minimum', + 'validation_maximum', 'validation_regex', ) def search(self, queryset, name, value): diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index 7eb7a6f70..66f746646 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -76,6 +76,11 @@ class CustomFieldBulkEditForm(ChangelogMessageMixin, OwnerMixin, BulkEditForm): required=False, widget=BulkEditNullBooleanSelect() ) + nulls_first = forms.NullBooleanField( + label=_('Nulls first'), + required=False, + widget=BulkEditNullBooleanSelect() + ) validation_minimum = forms.DecimalField( label=_('Minimum value'), required=False, @@ -96,7 +101,7 @@ class CustomFieldBulkEditForm(ChangelogMessageMixin, OwnerMixin, BulkEditForm): fieldsets = ( FieldSet('group_name', 'description', 'weight', 'required', 'unique', 'choice_set', name=_('Attributes')), - FieldSet('ui_visible', 'ui_editable', 'is_cloneable', name=_('Behavior')), + FieldSet('ui_visible', 'ui_editable', 'is_cloneable', 'nulls_first', name=_('Behavior')), FieldSet( 'validation_minimum', 'validation_maximum', 'validation_regex', 'validation_schema', name=_('Validation') diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 78f6d8c0b..492c69c17 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -81,7 +81,7 @@ class CustomFieldImportForm(OwnerCSVMixin, CSVModelForm): 'name', 'label', 'group_name', 'type', 'object_types', 'related_object_type', 'required', 'unique', 'description', 'search_weight', 'filter_logic', 'default', 'choice_set', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'validation_schema', 'ui_visible', 'ui_editable', - 'is_cloneable', 'owner', 'comments', + 'is_cloneable', 'nulls_first', 'owner', 'comments', ) diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index 686bb1eb9..4cdf601f9 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -47,7 +47,7 @@ class CustomFieldFilterForm(OwnerFilterMixin, SavedFiltersMixin, FilterForm): FieldSet('q', 'filter_id'), FieldSet('object_type_id', 'type', 'group_name', 'weight', 'required', 'unique', name=_('Attributes')), FieldSet('choice_set_id', 'related_object_type_id', name=_('Type Options')), - FieldSet('ui_visible', 'ui_editable', 'is_cloneable', name=_('Behavior')), + FieldSet('ui_visible', 'ui_editable', 'is_cloneable', 'nulls_first', name=_('Behavior')), FieldSet('validation_minimum', 'validation_maximum', 'validation_regex', name=_('Validation')), FieldSet('owner_group_id', 'owner_id', name=_('Ownership')), ) @@ -110,6 +110,13 @@ class CustomFieldFilterForm(OwnerFilterMixin, SavedFiltersMixin, FilterForm): choices=BOOLEAN_WITH_BLANK_CHOICES ) ) + nulls_first = forms.NullBooleanField( + label=_('Nulls first'), + required=False, + widget=forms.Select( + choices=BOOLEAN_WITH_BLANK_CHOICES + ) + ) validation_minimum = forms.DecimalField( label=_('Minimum value'), required=False diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 7395216d5..fc67c855e 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -119,7 +119,8 @@ class CustomFieldForm(ChangelogMessageMixin, OwnerMixin, forms.ModelForm): name=_('Custom Field') ), FieldSet( - 'search_weight', 'filter_logic', 'ui_visible', 'ui_editable', 'weight', 'is_cloneable', name=_('Behavior') + 'search_weight', 'filter_logic', 'ui_visible', 'ui_editable', 'weight', 'is_cloneable', 'nulls_first', + name=_('Behavior') ), ) diff --git a/netbox/extras/graphql/filters.py b/netbox/extras/graphql/filters.py index 271b209e2..779519789 100644 --- a/netbox/extras/graphql/filters.py +++ b/netbox/extras/graphql/filters.py @@ -193,6 +193,7 @@ class CustomFieldFilter(ChangeLoggedModelFilter): strawberry_django.filter_field() ) is_cloneable: FilterLookup[bool] | None = strawberry_django.filter_field() + nulls_first: FilterLookup[bool] | None = strawberry_django.filter_field() comments: StrFilterLookup | None = strawberry_django.filter_field() diff --git a/netbox/extras/migrations/0141_custom_field_nulls_first.py b/netbox/extras/migrations/0141_custom_field_nulls_first.py new file mode 100644 index 000000000..d8c2ebe9d --- /dev/null +++ b/netbox/extras/migrations/0141_custom_field_nulls_first.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("extras", "0140_imageattachment_image_size"), + ] + + operations = [ + migrations.AddField( + model_name="customfield", + name="nulls_first", + field=models.BooleanField(default=True), + ), + ] diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index 4f44eb3ac..1d3485f73 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -260,6 +260,11 @@ class CustomField(CloningMixin, ExportTemplatesMixin, OwnerMixin, ChangeLoggedMo verbose_name=_('is cloneable'), help_text=_('Replicate this value when cloning objects') ) + nulls_first = models.BooleanField( + default=True, + verbose_name=_('nulls first'), + help_text=_('Sort null values before non-null values when ordering by this field') + ) comments = models.TextField( verbose_name=_('comments'), blank=True @@ -271,6 +276,7 @@ class CustomField(CloningMixin, ExportTemplatesMixin, OwnerMixin, ChangeLoggedMo 'object_types', 'type', 'related_object_type', 'group_name', 'description', 'required', 'unique', 'search_weight', 'filter_logic', 'default', 'weight', 'validation_minimum', 'validation_maximum', 'validation_regex', 'validation_schema', 'choice_set', 'ui_visible', 'ui_editable', 'is_cloneable', + 'nulls_first', ) class Meta: diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index fac030061..5932ca89f 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -112,6 +112,10 @@ class CustomFieldTable(NetBoxTable): verbose_name=_('Is Cloneable'), false_mark=None ) + nulls_first = columns.BooleanColumn( + verbose_name=_('Nulls First'), + false_mark=None + ) validation_minimum = tables.Column( verbose_name=_('Minimum Value'), ) @@ -135,8 +139,8 @@ class CustomFieldTable(NetBoxTable): fields = ( 'pk', 'id', 'name', 'object_types', 'label', 'type', 'related_object_type', 'group_name', 'required', 'unique', 'default', 'description', 'search_weight', 'filter_logic', 'ui_visible', 'ui_editable', - 'is_cloneable', 'weight', 'choice_set', 'choices', 'validation_minimum', 'validation_maximum', - 'validation_regex', 'validation_schema', 'comments', 'created', 'last_updated', + 'is_cloneable', 'nulls_first', 'weight', 'choice_set', 'choices', 'validation_minimum', + 'validation_maximum', 'validation_regex', 'validation_schema', 'comments', 'created', 'last_updated', ) default_columns = ( 'pk', 'name', 'object_types', 'label', 'group_name', 'type', 'required', 'unique', 'description', diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 0952676d2..bdc19a65a 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -175,6 +175,7 @@ class CustomFieldTestCase(APIViewTestCases.APIViewTestCase): ] bulk_update_data = { 'description': 'New description', + 'nulls_first': False, } update_data = { 'object_types': ['dcim.device'], @@ -193,7 +194,8 @@ class CustomFieldTestCase(APIViewTestCases.APIViewTestCase): ), CustomField( name='cf2', - type='integer' + type='integer', + nulls_first=False ), CustomField( name='cf3', diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 3b83599b7..5cd193478 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -16,6 +16,7 @@ from extras.choices import * from extras.models import CustomField, CustomFieldChoiceSet from ipam.models import VLAN from netbox.choices import CSVDelimiterChoices, ImportFormatChoices +from netbox.tables.columns import CustomFieldColumn from utilities.testing import APITestCase, TestCase from virtualization.models import VirtualMachine @@ -69,6 +70,50 @@ class CustomFieldTestCase(TestCase): instance.refresh_from_db() self.assertIsNone(instance.custom_field_data.get(cf.name)) + def test_nulls_first_ordering(self): + """ + Verify that CustomFieldColumn.order() places null values first or last according to the + custom field's nulls_first attribute. + """ + cf = CustomField.objects.create( + name='order_field', + type=CustomFieldTypeChoices.TYPE_INTEGER, + required=False + ) + cf.object_types.set([self.object_type]) + + # Assign values to two of the three sites, leaving the third null + site_a = Site.objects.get(name='Site A') + site_a.custom_field_data[cf.name] = 1 + site_a.save() + site_b = Site.objects.get(name='Site B') + site_b.custom_field_data[cf.name] = 2 + site_b.save() + site_c = Site.objects.get(name='Site C') # no value (null) + + column = CustomFieldColumn(cf) + + # nulls_first=True (default): null value sorts before populated values when ascending + cf.nulls_first = True + queryset, _ = column.order(Site.objects.all(), is_descending=False) + self.assertEqual(list(queryset), [site_c, site_a, site_b]) + + # nulls_first=False: null value sorts after populated values when ascending + cf.nulls_first = False + queryset, _ = column.order(Site.objects.all(), is_descending=False) + self.assertEqual(list(queryset), [site_a, site_b, site_c]) + + # Null placement is independent of sort direction: nulls_first=True keeps the null value + # first even when sorting descending + cf.nulls_first = True + queryset, _ = column.order(Site.objects.all(), is_descending=True) + self.assertEqual(list(queryset), [site_c, site_b, site_a]) + + # nulls_first=False keeps the null value last even when sorting descending + cf.nulls_first = False + queryset, _ = column.order(Site.objects.all(), is_descending=True) + self.assertEqual(list(queryset), [site_b, site_a, site_c]) + def test_longtext_field(self): value = 'A' * 256 diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index a511f3a6c..29f859d9b 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -51,7 +51,8 @@ class CustomFieldTestCase(TestCase, ChangeLoggedFilterSetTests): filter_logic=CustomFieldFilterLogicChoices.FILTER_EXACT, ui_visible=CustomFieldUIVisibleChoices.IF_SET, ui_editable=CustomFieldUIEditableChoices.NO, - description='foobar2' + description='foobar2', + nulls_first=False ), CustomField( name='Custom Field 3', @@ -61,7 +62,8 @@ class CustomFieldTestCase(TestCase, ChangeLoggedFilterSetTests): filter_logic=CustomFieldFilterLogicChoices.FILTER_DISABLED, ui_visible=CustomFieldUIVisibleChoices.HIDDEN, ui_editable=CustomFieldUIEditableChoices.HIDDEN, - description='foobar3' + description='foobar3', + nulls_first=False ), CustomField( name='Custom Field 4', @@ -151,6 +153,12 @@ class CustomFieldTestCase(TestCase, ChangeLoggedFilterSetTests): params = {'description': ['foobar1', 'foobar2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_nulls_first(self): + params = {'nulls_first': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4) + params = {'nulls_first': False} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + class CustomFieldChoiceSetTestCase(TestCase, ChangeLoggedFilterSetTests): queryset = CustomFieldChoiceSet.objects.all() diff --git a/netbox/extras/ui/panels.py b/netbox/extras/ui/panels.py index 23a3da37b..dbfcd043b 100644 --- a/netbox/extras/ui/panels.py +++ b/netbox/extras/ui/panels.py @@ -154,6 +154,7 @@ class CustomFieldBehaviorPanel(panels.ObjectAttributesPanel): weight = attrs.NumericAttr('weight', label=_('Display weight')) ui_visible = attrs.ChoiceAttr('ui_visible', label=_('UI visible')) ui_editable = attrs.ChoiceAttr('ui_editable', label=_('UI editable')) + nulls_first = attrs.BooleanAttr('nulls_first', label=_('Nulls first')) class CustomFieldValidationPanel(panels.ObjectAttributesPanel): diff --git a/netbox/netbox/tables/columns.py b/netbox/netbox/tables/columns.py index 7753c585e..cd6ae2421 100644 --- a/netbox/netbox/tables/columns.py +++ b/netbox/netbox/tables/columns.py @@ -6,7 +6,8 @@ import django_tables2 as tables from django.conf import settings from django.contrib.auth.context_processors import auth from django.contrib.auth.models import AnonymousUser -from django.db.models import DateField, DateTimeField +from django.db.models import Case, DateField, DateTimeField, F, IntegerField, Value, When +from django.db.models.fields.json import KeyTextTransform from django.template import Context, Template from django.urls import reverse from django.utils.dateparse import parse_date @@ -527,6 +528,28 @@ class CustomFieldColumn(tables.Column): super().__init__(*args, **kwargs) + def order(self, queryset, is_descending): + # Order by the underlying JSON value, honoring the custom field's null placement preference. + # A missing key or a JSON null value is extracted as SQL NULL via the ->> (text) operator, + # whereas the -> (JSONB) operator used for value ordering treats JSON null as a sortable value. + # We therefore annotate an explicit rank to control null placement independently of JSONB sorting. + name = self.customfield.name + text_value = f'_cf_{name}_text' + null_rank = f'_cf_{name}_nullrank' + null_sort, value_sort = (0, 1) if self.customfield.nulls_first else (1, 0) + queryset = queryset.annotate(**{ + text_value: KeyTextTransform(name, 'custom_field_data'), + }).annotate(**{ + null_rank: Case( + When(**{f'{text_value}__isnull': True}, then=Value(null_sort)), + default=Value(value_sort), + output_field=IntegerField(), + ), + }) + value = F(f'custom_field_data__{name}') + ordering = (null_rank, value.desc() if is_descending else value.asc()) + return queryset.order_by(*ordering), True + @staticmethod def _linkify_item(item): if hasattr(item, 'get_absolute_url'):