Merge pull request #277 from cdtomkins/feature_add_custom_ordering_diff

feat: ✨ Add a new CustomOrderingDiff to support custom ordering

Author: cdtomkins

nautobot-app-ssot/{{ cookiecutter.project_slug }}/{{ cookiecutter.app_name }}/diff.py

@@ -0,0 +1,49 @@
+"""Custom Diff object for {{ cookiecutter.system_of_record }} SSoT integration."""
+
+from diffsync.diff import Diff
+from diffsync.enum import DiffSyncActions
+
+
+class CustomOrderingDiff(Diff):
+    """Customised Diff Object, extend it to support custom ordering."""
+
+    def get_children(self):
+        """Iterate over all child elements in all groups in self.children.
+
+        Check if a custom order method is defined, and otherwise use the default
+        method.
+        """
+        overall_deferred_children = []
+
+        for group in self.groups():
+            group_deferred_chdrn = []
+
+            for child in self.children[group].values():
+                # Custom handling to defer deletions until last
+                if child.action == DiffSyncActions.DELETE:
+                    group_deferred_chdrn.append(child)
+                else:
+                    yield child
+
+            # Custom handling logic for each group
+            if group_deferred_chdrn:
+                # Custom handling to order location deletions correctly
+                if group == "location":
+
+                    def location_depth(loc):
+                        keys = loc.keys
+                        depth = 0
+                        if keys.get("parent__name"):
+                            depth += 1
+                        if keys.get("parent__parent__name"):
+                            depth += 1
+                        # TODO: If locations are nested deeper, extend this functionality
+                        return depth
+
+                    # Sort the locations by depth
+                    group_deferred_chdrn.sort(key=location_depth)
+
+                overall_deferred_children.extend(group_deferred_chdrn)
+
+        # Reverse ALL deferred deletions from their natural (creation) order
+        yield from reversed(overall_deferred_children)

nautobot-app-ssot/{{ cookiecutter.project_slug }}/{{ cookiecutter.app_name }}/jobs.py

@@ -3,6 +3,7 @@
 from nautobot.apps.jobs import BooleanVar, register_jobs
 from nautobot_ssot.jobs.base import DataSource, DataTarget
 
+from {{ cookiecutter.app_name }}.diff import CustomOrderingDiff
 from {{ cookiecutter.app_name }}.diffsync.adapters import {{ cookiecutter.system_of_record_camel }}RemoteAdapter, {{ cookiecutter.system_of_record_camel }}NautobotAdapter
 
 name = "{{ cookiecutter.system_of_record }} SSoT"  # pylint: disable=invalid-name
@@ -89,6 +90,16 @@ def run(self, dryrun, memory_profiling, debug, *args, **kwargs):  # pylint: disa
         self.memory_profiling = memory_profiling
         super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs)
 
+    def execute_sync(self):
+        """Method to synchronize the difference from `self.diff`, from SOURCE to TARGET adapter.
+
+        Overridden to use a CustomOrderingDiff diff_class.
+        """
+        if self.source_adapter is not None and self.target_adapter is not None:
+            self.source_adapter.sync_to(self.target_adapter, flags=self.diffsync_flags, diff_class=CustomOrderingDiff)
+        else:
+            self.logger.warning("One of the adapters was not properly initialized prior to synchronization.")
+
 
 jobs = [{{ cookiecutter.system_of_record_camel }}DataSource, {{ cookiecutter.system_of_record_camel }}DataTarget]
 register_jobs(*jobs)

nautobot-app-ssot/{{ cookiecutter.project_slug }}/{{ cookiecutter.app_name }}/tests/test_{{ cookiecutter.system_of_record_slug }}_diff.py

@@ -0,0 +1,86 @@
+"""Test {{ cookiecutter.system_of_record }} SSoT integration customised Diff Object."""
+
+from unittest.mock import MagicMock
+
+from diffsync.enum import DiffSyncActions
+from nautobot.core.testing import TestCase
+
+from {{ cookiecutter.app_name }}.jobs import CustomOrderingDiff
+
+
+class TestCustomOrderingDiff(TestCase):
+    """Test the CustomOrderingDiff."""
+
+    def setUp(self):
+        """Setup CustomOrderingDiff instance."""
+        self.diff = CustomOrderingDiff()
+        self.diff.children = {}
+        self.diff.groups = MagicMock()
+
+    def test_yields_non_delete_first_then_deletes(self):
+        """Verify deletes are yielded last and reversed."""
+        self.diff.groups.return_value = ["group1"]
+        child1 = MagicMock()
+        child1.action = DiffSyncActions.DELETE
+        child2 = MagicMock()
+        child2.action = DiffSyncActions.CREATE
+        child3 = MagicMock()
+        child3.action = DiffSyncActions.DELETE
+        child4 = MagicMock()
+        child4.action = DiffSyncActions.UPDATE
+
+        self.diff.children = {"group1": {"a": child1, "b": child2, "c": child3, "d": child4}}
+
+        results = list(self.diff.get_children())
+        # Non-deletes first
+        self.assertIn(child2, results[:2])
+        self.assertIn(child4, results[:2])
+        # Deletes at the end (in reverse order)
+        self.assertEqual(results[-1], child1)
+        self.assertEqual(results[-2], child3)
+
+    def test_location_deletes_sorted_by_depth(self):
+        """Verify Location deletes are correctly sorted by depth."""
+        self.diff.groups.return_value = ["location"]
+
+        def make_loc(keys):
+            loc = MagicMock()
+            loc.action = DiffSyncActions.DELETE
+            loc.keys = keys
+            return loc
+
+        loc1 = make_loc({"parent__name": "parentA"})
+        loc2 = make_loc({"parent__name": "parentA", "parent__parent__name": "grandparent"})
+        loc3 = make_loc({})
+
+        self.diff.children = {"location": {"loc1": loc1, "loc2": loc2, "loc3": loc3}}
+
+        results = list(self.diff.get_children())
+
+        self.assertEqual(results, [loc2, loc1, loc3])
+
+    def test_mixed_groups(self):
+        """Verify mixed groups with mixed delete and non-delete children are returned in the correct order"""
+        self.diff.groups.return_value = ["group1", "location"]
+
+        # Group1 children
+        child1 = MagicMock()
+        child1.action = DiffSyncActions.UPDATE
+        child2 = MagicMock()
+        child2.action = DiffSyncActions.DELETE
+
+        def make_loc(keys):
+            loc = MagicMock()
+            loc.action = DiffSyncActions.DELETE
+            loc.keys = keys
+            return loc
+
+        loc1 = make_loc({"parent__name": "p1"})
+        loc2 = make_loc({})
+
+        self.diff.children = {"group1": {"c1": child1, "c2": child2}, "location": {"l1": loc1, "l2": loc2}}
+
+        results = list(self.diff.get_children())
+
+        self.assertEqual(results[0], child1)
+        self.assertEqual(results[-3:], [loc1, loc2, child2])