Harden OpenVMM Azure guest orchestration (#4420)

* Add cloud-init-based guest provisioning

(cherry picked from commit 6787224)

* Make OpenVMM launch robust in detached mode

(cherry picked from commit 03db0e8)

* Add TAP networking and host-side SSH reachability

(cherry picked from commit cdd05e7)

* Add OpenVMM guest smoke coverage and runner integration

Add an Azure smoke runbook and OpenVMM smoke suite that exercise guest boot,
restart, and stop/start on a prepared host.

Describe the LisaRunner guest-enabled flow explicitly:
- schedule environment initialization when the outer environment reaches Deployed
- initialize guest nodes after the outer environment connects
- allow guest-targeted tests that require Deployed to run once the outer host is Connected
- keep retry on the runner redeploy path instead of eagerly deleting the environment

Also wire the Azure/OpenVMM guest handling, schema updates, diagnostics, and
selftest coverage needed for this flow.

(cherry picked from commit ac4c15c)

* Harden OpenVMM cleanup and host handling

Make guest teardown best-effort so parent node cleanup and close continue even when a guest cleanup path fails.

Tighten OpenVMM host-side handling by using host-native paths for copied artifacts, validating whoami before TAP creation, guarding working directory deletion, scoping dnsmasq state to the OpenVMM interface, and restoring the original ip_forward state when SSH forwarding is torn down.

Also update the launcher wrapper to return the real OpenVMM PID so later liveness checks and forced cleanup target the VM process rather than the wrapper shell.

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: vyadavmsft

lisa/microsoft/runbook/openvmm/openvmm-azure-smoke.yml

@@ -0,0 +1,117 @@
+name: openvmm azure smoke
+extension:
+  - ../../testsuites
+variable:
+  - name: host_admin_username
+    value: lisatest
+  - name: host_admin_password
+    value: ""
+    is_secret: true
+  - name: host_admin_private_key_file
+    value: ""
+    is_secret: true
+  - name: guest_admin_username
+    value: lisatest
+  - name: guest_admin_password
+    value: ""
+    is_secret: true
+  - name: guest_admin_private_key_file
+    value: ""
+    is_secret: true
+  - name: guest_extra_user_data
+    value: ""
+  - name: subscription_id
+    value: ""
+  - name: location
+    value: "westus3"
+  - name: marketplace_image
+    value: ""
+  - name: vm_size
+    value: ""
+  - name: openvmm_binary
+    value: /usr/local/bin/openvmm
+  - name: openvmm_install_path
+    value: /usr/local/bin/openvmm
+  - name: openvmm_installer_repo
+    value: https://github.com/microsoft/openvmm.git
+  - name: openvmm_installer_ref
+    value: ""
+  - name: openvmm_installer_force_install
+    value: false
+  - name: openvmm_host_working_dir
+    value: /var/tmp
+  - name: uefi_firmware_path
+    value: ""
+  - name: uefi_firmware_is_remote_path
+    value: false
+  - name: disk_img_path
+    value: ""
+  - name: disk_img_is_remote_path
+    value: false
+  - name: tap_name
+    value: tap0
+  - name: bridge_name
+    value: ovmbr0
+  - name: tap_host_cidr
+    value: 10.0.0.1/24
+  - name: forwarded_port
+    value: 60022
+notifier:
+  - type: html
+transformer:
+  - type: openvmm_installer
+    phase: environment_connected
+    installer:
+      type: source
+      repo: $(openvmm_installer_repo)
+      ref: $(openvmm_installer_ref)
+      force_install: $(openvmm_installer_force_install)
+      install_path: $(openvmm_install_path)
+platform:
+  - type: azure
+    admin_username: $(host_admin_username)
+    admin_password: $(host_admin_password)
+    admin_private_key_file: $(host_admin_private_key_file)
+    guest_enabled: true
+    guests:
+      - type: openvmm
+        use_parent_capability: false
+        username: $(guest_admin_username)
+        password: $(guest_admin_password)
+        private_key_file: $(guest_admin_private_key_file)
+        cloud_init:
+          extra_user_data: $(guest_extra_user_data)
+        lisa_working_dir: $(openvmm_host_working_dir)
+        openvmm_binary: $(openvmm_binary)
+        boot_mode: uefi
+        capability:
+          core_count: 2
+          memory_mb: 2048
+        uefi:
+          firmware_path: $(uefi_firmware_path)
+          firmware_is_remote_path: $(uefi_firmware_is_remote_path)
+        disk_img: $(disk_img_path)
+        disk_img_is_remote_path: $(disk_img_is_remote_path)
+        serial:
+          mode: file
+        network:
+          mode: tap
+          address_mode: discover
+          tap_name: $(tap_name)
+          bridge_name: $(bridge_name)
+          tap_host_cidr: $(tap_host_cidr)
+          forward_ssh_port: true
+          forwarded_port: $(forwarded_port)
+    azure:
+      subscription_id: $(subscription_id)
+    requirement:
+      azure:
+        marketplace: $(marketplace_image)
+        location: $(location)
+        vm_size: $(vm_size)
+testcase:
+  - criteria:
+      name:
+        - verify_openvmm_guest_boot
+        - verify_openvmm_restart_via_platform
+        - verify_openvmm_stop_start_in_platform

lisa/microsoft/testsuites/openvmm/openvmm.py

@@ -1,60 +1,103 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-from assertpy import assert_that
+from typing import Any
 
-from lisa import Node, SkippedException, TestCaseMetadata, TestSuite, TestSuiteMetadata
+from lisa import (
+    Logger,
+    RemoteNode,
+    SkippedException,
+    TestCaseMetadata,
+    TestSuite,
+    TestSuiteMetadata,
+    simple_requirement,
+)
+from lisa.environment import EnvironmentStatus
 from lisa.features import StartStop
 from lisa.sut_orchestrator.openvmm.node import OpenVmmGuestNode
-from lisa.testsuite import simple_requirement
+from lisa.tools import Uname
 
 
 @TestSuiteMetadata(
     area="openvmm",
     category="functional",
     description="""
-    Smoke coverage for OpenVMM guest provisioning and platform lifecycle.
+    This test suite validates OpenVMM guests running on a prepared L1 host.
     """,
 )
-class OpenVmmSmokeTestSuite(TestSuite):
+class OpenVmmPlatformSuite(TestSuite):
+    def before_case(self, log: Logger, **kwargs: Any) -> None:
+        node = kwargs["node"]
+        if not isinstance(node, OpenVmmGuestNode):
+            raise SkippedException(
+                "This suite only applies to OpenVMM guest nodes. "
+                f"Actual node type: {type(node).__name__}."
+            )
+
     @TestCaseMetadata(
         description="""
-        Validate an OpenVMM guest is provisioned, reachable over SSH, and can
-        execute a simple command after launch.
+        This case validates that an OpenVMM guest is reachable over SSH and that
+        the guest booted successfully.
         """,
-        priority=1,
-        requirement=simple_requirement(supported_features=[StartStop]),
+        priority=0,
+        requirement=simple_requirement(
+            environment_status=EnvironmentStatus.Deployed,
+        ),
     )
-    def verify_openvmm_provisioning(self, node: Node) -> None:
-        openvmm_node = self._get_openvmm_guest(node)
+    def verify_openvmm_guest_boot(
+        self,
+        log: Logger,
+        node: RemoteNode,
+    ) -> None:
+        kernel_release = node.tools[Uname].get_linux_information().kernel_version_raw
+        log.info(f"Connected to OpenVMM guest kernel {kernel_release}")
 
-        result = openvmm_node.execute("echo openvmm-smoke", shell=True)
+    @TestCaseMetadata(
+        description="""
+        This case validates that platform restart keeps the OpenVMM guest
+        reachable after the restart.
+        """,
+        priority=0,
+        requirement=simple_requirement(
+            environment_status=EnvironmentStatus.Deployed,
+            supported_features=[StartStop],
+        ),
+    )
+    def verify_openvmm_restart_via_platform(
+        self,
+        log: Logger,
+        node: RemoteNode,
+    ) -> None:
+        start_stop = node.features[StartStop]
+        start_stop.restart()
 
-        result.assert_exit_code()
-        assert_that(result.stdout.strip()).is_equal_to("openvmm-smoke")
+        kernel_release = node.tools[Uname].get_linux_information().kernel_version_raw
+        log.info(f"OpenVMM guest returned after restart on kernel {kernel_release}")
 
     @TestCaseMetadata(
         description="""
-        Validate the OpenVMM StartStop feature can stop and start a guest while
-        preserving SSH connectivity for subsequent command execution.
+        This case validates that platform stop/start keeps the OpenVMM guest
+        reachable for subsequent command execution.
         """,
-        priority=1,
-        requirement=simple_requirement(supported_features=[StartStop]),
+        priority=0,
+        requirement=simple_requirement(
+            environment_status=EnvironmentStatus.Deployed,
+            supported_features=[StartStop],
+        ),
     )
-    def verify_openvmm_stop_start_in_platform(self, node: Node) -> None:
-        openvmm_node = self._get_openvmm_guest(node)
-
-        start_stop = openvmm_node.features[StartStop]
+    def verify_openvmm_stop_start_in_platform(
+        self,
+        log: Logger,
+        node: RemoteNode,
+    ) -> None:
+        start_stop = node.features[StartStop]
+        log.info("Stopping OpenVMM guest via platform")
         start_stop.stop(wait=True)
+        log.info("Starting OpenVMM guest via platform")
         start_stop.start(wait=True)
 
-        result = openvmm_node.execute("echo openvmm-recovered", shell=True)
-
-        result.assert_exit_code()
-        assert_that(result.stdout.strip()).is_equal_to("openvmm-recovered")
-
-    def _get_openvmm_guest(self, node: Node) -> OpenVmmGuestNode:
-        if not isinstance(node, OpenVmmGuestNode):
-            raise SkippedException("This suite only applies to OpenVMM guest nodes.")
-
-        return node
+        kernel_release = node.tools[Uname].get_linux_information().kernel_version_raw
+        log.info(
+            f"OpenVMM guest returned after platform stop/start on kernel "
+            f"{kernel_release}"
+        )

lisa/node.py

@@ -355,12 +355,30 @@ def execute_async(
         )
 
     def cleanup(self) -> None:
+        for guest in self.guests:
+            try:
+                guest.cleanup()
+            except Exception:
+                self.log.exception(
+                    "failed to clean up guest "
+                    f"'{guest.name or guest.index}' while cleaning node "
+                    f"'{self.name}'. Continuing parent cleanup."
+                )
         self.log.debug("cleaning up...")
         if hasattr(self, "_log_handler") and self._log_handler:
             remove_handler(self._log_handler, self.log)
             self._log_handler.close()
 
     def close(self) -> None:
+        for guest in self.guests:
+            try:
+                guest.close()
+            except Exception:
+                self.log.exception(
+                    "failed to close guest "
+                    f"'{guest.name or guest.index}' while closing node "
+                    f"'{self.name}'. Continuing parent close."
+                )
         self.log.debug("closing node connection...")
         if self._shell:
             self._shell.close()
@@ -553,7 +571,12 @@ def mark_dirty(self) -> None:
         self._is_dirty = True
 
     def test_connection(self) -> bool:
-        assert self._shell
+        if not self._shell:
+            self.log.debug(
+                f"connection test failed for node '{self.name}' because its "
+                "shell is not initialized"
+            )
+            return False
         if not self._shell.is_remote:
             return True
         self.log.debug("testing connection...")

lisa/runners/lisa_runner.py

@@ -200,6 +200,13 @@ def _dispatch_test_result(
         # run on deployed environment
         can_run_results = [x for x in can_run_results if x.can_run]
         if environment.status == EnvironmentStatus.Deployed and can_run_results:
+            if self._guest_enabled:
+                return self._generate_task(
+                    task_method=self._initialize_environment_task,
+                    environment=environment,
+                    test_results=can_run_results[:1],
+                )
+
             selected_test_results = self._get_test_result_to_run(
                 test_results=test_results, environment=environment
             )
@@ -341,6 +348,9 @@ def _initialize_environment_task(
                 phase=constants.TRANSFORMER_PHASE_ENVIRONMENT_CONNECTED,
                 environment=environment,
             )
+            if self._guest_enabled:
+                guest_environment = environment.get_guest_environment()
+                guest_environment.nodes.initialize()
         except Exception as e:
             self._attach_failed_environment_to_result(
                 environment=environment,
@@ -639,8 +649,10 @@ def _get_runnable_test_results(
             )
             and (
                 environment_status is None
-                or x.runtime_data.metadata.requirement.environment_status
-                == environment_status
+                or self._matches_environment_status(
+                    x.runtime_data.metadata.requirement.environment_status,
+                    environment_status,
+                )
             )
         ]
         if environment:
@@ -688,6 +700,23 @@ def _get_runnable_test_results(
         results = self._sort_test_results(results)
         return results
 
+    def _matches_environment_status(
+        self,
+        requirement_status: EnvironmentStatus,
+        actual_status: EnvironmentStatus,
+    ) -> bool:
+        if requirement_status == actual_status:
+            return True
+
+        if (
+            self._guest_enabled
+            and actual_status == EnvironmentStatus.Connected
+            and requirement_status == EnvironmentStatus.Deployed
+        ):
+            return True
+
+        return False
+
     def _get_test_result_to_run(
         self, test_results: List[TestResult], environment: Environment
     ) -> List[TestResult]:

lisa/sut_orchestrator/azure/platform_.py

@@ -807,7 +807,9 @@ def _get_node_information(self, node: Node) -> Dict[str, str]:  # noqa: C901
 
             # Guest nodes (like WslContainerNode) don't have features attribute
             # Skip security profile collection for guest nodes
-            if hasattr(node, "features"):
+            if hasattr(node, "features") and node.features.is_supported(
+                SecurityProfile
+            ):
                 security_profile = node.features[SecurityProfile].get_settings()
             else:
                 security_profile = None
@@ -973,7 +975,11 @@ def _get_kernel_version(self, node: Node) -> str:
             linux_information = node.tools[Uname].get_linux_information()
             result = linux_information.kernel_version_raw
         elif not node.is_connected or node.is_posix:
-            if not result and hasattr(node, ATTRIBUTE_FEATURES):
+            if (
+                not result
+                and hasattr(node, ATTRIBUTE_FEATURES)
+                and node.features.is_supported(features.SerialConsole)
+            ):
                 # try to get kernel version in Azure. use it, when uname doesn't work
                 node.log.debug("detecting kernel version from serial log...")
                 serial_console = node.features[features.SerialConsole]
@@ -1009,7 +1015,11 @@ def _get_wala_version(self, node: Node) -> str:
             node.log.debug(f"error on run waagent: {e}")
 
         if not node.is_connected or node.is_posix:
-            if not result and hasattr(node, ATTRIBUTE_FEATURES):
+            if (
+                not result
+                and hasattr(node, ATTRIBUTE_FEATURES)
+                and node.features.is_supported(features.SerialConsole)
+            ):
                 node.log.debug("detecting wala agent version from serial log...")
                 serial_console = node.features[features.SerialConsole]
                 result = serial_console.get_matched_str(WALA_VERSION_PATTERN)