tests: add initial tests for shared resources

Author: bertschinger

src/test_env/ha.rs

@@ -24,7 +24,20 @@ impl HaEnvironment {
     pub fn new_ha(test_id: String, agent_binary_path: &str, manager_binary_path: &str) -> Self {
         let ports = get_ports();
         let env = TestEnvironment::new("ha", &test_id, agent_binary_path, manager_binary_path);
-        let config = ha_config(ports, test_id.clone());
+        let config = ha_config(ports, &test_id);
+        env.write_out_config(&config);
+        Self {
+            env,
+            test_id,
+            ports,
+            config,
+        }
+    }
+
+    pub fn new_shared(test_id: String, agent_binary_path: &str, manager_binary_path: &str) -> Self {
+        let ports = get_ports();
+        let env = TestEnvironment::new("shared", &test_id, agent_binary_path, manager_binary_path);
+        let config = shared_config(ports, &test_id);
         env.write_out_config(&config);
         Self {
             env,
@@ -146,7 +159,7 @@ impl HaEnvironment {
         .unwrap();
     }
 
-    fn get_status(&self) -> http::ClusterJson {
+    pub fn get_status(&self) -> http::ClusterJson {
         let status = commands::status::get_status(Some(&self.socket_path())).unwrap();
         eprintln!("{status:?}");
         status
@@ -334,6 +347,16 @@ impl Drop for HaEnvironment {
     /// is, started on both hosts in a pair.
     fn drop(&mut self) {
         for resource in self.config.resources.iter() {
+            // The following resource types are not exclusive and are allowed to be double-started.
+            // XXX: Come up with a more robust way to detect this in arbitrary configs rather than
+            // just hard-coding this list?
+            if matches!(
+                resource.kind.as_str(),
+                "heartbeat/route" | "heartbeat/filesystem" | "heartbeat/export"
+            ) {
+                continue;
+            }
+
             if self.env.resource_is_started(resource, 0)
                 && self.env.resource_is_started(resource, 1)
             {
@@ -344,7 +367,7 @@ impl Drop for HaEnvironment {
 }
 
 /// Creates an HA-pair config for use in the ha tests.
-fn ha_config(ports: [u16; 2], test_id: String) -> Config {
+fn ha_config(ports: [u16; 2], test_id: &str) -> Config {
     let mut config = Config {
         hosts: Vec::new(),
         resources: Vec::new(),
@@ -399,3 +422,41 @@ fn ha_config(ports: [u16; 2], test_id: String) -> Config {
 
     config
 }
+
+fn shared_config(ports: [u16; 2], test_id: &str) -> Config {
+    let path = test_path("configs/nfs.yaml");
+    let config = std::fs::read_to_string(path).unwrap();
+
+    // Use the existing NFS example config...
+    let mut config: Config = serde_yaml::from_str(&config).unwrap();
+    // ...but the hosts and resource groups need to be fixed up to reference the specific test
+    // ports. so strip them out, just keeping the resources.
+    std::mem::take(&mut config.hosts);
+    std::mem::take(&mut config.resource_groups);
+
+    for (i, port) in ports.iter().enumerate() {
+        let my_hostname = || -> String { format!("127.0.0.1:{}", port) };
+        let partner_hostname =
+            || -> String { format!("127.0.0.1:{}", if i == 0 { ports[1] } else { ports[0] }) };
+
+        let host = config::Host {
+            hostname: my_hostname(),
+            fence_agent: Some("fence_test".to_string()),
+            fence_parameters: Some(HashMap::from([
+                ("target".to_string(), format!("{test_id}_{i}")),
+                ("test_id".to_string(), format!("shared/{test_id}")),
+            ])),
+        };
+
+        let resource_group = config::ResourceGroup {
+            home_host: my_hostname(),
+            failover_hosts: vec![partner_hostname()],
+            root: format!("ip_addr_{i}"),
+        };
+
+        config.hosts.push(host);
+        config.resource_groups.push(resource_group);
+    }
+
+    config
+}

src/test_env/mod.rs

@@ -11,6 +11,7 @@ use crate::{
 };
 
 pub mod ha;
+pub mod shared;
 
 /// Given a relative `path` in the test directory, prepend the
 /// full path to the test directory.
@@ -313,17 +314,25 @@ impl TestEnvironment {
     /// Get the path to the "resource state file" used in a test -- that is, the file whose
     /// presence indicates the resource is running and whose absence indicates it is stopped.
     fn get_resource_path(&self, resource: &config::Resource, agent: usize) -> String {
+        let fix_path = |path: &String| path.replace("/", "_");
+
         let path = match resource.kind.as_str() {
-            "heartbeat/ZFS" => &format!("zfs.{}", resource.parameters.get("pool").unwrap()),
-            "lustre/Lustre" => &format!(
+            "heartbeat/ZFS" => format!("zfs.{}", resource.parameters.get("pool").unwrap()),
+            "lustre/Lustre" => format!(
                 "lustre.{}",
-                resource
-                    .parameters
-                    .get("mountpoint")
-                    .unwrap()
-                    .replace("/", "_")
+                fix_path(resource.parameters.get("mountpoint").unwrap())
+            ),
+            "heartbeat/ip" => format!("ip.{}", resource.parameters.get("ip").unwrap()),
+            "heartbeat/route" => format!("route.{}", resource.parameters.get("route").unwrap()),
+            "heartbeat/filesystem" => format!(
+                "fs.{}",
+                fix_path(resource.parameters.get("directory").unwrap())
+            ),
+            "heartbeat/export" => format!(
+                "export.{}",
+                fix_path(resource.parameters.get("directory").unwrap())
             ),
-            _ => unreachable!(),
+            _ => todo!("Need to add resource kind '{}' to list.", resource.kind),
         };
         let path = format!("{}_{agent}.{}", self.test_id, path);
         self.private_dir_path.clone() + "/" + &path

src/test_env/shared.rs

@@ -0,0 +1,2 @@
+// SPDX-License-Identifier: MIT
+// Copyright 2026. Triad National Security, LLC.

tests/shared.rs

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: MIT
+// Copyright 2026. Triad National Security, LLC.
+
+#[cfg(test)]
+mod tests {
+    use halo_lib::test_env::ha::*;
+
+    /// Create a TestEnvironment for a test.
+    ///
+    /// The path to the remote binary needs to be determined here and passed into the
+    /// TestEnvironment constructor because the environment variable is only defined when compiling
+    /// tests.
+    fn test_env_helper(test_id: &str) -> HaEnvironment {
+        HaEnvironment::new_shared(
+            test_id.to_string(),
+            env!("CARGO_BIN_EXE_halo_remote"),
+            env!("CARGO_BIN_EXE_halo_manager"),
+        )
+    }
+
+    /// Startup, both agents running, all resources stopped.
+    /// Agents should start resources on their home nodes.
+    #[test]
+    fn startup1() {
+        let env = test_env_helper("startup1");
+        let _a = env.start_agent(0);
+        let _b = env.start_agent(1);
+        let _m = env.start_manager(true);
+
+        std::thread::sleep(std::time::Duration::from_secs(1));
+
+        let status = env.get_status();
+        for resource in status.resources {
+            let (st, _) = resource.single_host_status();
+            match resource.kind.as_str() {
+                "heartbeat/ip" => assert_eq!(st, "Running"),
+                _ => {
+                    for st in resource.status.values() {
+                        assert_eq!(st.status, "Running");
+                    }
+                }
+            }
+        }
+    }
+
+    /// All resources running on one host, after failback, exclusive resources should be running on
+    /// their home host and shared resources should be running everywhere.
+    #[test]
+    fn failback1() {
+        let env = test_env_helper("failback1");
+
+        env.start_resource("ip_addr_0", 0);
+        env.start_resource("ip_addr_1", 0);
+
+        let _a = env.start_agent(0);
+        let _b = env.start_agent(1);
+        let _m = env.start_manager(true);
+
+        std::thread::sleep(std::time::Duration::from_secs(1));
+
+        let status = env.get_status();
+        for resource in status.resources {
+            let st0 = resource.status.get(&env.agent_id(0)).unwrap();
+            assert_eq!(st0.status, "Running");
+
+            let st1 = &resource.status.get(&env.agent_id(1)).unwrap().status;
+            assert!((st1 == "Stopped") | (st1 == "Unknown"));
+        }
+
+        env.failback(1).unwrap();
+
+        std::thread::sleep(std::time::Duration::from_secs(1));
+
+        let status = env.get_status();
+        for resource in status.resources {
+            let (st, _) = resource.single_host_status();
+            match resource.kind.as_str() {
+                "heartbeat/ip" => assert_eq!(st, "Running"),
+                _ => {
+                    for st in resource.status.values() {
+                        assert_eq!(st.status, "Running");
+                    }
+                }
+            }
+        }
+    }
+}