fix: Live migration with conntrack state sync

modules/proxmox-ve/manager.nix

@@ -22,6 +22,7 @@ lib.mkIf cfg.enable {
         "pve-cluster.service"
       ];
       path = with pkgs; [
+        cfg.package
         btrfs-progs
         zfs
         bashInteractive

modules/proxmox-ve/qemu-server.nix

@@ -5,7 +5,19 @@
   ...
 }:
 
+let
+  pveDbusVmstate = pkgs.runCommand "pve-dbus-vmstate" { } ''
+    mkdir -p $out/lib/systemd/system $out/share/dbus-1/system.d
+    cp ${pkgs.pve-qemu-server}/lib/systemd/system/pve-dbus-vmstate@.service \
+      $out/lib/systemd/system/
+    cp ${pkgs.pve-qemu-server}/share/dbus-1/system.d/org.qemu.VMState1.conf \
+      $out/share/dbus-1/system.d/
+  '';
+in
 lib.mkIf config.services.proxmox-ve.enable {
+  systemd.packages = [ pveDbusVmstate ];
+  services.dbus.packages = [ pveDbusVmstate ];
+
   systemd.services.qmeventd = {
     description = "PVE Qemu Event Daemon";
     unitConfig.RequiresMountsFor = [ "/var/run" ];

pkgs/pve-ha-manager/default.nix

@@ -15,14 +15,16 @@
 }:
 
 let
+  pve-storage_ = pve-storage.override { inherit enableLinstor; };
   perlDeps = [
     pve-container
     pve-firewall
     pve-guest-common
-    pve-qemu-server
-    (pve-storage.override { inherit enableLinstor; })
+    (pve-qemu-server.override { pve-storage = pve-storage_; })
+    pve-storage_
   ];
   perlEnv = perl540.withPackages (_: perlDeps);
+  perlLibPath = lib.makeSearchPath "${perl540.libPrefix}/${perl540.version}" perlDeps;
 in
 
 perl540.pkgs.toPerlModule (
@@ -72,7 +74,7 @@ perl540.pkgs.toPerlModule (
       for bin in $out/bin/*; do
         wrapProgram $bin \
           --prefix PATH : ${lib.makeBinPath [ pve-qemu ]} \
-          --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}
+          --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}:${perlLibPath}
       done      
     '';
 

pkgs/pve-qemu-server/default.nix

@@ -8,15 +8,22 @@
   pkgconf,
   libsysprof-capture,
   pcre2,
+  makeWrapper,
   proxmox-backup-client,
+  pve-apiclient,
+  pve-cluster,
+  pve-common,
   pve-edk2-firmware,
   pve-firewall,
+  pve-guest-common,
   pve-qemu,
+  pve-storage,
   util-linux,
   uuid,
   findbin,
   termreadline,
   socat,
+  conntrack-tools,
   vncterm,
   swtpm,
   libglvnd,
@@ -27,6 +34,7 @@
 let
   perlDeps = with perl540.pkgs; [
     CryptOpenSSLRandom
+    ClassMethodMaker
     DataDumper
     DigestSHA
     FilePath
@@ -40,7 +48,12 @@ let
     MIMEBase64
     NetSSLeay
     PathTools
+    pve-apiclient
+    pve-cluster
+    pve-common
     pve-firewall
+    pve-guest-common
+    pve-storage
     ScalarListUtils
     Socket
     Storable
@@ -54,6 +67,7 @@ let
   ];
 
   perlEnv = perl540.withPackages (_: perlDeps);
+  perlLibPath = lib.makeSearchPath "${perl540.libPrefix}/${perl540.version}" perlDeps;
 in
 
 perl540.pkgs.toPerlModule (
@@ -92,6 +106,7 @@ perl540.pkgs.toPerlModule (
       glib
       json_c
       pkgconf
+      makeWrapper
       perlEnv
       libsysprof-capture
       pcre2
@@ -101,9 +116,10 @@ perl540.pkgs.toPerlModule (
 
     dontBuild = true;
 
-    # Create missing SERVICEDIR
+    # Create missing dirs
     preInstall = ''
       mkdir -p $out/lib/systemd/system
+      mkdir -p $out/share/dbus-1/system.d
     '';
 
     installPhase = ''
@@ -121,6 +137,9 @@ perl540.pkgs.toPerlModule (
     '';
 
     postFixup = ''
+      mv "$out"/usr/lib/systemd/system/* "$out/lib/systemd/system/"
+      mv "$out"/usr/share/dbus-1/system.d/* "$out/share/dbus-1/system.d/"
+
       find $out/lib $out/libexec -type f | xargs sed -i \
         -e "/ENV{'PATH'}/d" \
         -e "s|/usr/lib/qemu-server|$out/lib/qemu-server|" \
@@ -146,8 +165,25 @@ perl540.pkgs.toPerlModule (
         #-e "s|/usr/bin/vma||" \
         #-e "s|/usr/bin/pbs-restore||" \
 
+      find $out/lib/systemd/system -type f | xargs sed -i \
+        -e "s|/usr/libexec/qemu-server|$out/libexec/qemu-server|"
+
+      patchShebangs $out/.bin/
       patchShebangs $out/lib/
       patchShebangs $out/libexec/
+
+      find $out/.bin $out/libexec/qemu-server -type f -executable ! -name dbus-vmstate | while read -r bin; do
+        wrapProgram "$bin" \
+          --prefix PATH : ${lib.makeBinPath [ pve-qemu ]} \
+          --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}:${perlLibPath}
+      done
+
+      wrapProgram $out/libexec/qemu-server/dbus-vmstate \
+        --prefix PATH : ${lib.makeBinPath [
+          conntrack-tools
+          pve-qemu
+        ]} \
+        --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}:${perlLibPath}
     '';
 
     passthru.updateScript = pve-update-script {

tests/cluster-api-conntrack.nix

@@ -0,0 +1,68 @@
+{ lib, ... }:
+
+let
+  cluster = import ./cluster-common.nix { inherit lib; };
+in
+{
+  name = "pve-cluster-api-conntrack";
+
+  inherit (cluster) nodes;
+
+  testScript = ''
+    import json
+    import re
+    import shlex
+    import urllib.parse
+
+    ${cluster.clusterSetupScript}
+    ${cluster.vmSetupScript}
+    ${cluster.conntrackSetupScript}
+
+    auth = json.loads(
+      pve1.succeed(
+        "curl -sk --data-urlencode username=root@pam --data-urlencode password=mypassword "
+        "https://localhost:8006/api2/json/access/ticket"
+      )
+    )
+    ticket = auth["data"]["ticket"]
+    csrf = auth["data"]["CSRFPreventionToken"]
+
+    def pve_api(method, path, data=None):
+      cmd = ["curl", "-sk", "-X", method, "--cookie", f"PVEAuthCookie={ticket}"]
+      if method != "GET":
+        cmd += ["-H", f"CSRFPreventionToken: {csrf}"]
+      if data is not None:
+        for key, value in data.items():
+          cmd += ["--data-urlencode", f"{key}={value}"]
+      cmd.append(f"https://localhost:8006/api2/json{path}")
+      return json.loads(pve1.succeed(" ".join(shlex.quote(arg) for arg in cmd)))["data"]
+
+    upid = pve_api(
+      "POST",
+      "/nodes/pve1/qemu/${toString cluster.vmid}/migrate",
+      {
+        "target": "pve2",
+        "online": 1,
+        "with-local-disks": 1,
+        "targetstorage": "local",
+        "with-conntrack-state": 1,
+      },
+    )
+
+    task_path = urllib.parse.quote(upid, safe="")
+    while True:
+      task_status = pve_api("GET", f"/nodes/pve1/tasks/{task_path}/status")
+      if task_status["status"] == "stopped":
+        break
+      time.sleep(1)
+
+    assert task_status["exitstatus"] == "OK", task_status
+    task_log = pve_api("GET", f"/nodes/pve1/tasks/{task_path}/log")
+    task_log_text = "\n".join(entry.get("t", "") for entry in task_log)
+    assert re.search(r"migrated [1-9][0-9]* conntrack state entr", task_log_text), task_log_text
+
+    ${cluster.clusterValidationScript}
+    ${cluster.dbusVmstateValidationScript}
+    ${cluster.conntrackValidationScript}
+  '';
+}

tests/cluster-common.nix

@@ -0,0 +1,117 @@
+{ lib }:
+
+let
+  vmid = 100;
+
+  mkNode =
+    ipAddress: extraConfig:
+    { pkgs, ... }:
+    {
+      services.proxmox-ve = {
+        enable = true;
+        inherit ipAddress;
+        bridges = [ "vmbr0" ];
+      };
+
+      networking.bridges.vmbr0.interfaces = [ ];
+
+      virtualisation.diskSize = 4096;
+      virtualisation.memorySize = 2048;
+
+      # Give slower test VMs more time to migrate conntrack state and validate it.
+      boot.kernelModules = [
+        "nf_conntrack"
+        "nf_conntrack_netlink"
+      ];
+      boot.kernel.sysctl = {
+        "net.netfilter.nf_conntrack_udp_timeout" = 300;
+        "net.netfilter.nf_conntrack_udp_timeout_stream" = 300;
+      };
+    }
+    // extraConfig pkgs;
+in
+{
+  inherit vmid;
+
+  nodes = {
+    pve1 = mkNode "192.168.1.1" (pkgs: {
+      environment.systemPackages = with pkgs; [
+        openssl
+        conntrack-tools
+        netcat-openbsd
+      ];
+
+      users.users.root = {
+        password = "mypassword";
+        initialPassword = null;
+        hashedPassword = null;
+        hashedPasswordFile = null;
+      };
+    });
+
+    pve2 = mkNode "192.168.1.2" (pkgs: {
+      environment.systemPackages = with pkgs; [
+        conntrack-tools
+        netcat-openbsd
+      ];
+    });
+  };
+
+  clusterSetupScript = ''
+    import time
+
+    pve1.start()
+    pve2.start()
+    pve1.wait_for_unit("pveproxy.service")
+    pve1.wait_for_unit("sshd.service")
+    pve2.wait_for_unit("sshd.service")
+    assert "running" in pve1.succeed("pveproxy status")
+    assert "Proxmox" in pve1.succeed("curl -k https://localhost:8006")
+
+    pve1.succeed("pvecm create mycluster")
+    pve1.wait_for_unit("corosync.service")
+
+    pve2.wait_for_unit("multi-user.target")
+    time.sleep(10)
+
+    fingerprint = pve1.succeed("openssl x509 -noout -fingerprint -sha256 -in /etc/pve/local/pve-ssl.pem | cut -d= -f2")
+    pve2.succeed(f"pvesh create /cluster/config/join --hostname 192.168.1.1 --fingerprint {fingerprint.strip()} --password 'mypassword'")
+
+    assert "Yes" in pve2.succeed("pvecm status | grep Quorate")
+    assert "pve2" in pve1.succeed("pvecm nodes")
+  '';
+
+  vmSetupScript = ''
+    pve1.succeed(
+      "qm create ${toString vmid} --name migrate-me --memory 512 --cores 1 --kvm 0 --net0 virtio,bridge=vmbr0 --scsi0 local:4"
+    )
+    pve1.succeed("qm start ${toString vmid}")
+    pve1.succeed("qm status ${toString vmid} | grep -F running")
+  '';
+
+  conntrackSetupScript = ''
+    pve2.succeed("sh -c 'nohup nc -u -l 12345 >/tmp/conntrack-listener.log 2>&1 &'")
+    pve1.succeed("sh -c 'printf ping | nc -u -p 12346 -w1 192.168.1.2 12345 || true'")
+    pve1.succeed(
+      "conntrack -U -p udp -s 192.168.1.1 -d 192.168.1.2 --sport 12346 --dport 12345 -m ${toString vmid}"
+    )
+    pve1.succeed("conntrack -L -o extended -p udp -m ${toString vmid} | grep -F 'mark=${toString vmid}'")
+  '';
+
+  conntrackValidationScript = ''
+    pve2.wait_until_succeeds(
+      "conntrack -L -o extended -p udp -m ${toString vmid} | grep -F 'mark=${toString vmid}'",
+      timeout=30,
+    )
+  '';
+
+  dbusVmstateValidationScript = ''
+    pve1.fail("systemctl --quiet is-active pve-dbus-vmstate@${toString vmid}.service")
+    pve2.fail("systemctl --quiet is-active pve-dbus-vmstate@${toString vmid}.service")
+  '';
+
+  clusterValidationScript = ''
+    pve2.succeed("qm status ${toString vmid} | grep -F running")
+    pve1.succeed("pvesh get /cluster/resources --type vm --output-format json | grep -F '\"vmid\":${toString vmid}' | grep -F '\"node\":\"pve2\"'")
+  '';
+}

tests/cluster-conntrack.nix

@@ -0,0 +1,27 @@
+{ lib, ... }:
+
+let
+  cluster = import ./cluster-common.nix { inherit lib; };
+in
+{
+  name = "pve-cluster-conntrack";
+
+  inherit (cluster) nodes;
+
+  testScript = ''
+    import re
+
+    ${cluster.clusterSetupScript}
+    ${cluster.vmSetupScript}
+    ${cluster.conntrackSetupScript}
+
+    migrate_output = pve1.succeed(
+      "qm migrate ${toString cluster.vmid} pve2 --online --with-local-disks --targetstorage local --with-conntrack-state 1"
+    )
+    assert re.search(r"migrated [1-9][0-9]* conntrack state entr", migrate_output), migrate_output
+
+    ${cluster.clusterValidationScript}
+    ${cluster.dbusVmstateValidationScript}
+    ${cluster.conntrackValidationScript}
+  '';
+}