From 19e85518c851248c858c076a4cdabbfcd2970c1c Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:01:30 -0400 Subject: [PATCH 1/8] pve-qemu-server: install VMState files in NixOS paths --- pkgs/pve-qemu-server/default.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkgs/pve-qemu-server/default.nix b/pkgs/pve-qemu-server/default.nix index df21d2a4..9b523122 100644 --- a/pkgs/pve-qemu-server/default.nix +++ b/pkgs/pve-qemu-server/default.nix @@ -101,9 +101,10 @@ perl540.pkgs.toPerlModule ( dontBuild = true; - # Create missing SERVICEDIR + # Create missing dirs preInstall = '' mkdir -p $out/lib/systemd/system + mkdir -p $out/share/dbus-1/system.d ''; installPhase = '' @@ -121,6 +122,9 @@ perl540.pkgs.toPerlModule ( ''; postFixup = '' + mv "$out"/usr/lib/systemd/system/* "$out/lib/systemd/system/" + mv "$out"/usr/share/dbus-1/system.d/* "$out/share/dbus-1/system.d/" + find $out/lib $out/libexec -type f | xargs sed -i \ -e "/ENV{'PATH'}/d" \ -e "s|/usr/lib/qemu-server|$out/lib/qemu-server|" \ @@ -146,6 +150,9 @@ perl540.pkgs.toPerlModule ( #-e "s|/usr/bin/vma||" \ #-e "s|/usr/bin/pbs-restore||" \ + find $out/lib/systemd/system -type f | xargs sed -i \ + -e "s|/usr/libexec/qemu-server|$out/libexec/qemu-server|" + patchShebangs $out/lib/ patchShebangs $out/libexec/ ''; From 31467d70bb599bf5e4fe158ddc7833231dab2338 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:02:10 -0400 Subject: [PATCH 2/8] qemu-server: register VMState service and DBus policy Register only the VMState systemd unit and DBus system policy from pve-qemu-server, instead of importing the package's full unit set. This fixes the missing pve-dbus-vmstate@.service integration without overlapping with the module-owned qmeventd service. --- modules/proxmox-ve/qemu-server.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/modules/proxmox-ve/qemu-server.nix b/modules/proxmox-ve/qemu-server.nix index 152ed058..d9e1eb0f 100644 --- a/modules/proxmox-ve/qemu-server.nix +++ b/modules/proxmox-ve/qemu-server.nix @@ -5,7 +5,19 @@ ... }: +let + pveDbusVmstate = pkgs.runCommand "pve-dbus-vmstate" { } '' + mkdir -p $out/lib/systemd/system $out/share/dbus-1/system.d + cp ${pkgs.pve-qemu-server}/lib/systemd/system/pve-dbus-vmstate@.service \ + $out/lib/systemd/system/ + cp ${pkgs.pve-qemu-server}/share/dbus-1/system.d/org.qemu.VMState1.conf \ + $out/share/dbus-1/system.d/ + ''; +in lib.mkIf config.services.proxmox-ve.enable { + systemd.packages = [ pveDbusVmstate ]; + services.dbus.packages = [ pveDbusVmstate ]; + systemd.services.qmeventd = { description = "PVE Qemu Event Daemon"; unitConfig.RequiresMountsFor = [ "/var/run" ]; From d21cdd1bf9be02fe01be530661ae94fe652ec769 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:02:50 -0400 Subject: [PATCH 3/8] pve-qemu-server: wrap dbus-vmstate for conntrack sync Wrap the dbus-vmstate helper with conntrack-tools on PATH and its package-local Perl library path in PERL5LIB. This gives the helper the runtime it needs for conntrack state synchronization. --- pkgs/pve-qemu-server/default.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkgs/pve-qemu-server/default.nix b/pkgs/pve-qemu-server/default.nix index 9b523122..f6a7cf61 100644 --- a/pkgs/pve-qemu-server/default.nix +++ b/pkgs/pve-qemu-server/default.nix @@ -8,6 +8,7 @@ pkgconf, libsysprof-capture, pcre2, + makeWrapper, proxmox-backup-client, pve-edk2-firmware, pve-firewall, @@ -17,6 +18,7 @@ findbin, termreadline, socat, + conntrack-tools, vncterm, swtpm, libglvnd, @@ -54,6 +56,7 @@ let ]; perlEnv = perl540.withPackages (_: perlDeps); + perlLibPath = lib.makeSearchPath "${perl540.libPrefix}/${perl540.version}" perlDeps; in perl540.pkgs.toPerlModule ( @@ -92,6 +95,7 @@ perl540.pkgs.toPerlModule ( glib json_c pkgconf + makeWrapper perlEnv libsysprof-capture pcre2 @@ -155,6 +159,13 @@ perl540.pkgs.toPerlModule ( patchShebangs $out/lib/ patchShebangs $out/libexec/ + + wrapProgram $out/libexec/qemu-server/dbus-vmstate \ + --prefix PATH : ${lib.makeBinPath [ + conntrack-tools + pve-qemu + ]} \ + --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}:${perlLibPath} ''; passthru.updateScript = pve-update-script { From 3e8934f03e584a68f007dd00b0c8d67e38704047 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:03:27 -0400 Subject: [PATCH 4/8] pve-qemu-server: propagate Perl runtime dependencies Extend pve-qemu-server's runtime Perl environment so qm and its helpers can load the Proxmox Perl modules they reach at runtime. This adds the missing package dependencies and wraps the installed executables with the corresponding PERL5LIB and PATH entries. --- pkgs/pve-qemu-server/default.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkgs/pve-qemu-server/default.nix b/pkgs/pve-qemu-server/default.nix index f6a7cf61..786bc445 100644 --- a/pkgs/pve-qemu-server/default.nix +++ b/pkgs/pve-qemu-server/default.nix @@ -10,9 +10,14 @@ pcre2, makeWrapper, proxmox-backup-client, + pve-apiclient, + pve-cluster, + pve-common, pve-edk2-firmware, pve-firewall, + pve-guest-common, pve-qemu, + pve-storage, util-linux, uuid, findbin, @@ -29,6 +34,7 @@ let perlDeps = with perl540.pkgs; [ CryptOpenSSLRandom + ClassMethodMaker DataDumper DigestSHA FilePath @@ -42,7 +48,12 @@ let MIMEBase64 NetSSLeay PathTools + pve-apiclient + pve-cluster + pve-common pve-firewall + pve-guest-common + pve-storage ScalarListUtils Socket Storable @@ -157,9 +168,16 @@ perl540.pkgs.toPerlModule ( find $out/lib/systemd/system -type f | xargs sed -i \ -e "s|/usr/libexec/qemu-server|$out/libexec/qemu-server|" + patchShebangs $out/.bin/ patchShebangs $out/lib/ patchShebangs $out/libexec/ + find $out/.bin $out/libexec/qemu-server -type f -executable ! -name dbus-vmstate | while read -r bin; do + wrapProgram "$bin" \ + --prefix PATH : ${lib.makeBinPath [ pve-qemu ]} \ + --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}:${perlLibPath} + done + wrapProgram $out/libexec/qemu-server/dbus-vmstate \ --prefix PATH : ${lib.makeBinPath [ conntrack-tools From fa5fa48fa94b3155797f223c857e387b4814232b Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:03:36 -0400 Subject: [PATCH 5/8] pve-ha-manager: propagate Perl runtime dependencies Include pve-ha-manager's dependency Perl trees in the wrapped PERL5LIB so HA code can load the Proxmox modules it reaches at runtime. This matches the runtime model used for the qemu-server entry points and fixes the HA-side module lookup failures. --- pkgs/pve-ha-manager/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/pve-ha-manager/default.nix b/pkgs/pve-ha-manager/default.nix index 43f6a80b..aadade6f 100644 --- a/pkgs/pve-ha-manager/default.nix +++ b/pkgs/pve-ha-manager/default.nix @@ -15,14 +15,16 @@ }: let + pve-storage_ = pve-storage.override { inherit enableLinstor; }; perlDeps = [ pve-container pve-firewall pve-guest-common - pve-qemu-server - (pve-storage.override { inherit enableLinstor; }) + (pve-qemu-server.override { pve-storage = pve-storage_; }) + pve-storage_ ]; perlEnv = perl540.withPackages (_: perlDeps); + perlLibPath = lib.makeSearchPath "${perl540.libPrefix}/${perl540.version}" perlDeps; in perl540.pkgs.toPerlModule ( @@ -72,7 +74,7 @@ perl540.pkgs.toPerlModule ( for bin in $out/bin/*; do wrapProgram $bin \ --prefix PATH : ${lib.makeBinPath [ pve-qemu ]} \ - --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version} + --prefix PERL5LIB : $out/${perl540.libPrefix}/${perl540.version}:${perlLibPath} done ''; From 27b190bf4d34749fe544f3c4613d0626083daaf9 Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:03:55 -0400 Subject: [PATCH 6/8] tests: add CLI conntrack migration coverage --- tests/cluster-common.nix | 117 ++++++++++++++++++++++++++++++++++++ tests/cluster-conntrack.nix | 27 +++++++++ tests/cluster.nix | 57 +++--------------- tests/default.nix | 1 + 4 files changed, 152 insertions(+), 50 deletions(-) create mode 100644 tests/cluster-common.nix create mode 100644 tests/cluster-conntrack.nix diff --git a/tests/cluster-common.nix b/tests/cluster-common.nix new file mode 100644 index 00000000..64e9c3c7 --- /dev/null +++ b/tests/cluster-common.nix @@ -0,0 +1,117 @@ +{ lib }: + +let + vmid = 100; + + mkNode = + ipAddress: extraConfig: + { pkgs, ... }: + { + services.proxmox-ve = { + enable = true; + inherit ipAddress; + bridges = [ "vmbr0" ]; + }; + + networking.bridges.vmbr0.interfaces = [ ]; + + virtualisation.diskSize = 4096; + virtualisation.memorySize = 2048; + + # Give slower test VMs more time to migrate conntrack state and validate it. + boot.kernelModules = [ + "nf_conntrack" + "nf_conntrack_netlink" + ]; + boot.kernel.sysctl = { + "net.netfilter.nf_conntrack_udp_timeout" = 300; + "net.netfilter.nf_conntrack_udp_timeout_stream" = 300; + }; + } + // extraConfig pkgs; +in +{ + inherit vmid; + + nodes = { + pve1 = mkNode "192.168.1.1" (pkgs: { + environment.systemPackages = with pkgs; [ + openssl + conntrack-tools + netcat-openbsd + ]; + + users.users.root = { + password = "mypassword"; + initialPassword = null; + hashedPassword = null; + hashedPasswordFile = null; + }; + }); + + pve2 = mkNode "192.168.1.2" (pkgs: { + environment.systemPackages = with pkgs; [ + conntrack-tools + netcat-openbsd + ]; + }); + }; + + clusterSetupScript = '' + import time + + pve1.start() + pve2.start() + pve1.wait_for_unit("pveproxy.service") + pve1.wait_for_unit("sshd.service") + pve2.wait_for_unit("sshd.service") + assert "running" in pve1.succeed("pveproxy status") + assert "Proxmox" in pve1.succeed("curl -k https://localhost:8006") + + pve1.succeed("pvecm create mycluster") + pve1.wait_for_unit("corosync.service") + + pve2.wait_for_unit("multi-user.target") + time.sleep(10) + + fingerprint = pve1.succeed("openssl x509 -noout -fingerprint -sha256 -in /etc/pve/local/pve-ssl.pem | cut -d= -f2") + pve2.succeed(f"pvesh create /cluster/config/join --hostname 192.168.1.1 --fingerprint {fingerprint.strip()} --password 'mypassword'") + + assert "Yes" in pve2.succeed("pvecm status | grep Quorate") + assert "pve2" in pve1.succeed("pvecm nodes") + ''; + + vmSetupScript = '' + pve1.succeed( + "qm create ${toString vmid} --name migrate-me --memory 512 --cores 1 --kvm 0 --net0 virtio,bridge=vmbr0 --scsi0 local:4" + ) + pve1.succeed("qm start ${toString vmid}") + pve1.succeed("qm status ${toString vmid} | grep -F running") + ''; + + conntrackSetupScript = '' + pve2.succeed("sh -c 'nohup nc -u -l 12345 >/tmp/conntrack-listener.log 2>&1 &'") + pve1.succeed("sh -c 'printf ping | nc -u -p 12346 -w1 192.168.1.2 12345 || true'") + pve1.succeed( + "conntrack -U -p udp -s 192.168.1.1 -d 192.168.1.2 --sport 12346 --dport 12345 -m ${toString vmid}" + ) + pve1.succeed("conntrack -L -o extended -p udp -m ${toString vmid} | grep -F 'mark=${toString vmid}'") + ''; + + conntrackValidationScript = '' + pve2.wait_until_succeeds( + "conntrack -L -o extended -p udp -m ${toString vmid} | grep -F 'mark=${toString vmid}'", + timeout=30, + ) + ''; + + dbusVmstateValidationScript = '' + pve1.fail("systemctl --quiet is-active pve-dbus-vmstate@${toString vmid}.service") + pve2.fail("systemctl --quiet is-active pve-dbus-vmstate@${toString vmid}.service") + ''; + + clusterValidationScript = '' + pve2.succeed("qm status ${toString vmid} | grep -F running") + pve1.succeed("pvesh get /cluster/resources --type vm --output-format json | grep -F '\"vmid\":${toString vmid}' | grep -F '\"node\":\"pve2\"'") + ''; +} diff --git a/tests/cluster-conntrack.nix b/tests/cluster-conntrack.nix new file mode 100644 index 00000000..8087e4ab --- /dev/null +++ b/tests/cluster-conntrack.nix @@ -0,0 +1,27 @@ +{ lib, ... }: + +let + cluster = import ./cluster-common.nix { inherit lib; }; +in +{ + name = "pve-cluster-conntrack"; + + inherit (cluster) nodes; + + testScript = '' + import re + + ${cluster.clusterSetupScript} + ${cluster.vmSetupScript} + ${cluster.conntrackSetupScript} + + migrate_output = pve1.succeed( + "qm migrate ${toString cluster.vmid} pve2 --online --with-local-disks --targetstorage local --with-conntrack-state 1" + ) + assert re.search(r"migrated [1-9][0-9]* conntrack state entr", migrate_output), migrate_output + + ${cluster.clusterValidationScript} + ${cluster.dbusVmstateValidationScript} + ${cluster.conntrackValidationScript} + ''; +} diff --git a/tests/cluster.nix b/tests/cluster.nix index da85f77b..331a3a15 100644 --- a/tests/cluster.nix +++ b/tests/cluster.nix @@ -1,57 +1,14 @@ +{ lib, ... }: + +let + cluster = import ./cluster-common.nix { inherit lib; }; +in { name = "pve-cluster"; - nodes = { - pve1 = - { pkgs, ... }: - { - services.proxmox-ve = { - enable = true; - ipAddress = "192.168.1.1"; - }; - - environment.systemPackages = [ pkgs.openssl ]; - - users.users.root = { - password = "mypassword"; - initialPassword = null; - hashedPassword = null; - hashedPasswordFile = null; - }; - - virtualisation.memorySize = 2048; - }; - - pve2 = { - services.proxmox-ve = { - enable = true; - ipAddress = "192.168.1.2"; - }; - - virtualisation.memorySize = 2048; - }; - }; + inherit (cluster) nodes; testScript = '' - import time - pve1.start() - pve2.start() - pve1.wait_for_unit("pveproxy.service") - pve1.wait_for_unit("sshd.service") - pve2.wait_for_unit("sshd.service") - assert "running" in pve1.succeed("pveproxy status") - assert "Proxmox" in pve1.succeed("curl -k https://localhost:8006") - - pve1.succeed("pvecm create mycluster") - pve1.wait_for_unit("corosync.service") - - pve2.wait_for_unit("multi-user.target") - time.sleep(10) - - fingerprint = pve1.succeed("openssl x509 -noout -fingerprint -sha256 -in /etc/pve/local/pve-ssl.pem | cut -d= -f2") - pve2.succeed(f"pvesh create /cluster/config/join --hostname 192.168.1.1 --fingerprint {fingerprint.strip()} --password 'mypassword'") - - assert "Yes" in pve2.succeed("pvecm status | grep Quorate") - assert "pve2" in pve1.succeed("pvecm nodes") + ${cluster.clusterSetupScript} ''; } diff --git a/tests/default.nix b/tests/default.nix index 7262e385..3fe7b632 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -15,6 +15,7 @@ in test-pve-basic = runTest ./basic.nix; # test-pve-ceph = runTest ./ceph.nix; test-pve-cluster = runTest ./cluster.nix; + test-pve-cluster-conntrack = runTest ./cluster-conntrack.nix; test-pve-linstor = runTest ./linstor.nix; test-pve-vm = runTest ./vm.nix; } From 0371f3773efbf8aced489d48868ce8c28b7d7f5b Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:04:01 -0400 Subject: [PATCH 7/8] manager: add proxmox-ve tools to pvedaemon PATH Prepend cfg.package to the pvedaemon service PATH so worker processes can resolve the sibling Proxmox management tools they invoke at runtime, including pvesh during dbus-vmstate cleanup. This fixes the API and UI migration path. --- modules/proxmox-ve/manager.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/proxmox-ve/manager.nix b/modules/proxmox-ve/manager.nix index 2d5cc6f1..6da7631c 100644 --- a/modules/proxmox-ve/manager.nix +++ b/modules/proxmox-ve/manager.nix @@ -22,6 +22,7 @@ lib.mkIf cfg.enable { "pve-cluster.service" ]; path = with pkgs; [ + cfg.package btrfs-progs zfs bashInteractive From 156d0f510ff31dbd07f29ff5ce6a30dd9bd5af4f Mon Sep 17 00:00:00 2001 From: Ihar Hrachyshka Date: Sun, 12 Apr 2026 20:04:25 -0400 Subject: [PATCH 8/8] tests: add API conntrack migration coverage Add a separate migration regression test that talks to the real HTTP API, polls the resulting task, and checks the conntrack transfer and dbus-vmstate cleanup behavior. This covers the pveproxy and pvedaemon execution path used by the UI, which is distinct from qm migrate run from a shell. --- tests/cluster-api-conntrack.nix | 68 +++++++++++++++++++++++++++++++++ tests/default.nix | 1 + 2 files changed, 69 insertions(+) create mode 100644 tests/cluster-api-conntrack.nix diff --git a/tests/cluster-api-conntrack.nix b/tests/cluster-api-conntrack.nix new file mode 100644 index 00000000..35257daa --- /dev/null +++ b/tests/cluster-api-conntrack.nix @@ -0,0 +1,68 @@ +{ lib, ... }: + +let + cluster = import ./cluster-common.nix { inherit lib; }; +in +{ + name = "pve-cluster-api-conntrack"; + + inherit (cluster) nodes; + + testScript = '' + import json + import re + import shlex + import urllib.parse + + ${cluster.clusterSetupScript} + ${cluster.vmSetupScript} + ${cluster.conntrackSetupScript} + + auth = json.loads( + pve1.succeed( + "curl -sk --data-urlencode username=root@pam --data-urlencode password=mypassword " + "https://localhost:8006/api2/json/access/ticket" + ) + ) + ticket = auth["data"]["ticket"] + csrf = auth["data"]["CSRFPreventionToken"] + + def pve_api(method, path, data=None): + cmd = ["curl", "-sk", "-X", method, "--cookie", f"PVEAuthCookie={ticket}"] + if method != "GET": + cmd += ["-H", f"CSRFPreventionToken: {csrf}"] + if data is not None: + for key, value in data.items(): + cmd += ["--data-urlencode", f"{key}={value}"] + cmd.append(f"https://localhost:8006/api2/json{path}") + return json.loads(pve1.succeed(" ".join(shlex.quote(arg) for arg in cmd)))["data"] + + upid = pve_api( + "POST", + "/nodes/pve1/qemu/${toString cluster.vmid}/migrate", + { + "target": "pve2", + "online": 1, + "with-local-disks": 1, + "targetstorage": "local", + "with-conntrack-state": 1, + }, + ) + + task_path = urllib.parse.quote(upid, safe="") + while True: + task_status = pve_api("GET", f"/nodes/pve1/tasks/{task_path}/status") + if task_status["status"] == "stopped": + break + time.sleep(1) + + assert task_status["exitstatus"] == "OK", task_status + task_log = pve_api("GET", f"/nodes/pve1/tasks/{task_path}/log") + task_log_text = "\n".join(entry.get("t", "") for entry in task_log) + assert re.search(r"migrated [1-9][0-9]* conntrack state entr", task_log_text), task_log_text + + ${cluster.clusterValidationScript} + ${cluster.dbusVmstateValidationScript} + ${cluster.conntrackValidationScript} + ''; +} diff --git a/tests/default.nix b/tests/default.nix index 3fe7b632..5150b1d4 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -15,6 +15,7 @@ in test-pve-basic = runTest ./basic.nix; # test-pve-ceph = runTest ./ceph.nix; test-pve-cluster = runTest ./cluster.nix; + test-pve-cluster-api-conntrack = runTest ./cluster-api-conntrack.nix; test-pve-cluster-conntrack = runTest ./cluster-conntrack.nix; test-pve-linstor = runTest ./linstor.nix; test-pve-vm = runTest ./vm.nix;