Make LXC works

modules/proxmox-ve/container.nix

@@ -6,14 +6,87 @@
 }:
 
 lib.mkIf config.services.proxmox-ve.enable {
+  systemd.slices = {
+    "system-pve\\x2dcontainer" = {
+      description = "PVE LXC Container Slice";
+      documentation = ["man:pct"];
+
+      unitConfig = {
+        DefaultDependencies="no";
+      };
+    };
+  };
+
   systemd.services = {
+    lxc-monitord = {
+      description = "LXC Container Monitoring Daemon";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "syslog.service" "network.target"];
+
+      documentation = ["man:lxc"];
+
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${pkgs.lxc}/libexec/lxc/lxc-monitord --daemon";
+      };
+    };
+
+    lxc-net = {
+      description = "LXC network bridge setup";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target"];
+      before = ["lxc.service"];
+
+      documentation = ["man:lxc"];
+
+      unitConfig = {
+        ConditionVirtualization = "!lxc";
+      };
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = "yes";
+        ExecStart = "${pkgs.lxc}/libexec/lxc/lxc-net start";
+        ExecStop = "${pkgs.lxc}/libexec/lxc/lxc-net stop";
+      };
+
+      path = [
+        pkgs.iproute2
+        pkgs.iptables
+        pkgs.getent
+        pkgs.dnsmasq
+      ];
+    };
+
+    lxc = {
+      description = "LXC Container Initialization and Autoboot Code";
+      after = [ "network.target" "lxc-net.service" "remote-fs.target" ];
+      wants = [ "lxc-net.service" ];
+      documentation = ["man:lxc-autostart" "man:lxc"];
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = "yes";
+
+        #ExecStartPre = "${pkgs.lxc}/libexec/lxc/lxc-apparmor-load";
+        ExecStart = "${pkgs.lxc}/libexec/lxc/lxc-containers start";
+        ExecStop = "${pkgs.lxc}/libexec/lxc/lxc-containers stop";
+        #ExecReload = "${pkgs.lxc}/libexec/lxc/lxc-apparmor-load";
+        # Environment=BOOTUP=serial
+        # Environment=CONSOLETYPE=serial
+        Delegate = "yes";
+      };
+
+      wantedBy = [ "multi-user.target" ];
+    };
+
     pve-lxc-syscalld = {
       description = "Proxmox VE LXC Syscall Daemon";
       wantedBy = [ "multi-user.target" ];
       before = [ "pve-guests.service" ];
       serviceConfig = {
         Type = "notify";
-        ExecStart = "/usr/lib/x86_64-linux-gnu/pve-lxc-syscalld/pve-lxc-syscalld --system /run/pve/lxc-syscalld.sock";
+        ExecStart = "${pkgs.pve-lxc-syscalld}/bin/pve-lxc-syscalld --system /run/pve/lxc-syscalld.sock";
         RuntimeDirectory = "pve";
         Restart = "on-failure";
       };
@@ -27,47 +100,47 @@ lib.mkIf config.services.proxmox-ve.enable {
       after = [ "lxc.service" ];
       wants = [ "lxc.service" ];
       unitConfig = {
-        DefaultDependencies = false;
+        DefaultDependencies = "no";
         Documentation = "man:lxc-start man:lxc man:pct";
-        serviceConfig = {
-          Type = "simple";
-          Delegate = true;
-          KillMode = "mixed";
-          TimeoutStopSec = 120;
-          ExecStart = "${pkgs.lxc}/bin/lxc-start -F -n %i -o /dev/stderr -l DEBUG";
-          ExecStop = "${pkgs.pve-container}/share/lxc/pve-container-stop-wrapper %i";
-          # Environment=BOOTUP=serial
-          # Environment=CONSOLETYPE=serial
-          # Prevent container init from putting all its output into the journal
-          StandardOutput = null;
-          StandardError = "file:/run/pve/ct-%i.stderr";
-        };
       };
+      serviceConfig = {
+        Type = "simple";
+        Delegate = "yes";
+        KillMode = "mixed";
+        TimeoutStopSec = "120s";
+        ExecStart = "${pkgs.lxc}/bin/lxc-start -F -n %i -o /dev/stderr -l DEBUG";
+        ExecStop = "${pkgs.pve-container}/share/lxc/pve-container-stop-wrapper %i";
+        # Environment=BOOTUP=serial
+        # Environment=CONSOLETYPE=serial
+        # Prevent container init from putting all its output into the journal
+        StandardOutput = "null";
+        StandardError = "file:/run/pve/ct-%i.stderr";
+      };
+    };
 
-      "pve-container@" = {
-        # based on lxc@.service, but without an install section because
-        # starting and stopping should be initiated by PVE code, not
-        # systemd.
-        description = "PVE LXC Container: %i";
-        after = [ "lxc.service" ];
-        wants = [ "lxc.service" ];
-        unitConfig = {
-          DefaultDependencies = false;
-          Documentation = "man:lxc-start man:lxc man:pct";
-        };
-        serviceConfig = {
-          Type = "simple";
-          Delegate = true;
-          KillMode = "mixed";
-          TimeoutStopSec = 120;
-          ExecStart = "${pkgs.lxc}/bin/lxc-start -F -n %i";
-          ExecStop = "${pkgs.pve-container}/share/lxc/pve-container-stop-wrapper %i";
-          # Environment=BOOTUP=serial
-          # Environment=CONSOLETYPE=serial
-          # Prevent container init from putting all its output into the journal
-          StandardOutput = null;
-          StandardError = "file:/run/pve/ct-%i.stderr";
-        };
+    "pve-container@" = {
+      # based on lxc@.service, but without an install section because
+      # starting and stopping should be initiated by PVE code, not
+      # systemd.
+      description = "PVE LXC Container: %i";
+      after = [ "lxc.service" ];
+      wants = [ "lxc.service" ];
+      unitConfig = {
+        DefaultDependencies = "no";
+        Documentation = "man:lxc-start man:lxc man:pct";
+      };
+      serviceConfig = {
+        Type = "simple";
+        Delegate = "yes";
+        KillMode = "mixed";
+        TimeoutStopSec = "120s";
+        ExecStart = "${pkgs.lxc}/bin/lxc-start -F -n %i";
+        ExecStop = "${pkgs.pve-container}/share/lxc/pve-container-stop-wrapper %i";
+        # Environment=BOOTUP=serial
+        # Environment=CONSOLETYPE=serial
+        # Prevent container init from putting all its output into the journal
+        StandardOutput = "null";
+        StandardError = "file:/run/pve/ct-%i.stderr";
       };
     };
   };

modules/proxmox-ve/default.nix

@@ -20,6 +20,7 @@ in
     ./ceph.nix
     ./bridges.nix
     ./cluster.nix
+    ./container.nix
     # ./firewall.nix
     # ./ha-manager.nix
     ./linstor.nix

modules/proxmox-ve/manager.nix

@@ -21,8 +21,11 @@ lib.mkIf cfg.enable {
         "corosync.service"
         "pve-cluster.service"
       ];
-      path = with pkgs; [ btrfs-progs zfs bashInteractive cdrkit swtpm ];
+      path = with pkgs; [ e2fsprogs btrfs-progs zfs bashInteractive cdrkit swtpm ];
       serviceConfig = {
+        ExecStartPre = [
+          "${pkgs.coreutils}/bin/mkdir -p /run/pve"
+        ];
         ExecStart = "${cfg.package}/bin/pvedaemon start";
         ExecStop = "${cfg.package}/bin/pvedaemon stop";
         ExecReload = "${cfg.package}/bin/pvedaemon restart";
@@ -125,7 +128,7 @@ lib.mkIf cfg.enable {
         "pve-guests.service"
         "pve-storage.target"
       ];
-      path = with pkgs; [ btrfs-progs zfs ];
+      path = with pkgs; [ e2fsprogs btrfs-progs zfs ];
       serviceConfig = {
         ExecStartPre = [
           "${pkgs.coreutils}/bin/touch /var/lib/pve-manager/pve-replication-state.lck"
@@ -144,7 +147,7 @@ lib.mkIf cfg.enable {
       description = "PVE Status Daemon";
       wants = [ "pve-cluster.service" ];
       after = [ "pve-cluster.service" ];
-      path = with pkgs; [ btrfs-progs zfs ];
+      path = with pkgs; [ e2fsprogs btrfs-progs zfs ];
       serviceConfig = {
         ExecStart = "${cfg.package}/bin/pvestatd start";
         ExecStop = "${cfg.package}/bin/pvestatd stop";

pkgs/default.nix

@@ -29,6 +29,7 @@ let
     unifont_hex = callPackage ./unifont { };
     vncterm = callPackage ./vncterm { };
     cstream = callPackage ./cstream { };
+    lxc = callPackage ./lxc { };
 
     mkRegistry = callPackage ./proxmox-registry { };
 
@@ -57,6 +58,7 @@ let
     pve-rs = callPackage ./pve-rs { };
     pve-storage = callPackage ./pve-storage { };
     pve-xtermjs = callPackage ./pve-xtermjs { };
+    pve-lxc-syscalld = callPackage ./pve-lxc-syscalld { };
 
     linstor-api-py = callPackage ./linstor-api-py { };
     linstor-client = callPackage ./linstor-client { };

pkgs/lxc/default.nix

@@ -0,0 +1,136 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  makeWrapper,
+
+  bashInteractive,
+  dbus,
+  docbook2x,
+  libapparmor,
+  libcap,
+  libseccomp,
+  libselinux,
+  meson,
+  ninja,
+  nixosTests,
+  openssl,
+  shadow,
+  apparmor-parser,
+  bash,
+  pkg-config,
+  coreutils,
+  systemd,
+
+  nix-update-script,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "lxc";
+  version = "6.0.4";
+
+  src = fetchFromGitHub {
+    owner = "lxc";
+    repo = "lxc";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-zmL568PprrpIWTVCkScXHEzTZ+NduSH4r8ETnz4NY64=";
+  };
+
+  nativeBuildInputs = [
+    docbook2x
+    meson
+    ninja
+    pkg-config
+  ];
+
+  buildInputs = [
+    # some hooks use compgen
+    bashInteractive
+    dbus
+    libapparmor
+    libcap
+    libseccomp
+    libselinux
+    openssl
+    systemd
+    makeWrapper
+  ];
+
+  patches = [
+    # fix docbook2man version detection
+    ./docbook-hack.patch
+
+    # Fix hardcoded path of lxc-user-nic
+    # This is needed to use unprivileged containers
+    ./user-nic.diff
+  ];
+
+  mesonFlags = [
+    "-Dinstall-init-files=true"
+    "-Dinstall-state-dirs=false"
+    "-Dspecfile=false"
+    "-Dtools-multicall=false"
+    "-Dtools=true"
+    "-Dusernet-config-path=/etc/lxc/lxc-usernet"
+    "-Ddistrosysconfdir=${placeholder "out"}/etc/lxc"
+    "-Dsystemd-unitdir=${placeholder "out"}/lib/systemd/system"
+  ];
+
+  # /run/current-system/sw/share
+  postInstall = ''
+    substituteInPlace $out/etc/lxc/lxc --replace-fail "$out/etc/lxc" "/etc/lxc"
+    substituteInPlace $out/libexec/lxc/lxc-net --replace-fail "$out/etc/lxc" "/etc/lxc"
+
+    substituteInPlace $out/share/lxc/templates/lxc-download --replace-fail "$out/share" "/run/current-system/sw/share"
+    substituteInPlace $out/share/lxc/templates/lxc-local --replace-fail "$out/share" "/run/current-system/sw/share"
+    substituteInPlace $out/share/lxc/templates/lxc-oci --replace-fail "$out/share" "/run/current-system/sw/share"
+
+    substituteInPlace $out/share/lxc/config/userns.conf --replace-fail "$out/share" "/run/current-system/sw/share"
+    substituteInPlace $out/share/lxc/config/oci.common.conf --replace-fail "$out/share" "/run/current-system/sw/share"
+
+    sed -i $out/libexec/lxc/lxc-containers \
+      -e "s|touch|${coreutils}/bin/touch|" \
+      -e "s|rm|${coreutils}/bin/rm|"
+  '';
+
+  postFixup = ''
+      for bin in $out/bin/*; do
+        wrapProgram $bin \
+          --prefix PATH : ${lib.makeBinPath [ bash apparmor-parser "/run/wrappers" ]}
+      done
+  '';
+
+  enableParallelBuilding = true;
+
+  doCheck = true;
+
+  passthru = {
+    tests = {
+      incus-lts = nixosTests.incus-lts.container;
+      lxc = nixosTests.lxc;
+      lxd = nixosTests.lxd.container;
+    };
+
+    updateScript = nix-update-script {
+      extraArgs = [
+        "--version-regex"
+        "v(6\\.0\\.*)"
+      ];
+    };
+  };
+
+  meta = {
+    homepage = "https://linuxcontainers.org/";
+    description = "Userspace tools for Linux Containers, a lightweight virtualization system";
+    license = lib.licenses.gpl2;
+
+    longDescription = ''
+      LXC containers are often considered as something in the middle between a chroot and a
+      full fledged virtual machine. The goal of LXC is to create an environment as close as
+      possible to a standard Linux installation but without the need for a separate kernel.
+    '';
+
+    platforms = lib.platforms.linux;
+    teams = [ lib.teams.lxc ];
+  };
+})

pkgs/lxc/docbook-hack.patch

@@ -0,0 +1,21 @@
+diff --git a/meson.build b/meson.build
+index d1527679e..360824994 100644
+--- a/meson.build
++++ b/meson.build
+@@ -320,15 +320,7 @@ docconf.set('LXC_USERNIC_CONF', lxc_user_network_conf)
+ docconf.set('LXC_USERNIC_DB', lxc_user_network_db)
+ docconf.set('PACKAGE_VERSION', version_data.get('LXC_VERSION'))
+ docconf.set('docdtd', '"-//OASIS//DTD DocBook XML" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd"')
+-sgml2man = find_program('docbook2X2man', 'docbook2x-man', 'db2x_docbook2man', 'docbook2man', 'docbook-to-man', required: false, version: '>=0.8')
+-if not sgml2man.found()
+-    sgml2man = find_program('docbook2man', required: false, version: '<0.8')
+-    if sgml2man.found()
+-        docconf.set('docdtd', '"-//Davenport//DTD DocBook V3.0//EN"')
+-    elif want_mans
+-        error('missing required docbook2x or docbook-utils dependency')
+-    endif
+-endif
++sgml2man = find_program('docbook2X2man', 'docbook2x-man', 'db2x_docbook2man', 'docbook2man', 'docbook-to-man', required: false)
+
+ ## Threads.
+ threads = dependency('threads')