From 790dd0d1f493789b163b4626a8252dece01a750c Mon Sep 17 00:00:00 2001 From: Val Packett Date: Fri, 31 Oct 2025 04:44:28 -0300 Subject: [PATCH] Integrate systemd Not fully optimized yet, but shouldn't have any regressions --- .gitmodules | 5 +- flake.lock | 23 ++-- flake.nix | 296 +++++++++++++++++++++++++++++++++++++++--------- libkrun | 2 +- libkrunfw | 1 + munix | 18 +-- munix-init-root | 27 ----- munix-init-user | 4 - muvm | 2 +- 9 files changed, 270 insertions(+), 108 deletions(-) create mode 160000 libkrunfw delete mode 100755 munix-init-root delete mode 100755 munix-init-user diff --git a/.gitmodules b/.gitmodules index af2cfdd..a1452cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,7 @@ url = https://github.com/valpackett/muvm [submodule "libkrun"] path = libkrun - url = https://github.com/valpackett/libkrun + url = https://github.com/containers/libkrun +[submodule "libkrunfw"] + path = libkrunfw + url = https://github.com/valpackett/libkrunfw diff --git a/flake.lock b/flake.lock index becdabc..d60e81e 100644 --- a/flake.lock +++ b/flake.lock @@ -38,21 +38,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1758078198, - "narHash": "sha256-60ojlXp42UZYTOAgsiKPZUab1Aa8pXTIo+rz0zoaynI=", - "ref": "val/tsvwswkqrrsr", - "rev": "1f0fc70eb049852a18a5203af2204bd9f5729f29", - "shallow": true, - "submodules": true, - "type": "git", - "url": "https://github.com/valpackett/nixpkgs" + "lastModified": 1761672384, + "narHash": "sha256-o9KF3DJL7g7iYMZq9SWgfS1BFlNbsm6xplRjVlOCkXI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "08dacfca559e1d7da38f3cf05f1f45ee9bfd213c", + "type": "github" }, "original": { - "ref": "val/tsvwswkqrrsr", - "shallow": true, - "submodules": true, - "type": "git", - "url": "https://github.com/valpackett/nixpkgs" + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" } }, "root": { diff --git a/flake.nix b/flake.nix index 9ab4655..62ecfee 100644 --- a/flake.nix +++ b/flake.nix @@ -1,37 +1,246 @@ { inputs = { self.submodules = true; - # nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - # https://github.com/NixOS/nixpkgs/pull/444133 - nixpkgs.url = "git+https://github.com/valpackett/nixpkgs?shallow=1&submodules=1&ref=val/tsvwswkqrrsr"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + # nixpkgs.url = "git+https://github.com/valpackett/nixpkgs?shallow=1&submodules=1&ref=.."; flake-utils.url = "github:numtide/flake-utils"; virtwl.url = "git+https://github.com/valpackett/wayland-proxy-virtwl?shallow=1&submodules=1&ref=wip"; virtwl.inputs.nixpkgs.follows = "nixpkgs"; }; - outputs = {self, nixpkgs, flake-utils, virtwl, ...}: - flake-utils.lib.eachDefaultSystem (system: + outputs = { self, nixpkgs, flake-utils, virtwl, ... }: { + nixosModules.default = { pkgs, lib, utils, config, ... }: let + useTTY = { + TTYPath = "/dev/hvc0"; + StandardOutput = "tty"; + StandardInput = "tty"; + StandardError = "tty"; + }; + runtimeDir = "/run/vm-user"; + in { + boot.isContainer = true; + fileSystems."/".device = lib.mkDefault "/dev/sda"; # dummy + + # Disable unused things + environment.defaultPackages = lib.mkDefault [ ]; + documentation = { + enable = lib.mkDefault false; + doc.enable = lib.mkDefault false; + info.enable = lib.mkDefault false; + man.enable = lib.mkDefault false; + nixos.enable = lib.mkDefault false; + }; + services.logrotate.enable = false; + services.udisks2.enable = false; + system.tools.nixos-generate-config.enable = false; + system.activationScripts.specialfs = lib.mkForce ""; + systemd.coredump.enable = false; + networking.firewall.enable = false; + powerManagement.enable = false; + boot.kexec.enable = false; + console.enable = false; + + # Configure activation / systemd + boot.initrd.systemd.enable = true; # for etc.overlay, but we don't have initrd + system.etc.overlay.enable = true; # erofs + system.etc.overlay.mutable = false; + systemd.sysusers.enable = true; + services.userborn.enable = false; + services.udev.enable = lib.mkDefault true; + services.udev.packages = lib.mkDefault []; + environment.etc."resolv.conf".text = "# to be overridden with mount"; + environment.etc."machine-id".text = "# to be overridden with mount"; + environment.etc."systemd/system".source = lib.mkForce (utils.systemdUtils.lib.generateUnits { + type = "system"; + units = config.systemd.units; + upstreamUnits = [ + "sysinit.target" + "local-fs.target" + "nss-user-lookup.target" + "umount.target" + "shutdown.target" + "reboot.target" + "exit.target" + "final.target" + "systemd-exit.service" + "systemd-journald.socket" + "systemd-journald-audit.socket" + "systemd-journald-dev-log.socket" + "systemd-journald.service" + "systemd-sysusers.service" + "systemd-udevd-kernel.socket" + "systemd-udevd-control.socket" + "systemd-udevd.service" + "systemd-tmpfiles-setup.service" + "user.slice" + ]; + upstreamWants = ["multi-user.target.wants"]; + }); + + # systemd.package = pkgs.systemdMinimal; # no sysusers + systemd.defaultUnit = "microvm.target"; + systemd.targets.microvm = { + description = "Minimal microVM system"; + wants = ["systemd-journald.socket" "systemd-udevd.service" "dbus.socket"]; + unitConfig.AllowIsolate = "yes"; + }; + systemd.services.systemd-remount-fs.enable = lib.mkForce false; + systemd.services.systemd-pstore.enable = lib.mkForce false; + systemd.services.lastlog2-import.enable = lib.mkForce false; + systemd.services.suid-sgid-wrappers.enable = lib.mkForce false; + systemd.services.systemd-sysusers.serviceConfig = { + ExecStartPre = lib.mkForce []; + ExecStartPost = lib.mkForce []; + }; + systemd.services.microvm-nixos-activation = { + enable = true; + description = "NixOS Activation"; + wantedBy = ["local-fs.target"]; + before = ["systemd-tmpfiles-setup.service" "systemd-sysusers.service"]; + requires = ["systemd-tmpfiles-setup.service" "systemd-sysusers.service"]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + PassEnvironment = ["MICROVM_CLOSURE" "MICROVM_UID" "MICROVM_GID"]; + } // useTTY; + script = '' + PATH=$MICROVM_CLOSURE/sw/bin + cp /etc/resolv.conf /run/ + $MICROVM_CLOSURE/activate || true + mount --bind /run/resolv.conf /etc/resolv.conf + mount --bind /run/machine-id /etc/machine-id + chown 1337:1337 /run + ''; + }; + + # Configure user account + users.mutableUsers = false; + users.users.appvm = { + uid = 1337; + group = "appvm"; + isNormalUser = false; + isSystemUser = true; + home = "/home/appvm"; + description = "microVM User"; + extraGroups = [ "wheel" "video" "input" "systemd-journal" ]; + }; + users.groups.appvm.gid = 1337; + users.allowNoPasswordLogin = true; + systemd.tmpfiles.rules = ["d ${runtimeDir} 1337 1337 -"]; + + # Configure services + + systemd.services.muvm-remote = { + enable = true; + description = "microVM Application runner"; + onFailure = ["exit.target"]; + onSuccess = ["exit.target"]; + after = ["microvm-nixos-activation.service"]; + wantedBy = ["microvm.target"]; + serviceConfig = { + Type = "exec"; + PassEnvironment = ["TERM" "MUVM_REMOTE_CONFIG"]; # "KRUN_CONFIG"]; + Environment = ["XDG_RUNTIME_DIR=${runtimeDir}" "WAYLAND_DISPLAY=wayland-1" "PATH=/run/current-system/sw/bin"]; + User = "appvm"; + Group = "appvm"; + ExecStart = "/opt/bin/muvm-remote"; + ExecStopPost = ''+${pkgs.python3}/bin/python -c "import os,fcntl,struct;print(os.getenv('EXIT_STATUS', '1'));fcntl.ioctl(os.open('/', os.O_RDONLY), 0x7602, int(os.getenv('EXIT_STATUS', '1')))"''; + } // useTTY; + }; + + systemd.services.muvm-configure-network = { + enable = true; + description = "microVM Network configuration"; + wantedBy = ["microvm.target"]; + serviceConfig.Type = "oneshot"; + serviceConfig.ExecStart = "/opt/bin/muvm-configure-network"; + }; + + systemd.sockets.muvm-pwbridge = { + enable = true; + description = "PipeWire cross-domain proxy socket"; + wantedBy = ["microvm.target"]; + partOf = ["muvm-pwbridge.service"]; + listenStreams = [ "${runtimeDir}/pipewire-0" ]; + socketConfig = { + SocketUser = "appvm"; + SocketGroup = "appvm"; + }; + }; + systemd.services.muvm-pwbridge = { + enable = true; + description = "PipeWire cross-domain proxy"; + requires = ["muvm-pwbridge.socket"]; + serviceConfig.Type = "exec"; + serviceConfig.ExecStart = "/opt/bin/muvm-pwbridge"; + }; + + systemd.services.wait-for-udev = { + enable = true; + description = "Wait for device rules being applied"; + wantedBy = ["microvm.target"]; + requires = ["systemd-udevd.service"]; + after = ["systemd-udevd.service"]; + serviceConfig = { + Type = "oneshot"; + ExecStart = [ + "udevadm trigger --action=add" + "udevadm settle" + ]; + }; + }; + systemd.sockets.wayland-proxy-virtwl = { + enable = true; + description = "Wayland cross-domain proxy socket"; + wantedBy = ["microvm.target"]; + partOf = ["wayland-proxy-virtwl.service"]; + listenStreams = [ "${runtimeDir}/wayland-1" ]; + socketConfig = { + SocketUser = "appvm"; + SocketGroup = "appvm"; + FileDescriptorName = "wayland"; + }; + }; + systemd.services.wayland-proxy-virtwl = { + enable = true; + description = "Wayland cross-domain proxy"; + after = ["wait-for-udev.service"]; + requires = ["wayland-proxy-virtwl.socket" "wait-for-udev.service"]; + serviceConfig = { + ExecStart = "${virtwl.packages.x86_64-linux.proxy}/bin/wayland-proxy-virtwl --virtio-gpu"; + Environment = ["XDG_RUNTIME_DIR=${runtimeDir}"]; + User = "appvm"; + Group = "appvm"; + }; + }; + + hardware.graphics.enable = true; + hardware.graphics.package = self.packages.${pkgs.system}.mesa; + }; + } // flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; + config.allowUnfree = true; }; in { - packages.libkrunfw = pkgs.libkrunfw; - # packages.libkrunfw = (pkgs.libkrunfw.overrideAttrs { - # version = "4.10.0"; - # src = pkgs.fetchFromGitHub { - # owner = "containers"; - # repo = "libkrunfw"; - # tag = "v4.10.0"; - # hash = "sha256-mq2gw0+xL6qUZE/fk0vLT3PEpzPV8p+iwRFJHXVOMnk="; - # }; - # kernelSrc = pkgs.fetchurl { - # url = "mirror://kernel/linux/kernel/v6.x/linux-6.12.34.tar.xz"; - # hash = "sha256-p/P+OB9n7KQXLptj77YaFL1/nhJ44DYD0P9ak/Jwwk0="; - # }; - # cargoHash = ""; - # }); + # packages.libkrunfw = pkgs.libkrunfw; + packages.libkrunfw = pkgs.libkrunfw.overrideAttrs (old: { + version = "5.0.0"; + src = ./libkrunfw; + # src = pkgs.fetchFromGitHub { + # owner = "containers"; + # repo = "libkrunfw"; + # tag = "v4.10.0"; + # hash = "sha256-mq2gw0+xL6qUZE/fk0vLT3PEpzPV8p+iwRFJHXVOMnk="; + # }; + kernelSrc = pkgs.fetchurl { + url = "mirror://kernel/linux/kernel/v6.x/linux-6.12.44.tar.xz"; + hash = "sha256-tlAhDtMCeyJJadFIqjd0Uqmq065/KFGr7dMa3+8Wva4="; + }; + # buildInputs = old.buildInputs; + }); packages.libkrun = (pkgs.libkrun.override { withBlk = true; @@ -62,15 +271,9 @@ munixScript = (pkgs.writeScriptBin "munix" (builtins.readFile ./munix)).overrideAttrs(old: { buildCommand = "${old.buildCommand}\n patchShebangs $out"; }); - munixInitRootHook = (pkgs.writeScriptBin "munix-init-root" (builtins.readFile ./munix-init-root)).overrideAttrs(old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - }); - munixInitUserHook = (pkgs.writeScriptBin "munix-init-user" (builtins.readFile ./munix-init-user)).overrideAttrs(old: { - buildCommand = "${old.buildCommand}\n patchShebangs $out"; - }); in pkgs.symlinkJoin { name = "munix"; - paths = [ munixScript munixInitRootHook munixInitUserHook self.packages.${system}.muvm pkgs.passt pkgs.bubblewrap ]; + paths = [ munixScript self.packages.${system}.muvm pkgs.passt pkgs.bubblewrap ]; buildInputs = [ pkgs.makeWrapper ]; postBuild = '' wrapProgram $out/bin/munix --prefix PATH : $out/bin @@ -111,55 +314,40 @@ nixosConfigurations.testvm = nixpkgs.lib.nixosSystem { inherit system; modules = [ - { + self.nixosModules.default + ({ ... }: { system.stateVersion = "25.11"; - fileSystems."/".device = pkgs.lib.mkDefault "/dev/sda"; - boot.isContainer = true; - users.mutableUsers = false; - users.users.appvm = { - uid = 1337; - isNormalUser = true; - home = "/home/appvm"; - description = "microVM User"; - extraGroups = [ "wheel" "video" "input" ]; - }; - users.groups.appvm.gid = 1337; - users.allowNoPasswordLogin = true; - - hardware.graphics.enable = true; - hardware.graphics.package = self.packages.${system}.mesa; - system.replaceDependencies.replacements = [ - { - original = pkgs.mesa; - replacement = self.packages.${system}.mesa; - } - ]; + fonts.packages = [ pkgs.adwaita-fonts pkgs.dejavu_fonts ]; environment.systemPackages = [ pkgs.fastfetch pkgs.htop - pkgs.radeontop virtwl.packages.x86_64-linux.proxy pkgs.wayland-utils pkgs.weston pkgs.waycheck pkgs.vulkan-tools - pkgs.glxinfo pkgs.glmark2 pkgs.mesa-demos pkgs.xorg.xeyes pkgs.xterm - pkgs.vkquake + # pkgs.vkquake # build broken: Program 'spirv-remap' not found + pkgs.veloren pkgs.kdePackages.kate - pkgs.adwaita-fonts pkgs.adwaita-icon-theme + pkgs.amberol + pkgs.bustle + pkgs.d-spy pkgs.gnome-text-editor pkgs.firefox pkgs.ffmpeg-full pkgs.mpv pkgs.libva-utils + pkgs.tailscale + pkgs.zerotierone + pkgs.localsend ]; - } + }) ]; }; diff --git a/libkrun b/libkrun index bd97a39..72b8b08 160000 --- a/libkrun +++ b/libkrun @@ -1 +1 @@ -Subproject commit bd97a39bfaa46147933beca7e16513b6820bb031 +Subproject commit 72b8b0870c3ecad0388f52150a68f79083fcc86f diff --git a/libkrunfw b/libkrunfw new file mode 160000 index 0000000..4b98077 --- /dev/null +++ b/libkrunfw @@ -0,0 +1 @@ +Subproject commit 4b98077866355ecfed46781ab6143d21127bf977 diff --git a/munix b/munix index b15e3e0..18354bb 100755 --- a/munix +++ b/munix @@ -136,21 +136,22 @@ exec bwrap --unshare-all --share-net \ --dev /dev --dir /dev/input --dev-bind /dev/kvm /dev/kvm \ --ro-bind "$MUVM_PATH" /run/munix/muvm \ --ro-bind "$PASST_PATH" /run/munix/passt \ - --ro-bind "$MICROVM_CLOSURE/sw/bin/env" /usr/bin/env \ - --ro-bind "$SCRIPT_PATH/munix-init-root" /usr/bin/munix-init-root \ - --ro-bind "$SCRIPT_PATH/munix-init-user" /usr/bin/munix-init-user \ + --ro-bind "$MUVM_PATH/muvm-guest" /opt/bin/muvm-remote \ + --ro-bind "$MUVM_PATH/muvm-guest" /opt/bin/muvm-configure-network \ + --ro-bind "$MUVM_PATH/muvm-guest" /opt/bin/muvm-pwbridge \ + --symlink "$MICROVM_CLOSURE/etc/systemd" /etc/systemd \ --ro-bind /nix/store /nix/store \ --ro-bind /run/systemd/resolve /run/systemd/resolve \ - --ro-bind /etc/resolv.conf /etc/resolv.conf \ --file 11 /etc/passwd \ --file 12 /etc/group \ + --file 13 /etc/resolv.conf \ --dir "$XDG_RUNTIME_DIR" \ --setenv PATH "/run/munix/muvm:/run/munix/passt:$MICROVM_CLOSURE/sw/bin" \ "${BWRAP_ARGS[@]}" \ muvm \ - -x /usr/bin/munix-init-root -X /usr/bin/munix-init-user \ + --custom-init-cmdline "$MICROVM_CLOSURE/sw/sbin/init --log-target=console" \ "${MUVM_ARGS[@]}" \ - -e MUVM_UDEVD_PATH="$MICROVM_CLOSURE/sw/bin/true" \ + -e container=munix \ -e MICROVM_CLOSURE="$MICROVM_CLOSURE" \ -e MICROVM_UID="$MICROVM_UID" -e MICROVM_GID="$MICROVM_GID" \ -i -t "${MICROVM_COMMAND[@]}" \ @@ -163,4 +164,7 @@ EOF munix:x:$MICROVM_GID: nogroup:x:65534: EOF -) +) \ + 13< /etc/resolv.conf + +# --log-level=debug diff --git a/munix-init-root b/munix-init-root deleted file mode 100755 index 71af6ea..0000000 --- a/munix-init-root +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -unset LANGUAGE LC_ALL LC_CTYPE LC_NUMERIC LC_COLLATE LC_TIME LC_MESSAGES LC_MONETARY LC_ADDRESS LC_IDENTIFICATION LC_MEASUREMENT LC_PAPER LC_TELEPHONE LC_NAME LANG 2>/dev/null # perl, calm down -PATH=$MICROVM_CLOSURE/sw/bin - -echo "=> Initializing microVM environment" - -mkdir /run/log - -# Prepare mounts for paths that activation writes to -mount -t tmpfs tmpfs /etc -cp /run/muvm-host/etc/resolv.conf /etc/resolv.conf -mount -t tmpfs tmpfs /usr/bin -cp /run/muvm-host/usr/bin/munix-init-user /usr/bin/munix-init-user -mount -t tmpfs tmpfs /dev/shm # tries to remount, fails on virtiofs due to unexpected opts - -$MICROVM_CLOSURE/activate >/run/log/activate.spam 2>&1 - -umount /dev/shm # restore the original virtiofs dax shm though - -chown $MICROVM_UID:$MICROVM_GID /run /dev # avoid "Detected unsafe path transition" -systemd-tmpfiles --create >/run/log/tmpfiles.spam 2>&1 - -systemd-machine-id-setup >/dev/null 2>&1 - -$MICROVM_CLOSURE/sw/lib/systemd/systemd-udevd >/run/log/udevd.spam 2>&1 & - -echo "=> microVM environment ready!" diff --git a/munix-init-user b/munix-init-user deleted file mode 100755 index 2e7b672..0000000 --- a/munix-init-user +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -PATH=$MICROVM_CLOSURE/sw/bin -mkdir -p ~/.var/log -wayland-proxy-virtwl --virtio-gpu >~/.var/log/wayland-proxy.log 2>&1 & diff --git a/muvm b/muvm index e2512d1..d6c7496 160000 --- a/muvm +++ b/muvm @@ -1 +1 @@ -Subproject commit e2512d119e2d3959ed06afd5f586af65de1fe0a7 +Subproject commit d6c7496fdb40d825f5fdd0777f1e613432c5a1b2