From 4602228be9fc5d6a27f4e40775c1237c4d30f7b2 Mon Sep 17 00:00:00 2001 From: Val Packett Date: Thu, 25 Sep 2025 22:29:24 -0300 Subject: [PATCH] Initial commit --- .envrc | 1 + .gitignore | 2 + .gitmodules | 6 ++ LICENSE.md | 18 ++++++ README.md | 24 +++++++ flake.lock | 123 +++++++++++++++++++++++++++++++++++ flake.nix | 167 ++++++++++++++++++++++++++++++++++++++++++++++++ libkrun | 1 + munix | 35 ++++++++++ munix-init-root | 27 ++++++++ munix-init-user | 4 ++ muvm | 1 + radvmmio.patch | 42 ++++++++++++ 13 files changed, 451 insertions(+) create mode 100644 .envrc create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 160000 libkrun create mode 100755 munix create mode 100755 munix-init-root create mode 100755 munix-init-user create mode 160000 muvm create mode 100644 radvmmio.patch diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4812d58 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +result +.direnv/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..af2cfdd --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "muvm"] + path = muvm + url = https://github.com/valpackett/muvm +[submodule "libkrun"] + path = libkrun + url = https://github.com/valpackett/libkrun diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..b08d8ca --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,18 @@ +Copyright 2025 Invisible Things Lab, Clan contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6a770f9 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# munix + +WIP: A microVM runner for NixOS systems with desktop integration, powered by muvm/libkrun. + +## Development + +Building an example closure: + +``` +nix build '.#nixosConfigurations.x86_64-linux.testvm.config.system.build.toplevel' +``` + +Running the nix build: + +``` +nix run '.#packages.x86_64-linux.munix' $(readlink result) +``` + +Working on muvm & munix locally (not built into the nix store): + +``` +cd muvm && cargo build --locked --release +PATH=$PWD/muvm/target/release:$PATH ./munix $(readlink result) +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..becdabc --- /dev/null +++ b/flake.lock @@ -0,0 +1,123 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1758078198, + "narHash": "sha256-60ojlXp42UZYTOAgsiKPZUab1Aa8pXTIo+rz0zoaynI=", + "ref": "val/tsvwswkqrrsr", + "rev": "1f0fc70eb049852a18a5203af2204bd9f5729f29", + "shallow": true, + "submodules": true, + "type": "git", + "url": "https://github.com/valpackett/nixpkgs" + }, + "original": { + "ref": "val/tsvwswkqrrsr", + "shallow": true, + "submodules": true, + "type": "git", + "url": "https://github.com/valpackett/nixpkgs" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "virtwl": "virtwl" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "virtwl": { + "inputs": { + "flake-utils": "flake-utils_2", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1758075582, + "narHash": "sha256-o2lpXQLaM9QcZVr+sAxvh83CqJW1QkFhfja6K40ndmA=", + "ref": "wip", + "rev": "5a5df73a11b2f6bf671a8fc89926ac993e0fbb78", + "shallow": true, + "submodules": true, + "type": "git", + "url": "https://github.com/valpackett/wayland-proxy-virtwl" + }, + "original": { + "ref": "wip", + "shallow": true, + "submodules": true, + "type": "git", + "url": "https://github.com/valpackett/wayland-proxy-virtwl" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..e3b9182 --- /dev/null +++ b/flake.nix @@ -0,0 +1,167 @@ +{ + inputs = { + self.submodules = true; + # nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + # https://github.com/NixOS/nixpkgs/pull/444133 + nixpkgs.url = "git+https://github.com/valpackett/nixpkgs?shallow=1&submodules=1&ref=val/tsvwswkqrrsr"; + flake-utils.url = "github:numtide/flake-utils"; + virtwl.url = "git+https://github.com/valpackett/wayland-proxy-virtwl?shallow=1&submodules=1&ref=wip"; + virtwl.inputs.nixpkgs.follows = "nixpkgs"; + }; + + outputs = {self, nixpkgs, flake-utils, virtwl, ...}: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + }; + in { + + packages.libkrunfw = pkgs.libkrunfw; + # packages.libkrunfw = (pkgs.libkrunfw.overrideAttrs { + # version = "4.10.0"; + # src = pkgs.fetchFromGitHub { + # owner = "containers"; + # repo = "libkrunfw"; + # tag = "v4.10.0"; + # hash = "sha256-mq2gw0+xL6qUZE/fk0vLT3PEpzPV8p+iwRFJHXVOMnk="; + # }; + # kernelSrc = pkgs.fetchurl { + # url = "mirror://kernel/linux/kernel/v6.x/linux-6.12.34.tar.xz"; + # hash = "sha256-p/P+OB9n7KQXLptj77YaFL1/nhJ44DYD0P9ak/Jwwk0="; + # }; + # cargoHash = ""; + # }); + + packages.libkrun = (pkgs.libkrun.override { + withBlk = true; + withGpu = true; + withSound = true; + withNet = true; + libkrunfw = self.packages.${system}.libkrunfw; + }).overrideAttrs (old: { + src = ./libkrun; + cargoDeps = pkgs.rustPlatform.importCargoLock { + lockFile = ./libkrun/Cargo.lock; + }; + # mesonFlags = [ (pkgs.lib.mesonOption "decoders" "gles,vulkan,composer") ]; # no magma(?) + }); + + packages.muvm = (pkgs.muvm.override { + libkrun = self.packages.${system}.libkrun; + }).overrideAttrs (old: { + postPatch = ""; # no more sysctl; udevd now takes the var anyway; XXX: fex + MUVM_UDEVD_PATH = "${pkgs.systemd}/lib/systemd/systemd-udevd"; + src = ./muvm; + cargoDeps = pkgs.rustPlatform.importCargoLock { + lockFile = ./muvm/Cargo.lock; + }; + }); + + packages.munix = let + munixScript = (pkgs.writeScriptBin "munix" (builtins.readFile ./munix)).overrideAttrs(old: { + buildCommand = "${old.buildCommand}\n patchShebangs $out"; + }); + munixInitRootHook = (pkgs.writeScriptBin "munix-init-root" (builtins.readFile ./munix-init-root)).overrideAttrs(old: { + buildCommand = "${old.buildCommand}\n patchShebangs $out"; + }); + munixInitUserHook = (pkgs.writeScriptBin "munix-init-user" (builtins.readFile ./munix-init-user)).overrideAttrs(old: { + buildCommand = "${old.buildCommand}\n patchShebangs $out"; + }); + in pkgs.symlinkJoin { + name = "munix"; + paths = [ munixScript munixInitRootHook munixInitUserHook self.packages.${system}.muvm pkgs.passt pkgs.bubblewrap ]; + buildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + wrapProgram $out/bin/munix --prefix PATH : $out/bin + ''; + }; + + devShells.default = let + projects = with self.packages.${system}; [ libkrun muvm ]; + in pkgs.mkShell { + MUVM_UDEVD_PATH = "${pkgs.systemd}/lib/systemd/systemd-udevd"; + nativeBuildInputs = pkgs.lib.concatMap (pkg: pkg.nativeBuildInputs) projects; + buildInputs = (pkgs.lib.concatMap (pkg: pkg.buildInputs) projects) ++ (with self.packages.${system}; [ + # virglrenderer + ]) ++ (with pkgs; [ + cargo + rust-analyzer + rustfmt + passt + bubblewrap + ]); + }; + + packages.mesa = (pkgs.mesa.override { + vulkanDrivers = [ + "amd" + "intel" + "microsoft-experimental" # removing this breaks the build + "nouveau" + "swrast" + "virtio" + "gfxstream" # probably not going to use this though + ]; + }).overrideAttrs (new: old: { + mesonFlags = old.mesonFlags ++ [ (pkgs.lib.mesonBool "amdgpu-virtio" true) ]; + patches = old.patches ++ [ ./radvmmio.patch ]; # already merged to git + }); + + nixosConfigurations.testvm = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + { + system.stateVersion = "25.11"; + fileSystems."/".device = pkgs.lib.mkDefault "/dev/sda"; + boot.isContainer = true; + + users.mutableUsers = false; + users.users.appvm = { + uid = 1001; + isNormalUser = true; + home = "/home/appvm"; + description = "microVM User"; + extraGroups = [ "wheel" "video" "input" ]; + }; + users.groups.appvm.gid = 1001; + users.allowNoPasswordLogin = true; + + hardware.graphics.enable = true; + hardware.graphics.package = self.packages.${system}.mesa; + system.replaceDependencies.replacements = [ + { + original = pkgs.mesa; + replacement = self.packages.${system}.mesa; + } + ]; + environment.systemPackages = [ + pkgs.fastfetch + pkgs.htop + pkgs.radeontop + virtwl.packages.x86_64-linux.proxy + pkgs.wayland-utils + pkgs.weston + pkgs.waycheck + pkgs.vulkan-tools + pkgs.glxinfo + pkgs.glmark2 + pkgs.mesa-demos + pkgs.xorg.xeyes + pkgs.xterm + pkgs.vkquake + pkgs.kdePackages.kate + pkgs.adwaita-fonts + pkgs.adwaita-icon-theme + pkgs.gnome-text-editor + pkgs.firefox + pkgs.ffmpeg-full + pkgs.mpv + pkgs.libva-utils + ]; + } + ]; + }; + + }); +} diff --git a/libkrun b/libkrun new file mode 160000 index 0000000..bd97a39 --- /dev/null +++ b/libkrun @@ -0,0 +1 @@ +Subproject commit bd97a39bfaa46147933beca7e16513b6820bb031 diff --git a/munix b/munix new file mode 100755 index 0000000..63aa305 --- /dev/null +++ b/munix @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +: "${MICROVM_SHELL:=bash}" +SCRIPT_PATH=$(dirname $(realpath -s $0)) +MUVM_PATH=$(dirname $(which muvm)) +PASST_PATH=$(dirname $(which passt)) +export TMP=/tmp TMPDIR=/tmp TEMP=/tmp TEMPDIR=/tmp LC_ALL=C +unset DISPLAY XAUTHORITY # or: --bind /tmp/.X11-unix /tmp/.X11-unix --bind $XAUTHORITY $XAUTHORITY +exec bwrap --unshare-all --share-net \ + --uid 1001 --gid 1001 \ + --tmpfs / \ + --dir /run --dir /var --symlink /run /var/run --dir /tmp \ + --proc /proc --ro-bind /sys /sys \ + --dev /dev --dir /dev/input --dev-bind /dev/kvm /dev/kvm --dev-bind /dev/dri /dev/dri \ + --ro-bind "$MUVM_PATH" /run/munix/muvm \ + --ro-bind "$PASST_PATH" /run/munix/passt \ + --ro-bind "$1/sw/bin/env" /usr/bin/env \ + --ro-bind "$SCRIPT_PATH/munix-init-root" /usr/bin/munix-init-root \ + --ro-bind "$SCRIPT_PATH/munix-init-user" /usr/bin/munix-init-user \ + --ro-bind /nix/store /nix/store \ + --ro-bind /run/opengl-driver /run/opengl-driver \ + --ro-bind /run/systemd/resolve /run/systemd/resolve \ + --ro-bind /etc/resolv.conf /etc/resolv.conf \ + --ro-bind /etc/group /etc/group \ + --ro-bind /etc/passwd /etc/passwd \ + --dir "$XDG_RUNTIME_DIR" \ + --bind "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" \ + --bind "$XDG_RUNTIME_DIR/pipewire-0" "$XDG_RUNTIME_DIR/pipewire-0" \ + --bind $HOME/Downloads/baseq1 $HOME/Downloads/baseq1 \ + --setenv WAYLAND_DISPLAY "$WAYLAND_DISPLAY" \ + --setenv PATH "/run/munix/muvm:/run/munix/passt:$1/sw/bin" \ + muvm \ + -x /usr/bin/munix-init-root -X /usr/bin/munix-init-user --udevd-path="$1/sw/bin/true" \ + -e WAYLAND_DISPLAY=wayland-1 \ + -e MICROVM_CLOSURE="$1" \ + -i -t "$1/sw/bin/$MICROVM_SHELL" diff --git a/munix-init-root b/munix-init-root new file mode 100755 index 0000000..4a166a9 --- /dev/null +++ b/munix-init-root @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +unset LANGUAGE LC_ALL LC_CTYPE LC_NUMERIC LC_COLLATE LC_TIME LC_MESSAGES LC_MONETARY LC_ADDRESS LC_IDENTIFICATION LC_MEASUREMENT LC_PAPER LC_TELEPHONE LC_NAME LANG 2>/dev/null # perl, calm down +PATH=$MICROVM_CLOSURE/sw/bin + +echo "=> Initializing microVM environment" + +mkdir /run/log + +# Prepare mounts for paths that activation writes to +mount -t tmpfs tmpfs /etc +cp /run/muvm-host/etc/resolv.conf /etc/resolv.conf +mount -t tmpfs tmpfs /usr/bin +cp /run/muvm-host/usr/bin/munix-init-user /usr/bin/munix-init-user +mount -t tmpfs tmpfs /dev/shm # tries to remount, fails on virtiofs due to unexpected opts + +$MICROVM_CLOSURE/activate >/run/log/activate.spam 2>&1 + +umount /dev/shm # restore the original virtiofs dax shm though + +chown 1001:1001 /run /dev # avoid "Detected unsafe path transition" +systemd-tmpfiles --create >/run/log/tmpfiles.spam 2>&1 + +systemd-machine-id-setup >/dev/null 2>&1 + +$MICROVM_CLOSURE/sw/lib/systemd/systemd-udevd >/run/log/udevd.spam 2>&1 & + +echo "=> microVM environment ready!" diff --git a/munix-init-user b/munix-init-user new file mode 100755 index 0000000..2e7b672 --- /dev/null +++ b/munix-init-user @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +PATH=$MICROVM_CLOSURE/sw/bin +mkdir -p ~/.var/log +wayland-proxy-virtwl --virtio-gpu >~/.var/log/wayland-proxy.log 2>&1 & diff --git a/muvm b/muvm new file mode 160000 index 0000000..70badbc --- /dev/null +++ b/muvm @@ -0,0 +1 @@ +Subproject commit 70badbc44d421c49f82130321e35b949f8b4ed14 diff --git a/radvmmio.patch b/radvmmio.patch new file mode 100644 index 0000000..c3ff0bf --- /dev/null +++ b/radvmmio.patch @@ -0,0 +1,42 @@ +From c2fd030644cf2074f9d2ffd155839b2e943473d0 Mon Sep 17 00:00:00 2001 +From: Val Packett +Date: Wed, 10 Sep 2025 15:44:41 -0300 +Subject: [PATCH] radv: detect platform:virtio-mmio devices for virtgpu native + context + +VirtIO devices can be configured as platform devices instead of PCI, +which is especially common in microVM projects like libkrun. + +Let's allow RADV to probe MMIO virtgpu devices. + +Signed-off-by: Val Packett +--- + src/amd/vulkan/radv_physical_device.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/src/amd/vulkan/radv_physical_device.c b/src/amd/vulkan/radv_physical_device.c +index 07b5b000b3ae..cd6f4c3488a1 100644 +--- a/src/amd/vulkan/radv_physical_device.c ++++ b/src/amd/vulkan/radv_physical_device.c +@@ -2507,14 +2507,15 @@ create_drm_physical_device(struct vk_instance *vk_instance, struct _drmDevice *d + #ifndef _WIN32 + bool supported_device = false; + +- if (!(device->available_nodes & (1 << DRM_NODE_RENDER)) || device->bustype != DRM_BUS_PCI) ++ if (!(device->available_nodes & (1 << DRM_NODE_RENDER))) + return VK_ERROR_INCOMPATIBLE_DRIVER; + + #ifdef HAVE_AMDGPU_VIRTIO +- supported_device |= device->deviceinfo.pci->vendor_id == VIRTGPU_PCI_VENDOR_ID; ++ supported_device |= device->bustype == DRM_BUS_PCI && device->deviceinfo.pci->vendor_id == VIRTGPU_PCI_VENDOR_ID; ++ supported_device |= device->bustype == DRM_BUS_PLATFORM; /* virtio-mmio */ + #endif + +- supported_device |= device->deviceinfo.pci->vendor_id == ATI_VENDOR_ID; ++ supported_device |= device->bustype == DRM_BUS_PCI && device->deviceinfo.pci->vendor_id == ATI_VENDOR_ID; + + if (!supported_device) + return VK_ERROR_INCOMPATIBLE_DRIVER; +-- +2.50.1 +