Introduce micro-activate (RIIR activate script + tiny bit of tmpfiles)

Instead of interpreting all that shell and running actual tmpfiles, use
a tiny stage before systemd that mounts a tmpfs at /run (preventing
systemd from doing the same), populates it with NixOS symlinks and
preserved resolv.conf, and mounts the immutable /etc overlay before
passing control over to systemd.

.gitignore

@@ -1,3 +1,5 @@
 result
 /testvm*
+/target
+/micro-activate
 .direnv/

micro-activate.rs

@@ -0,0 +1,124 @@
+use std::os::raw::{c_char, c_int, c_ulong, c_void};
+use std::os::unix::ffi::OsStrExt;
+use std::os::unix::process::CommandExt;
+
+const MS_RDONLY: c_ulong = 0x01;
+const MS_NOSUID: c_ulong = 0x02;
+const MS_NODEV: c_ulong = 0x04;
+const MS_RELATIME: c_ulong = 0x200000;
+const MS_STRICTATIME: c_ulong = 0x1000000;
+
+unsafe extern "C" {
+    fn mount(
+        src: *const c_char,
+        target: *const c_char,
+        fstype: *const c_char,
+        flags: c_ulong,
+        data: *const c_void,
+    ) -> c_int;
+}
+
+fn parse_tmpfiles_line(line: &str) -> Option<(&str, &str)> {
+    // NOTE: does not support actual whitespace inside quotes
+    // (that's not gonna appear in these files we parse)
+    let mut it = line
+        .split_whitespace()
+        .map(|s| s.trim_start_matches('\'').trim_end_matches('\''));
+    let instr = it.next()?;
+    if !instr.starts_with('L') {
+        return None;
+    }
+    let src = it.next()?;
+    let _ = it.next()?;
+    let _ = it.next()?;
+    let _ = it.next()?;
+    let _ = it.next()?;
+    let dst = it.next()?;
+    Some((src, dst))
+}
+
+fn link_tmpfiles(contents: &[u8]) -> Result<(), std::io::Error> {
+    for (src, dst) in str::from_utf8(contents)
+        .unwrap()
+        .lines()
+        .flat_map(parse_tmpfiles_line)
+    {
+        std::os::unix::fs::symlink(dst, src)?;
+    }
+    Ok(())
+}
+
+fn main() -> Result<(), std::io::Error> {
+    let closure = std::env::var("MICROVM_CLOSURE").unwrap();
+
+    // systemd really wants /run to be a mountpoint and will mount a tmpfs on its own
+    // if it's not already a mountpoint. Well, it's correct: reaching into virtiofs
+    // (which is what not-mounting would entail) for /run stuff is not great.
+    //
+    // Let's preserve the fixed passed-in files and set up the NixOS symlinks in the new mount.
+    let resolv_conf = std::fs::read("/run/resolv.conf")?;
+    let machine_id = std::fs::read("/run/machine-id")?;
+    assert_eq!(
+        unsafe {
+            mount(
+                c"tmpfs".as_ptr(),
+                c"/run".as_ptr(),
+                c"tmpfs".as_ptr(),
+                MS_NOSUID | MS_NODEV | MS_STRICTATIME,
+                std::ptr::null(),
+            )
+        },
+        0
+    );
+    std::fs::write("/run/resolv.conf", &resolv_conf)?;
+    std::fs::write("/run/machine-id", &machine_id)?;
+    std::os::unix::fs::symlink(&closure, "/run/current-system")?;
+    if let Ok(tmp_graphics) =
+        std::fs::read(format!("{closure}/etc/tmpfiles.d/graphics-driver.conf"))
+    {
+        link_tmpfiles(&tmp_graphics)?;
+    } else {
+        eprintln!("[micro-activate] Could not find the closure's graphics-driver.conf!");
+    }
+
+    // We need the /etc metadata overlay not just for abstract correctness, but even just to
+    // allow the regular user to run systemctl (it doesn't like passwd being owned by non-root)..
+    let metadata_img = std::fs::read_link(format!("{closure}/etc-metadata-image"))
+        .expect("The closure must use an immutable /etc overlay!");
+    let basedir = std::fs::read_link(format!("{closure}/etc-basedir"))
+        .expect("The closure must use an immutable /etc overlay!");
+    let overlay_opts = std::ffi::CString::new(format!(
+        "redirect_dir=on,metacopy=on,lowerdir=/run/etc.meta::{}",
+        basedir.display()
+    ))
+    .unwrap();
+    std::fs::create_dir("/run/etc.meta")?;
+    std::fs::remove_file("/etc")?;
+    std::fs::create_dir("/etc")?;
+    unsafe {
+        assert_eq!(
+            mount(
+                metadata_img.as_os_str().as_bytes().as_ptr(),
+                c"/run/etc.meta".as_ptr(),
+                c"erofs".as_ptr(),
+                MS_RDONLY | MS_NODEV | MS_NOSUID,
+                std::ptr::null(),
+            ),
+            0
+        );
+        assert_eq!(
+            mount(
+                c"overlay".as_ptr(),
+                c"/etc".as_ptr(),
+                c"overlay".as_ptr(),
+                MS_NODEV | MS_NOSUID | MS_RELATIME,
+                overlay_opts.as_ptr() as *const c_void,
+            ),
+            0
+        );
+    }
+
+    let mut args = std::env::args_os().skip(1);
+    let cmd = args.next().unwrap();
+    Err(std::process::Command::new(cmd).args(args).exec())
+}

munix

@@ -180,38 +180,31 @@ BWRAP_ARGS+=(
 bwrap --unshare-all --share-net \
   --uid $MICROVM_UID --gid $MICROVM_GID \
   --tmpfs / \
-  --dir /run --dir /var --symlink /run /var/run --dir /tmp --dir /mnt \
+  --dir /run --dir /var --symlink /run /var/run --dir /tmp --dir /mnt --dir /bin --dir /usr/bin \
   --proc /proc --ro-bind /sys /sys \
   --dev /dev --dir /dev/input --dev-bind /dev/kvm /dev/kvm \
   --ro-bind "$MUVM_PATH" /run/munix/muvm \
   --ro-bind "$PASST_PATH" /run/munix/passt \
+  --ro-bind "$SCRIPT_PATH/micro-activate" /opt/bin/micro-activate \
   --ro-bind "$MUVM_PATH/muvm-guest" /opt/bin/muvm-remote \
   --ro-bind "$MUVM_PATH/muvm-guest" /opt/bin/muvm-configure-network \
   --ro-bind "$MUVM_PATH/muvm-guest" /opt/bin/muvm-pwbridge \
-  --symlink "$MICROVM_CLOSURE/etc/systemd" /etc/systemd \
+  --symlink "$MICROVM_CLOSURE/etc" /etc \
+  --symlink "$MICROVM_CLOSURE/sw/bin/sh" /bin/sh \
+  --symlink "$MICROVM_CLOSURE/sw/bin/env" /usr/bin/env \
+  --symlink "$MICROVM_CLOSURE" /run/current-system \
   --ro-bind /nix/store /nix/store \
-  --ro-bind /run/systemd/resolve /run/systemd/resolve \
-  --file 11 /etc/passwd \
-  --file 12 /etc/group \
-  --file 13 /etc/resolv.conf \
+  --file 12 /run/machine-id \
+  --file 13 /run/resolv.conf \
   --dir "$XDG_RUNTIME_DIR" \
   --setenv PATH "/run/munix/muvm:/run/munix/passt:$MICROVM_CLOSURE/sw/bin" \
   "${BWRAP_ARGS[@]}" \
   muvm \
-    --custom-init-cmdline "$MICROVM_CLOSURE/sw/sbin/init --log-target=console systemd.set_credential=sidebus.port:50000" \
+    --custom-init-cmdline "/opt/bin/micro-activate $MICROVM_CLOSURE/sw/sbin/init --log-target=console systemd.set_credential=sidebus.port:50000" \
     "${MUVM_ARGS[@]}" \
     -e container=munix \
     -e MICROVM_CLOSURE="$MICROVM_CLOSURE" \
     -e MICROVM_UID="$MICROVM_UID" -e MICROVM_GID="$MICROVM_GID" \
     -i -t "${MICROVM_COMMAND[@]}" \
-  11< <(cat <<EOF
-munix:x:$MICROVM_UID:$MICROVM_GID:Hypervisor:/:/run/current-system/sw/bin/nologin
-nobody:x:65534:65534:Unprivileged account:/var/empty:/run/current-system/sw/bin/nologin
-EOF
-) \
-  12< <(cat <<EOF
-munix:x:$MICROVM_GID:
-nogroup:x:65534:
-EOF
-) \
+  12< /etc/machine-id \
   13< /etc/resolv.conf

nixosModules/default.nix

@@ -81,7 +81,6 @@ in {
       "systemd-udevd-kernel.socket"
       "systemd-udevd-control.socket"
       "systemd-udevd.service"
-      "systemd-tmpfiles-setup.service"
       "user.slice"
     ];
     upstreamWants = ["multi-user.target.wants"];
@@ -99,24 +98,6 @@ in {
   systemd.services.systemd-pstore.enable = lib.mkForce false;
   systemd.services.lastlog2-import.enable = lib.mkForce false;
   systemd.services.suid-sgid-wrappers.enable = lib.mkForce false;
-  systemd.services.microvm-nixos-activation = {
-    enable = true;
-    description = "NixOS Activation";
-    wantedBy = ["local-fs.target"];
-    before = ["systemd-tmpfiles-setup.service"];
-    requires = ["systemd-tmpfiles-setup.service"];
-    unitConfig.DefaultDependencies = false;
-    serviceConfig = {
-      Type = "oneshot";
-      PassEnvironment = ["MICROVM_CLOSURE" "MICROVM_UID" "MICROVM_GID"];
-    } // useTTY;
-    script = ''
-      PATH=$MICROVM_CLOSURE/sw/bin
-      cp /etc/resolv.conf /run/
-      $MICROVM_CLOSURE/activate || true
-      chown 1337:1337 /run
-    '';
-  };
 
   # Configure user accounts
   # The immutable overlay wants userborn or sysusers.. we just want baked-in files w/o running a service.
@@ -139,7 +120,6 @@ in {
   };
   users.groups.appvm.gid = 1337;
   users.allowNoPasswordLogin = true;
-  systemd.tmpfiles.rules = ["d ${runtimeDir} 0755 1337 1337 -"];
 
   # Configure services
 
@@ -148,7 +128,6 @@ in {
     description = "microVM Application runner";
     onFailure = ["exit.target"];
     onSuccess = ["exit.target"];
-    after = ["microvm-nixos-activation.service"];
     wantedBy = ["microvm.target"];
     serviceConfig = {
       Type = "exec";

packages/munix/default.nix

@@ -1,12 +1,25 @@
-{ writeScriptBin, symlinkJoin, makeWrapper, muvm, passt, bubblewrap, sidebus-broker, mesa }:
+{ stdenv, writeScriptBin, symlinkJoin, makeWrapper, muvm, passt, bubblewrap, sidebus-broker, mesa, rustc }:
 
 let
   munixScript = (writeScriptBin "munix" (builtins.readFile ../../munix)).overrideAttrs(old: {
     buildCommand = "${old.buildCommand}\n patchShebangs $out";
   });
+  microActivate = stdenv.mkDerivation {
+    name = "micro-activate";
+    src = ../../micro-activate.rs;
+    dontUnpack = true;
+    nativeBuildInputs = [ rustc ];
+    buildPhase = ''
+      rustc -C opt-level=s -C panic=abort --edition 2024 -o micro-activate $src
+    '';
+    installPhase = ''
+      mkdir -p $out/bin
+      mv micro-activate $out/bin
+    '';
+  };
 in symlinkJoin {
   name = "munix";
-  paths = [ munixScript muvm passt bubblewrap sidebus-broker ];
+  paths = [ munixScript microActivate muvm passt bubblewrap sidebus-broker ];
   buildInputs = [ makeWrapper ];
   postBuild = ''
     wrapProgram $out/bin/munix --prefix PATH : $out/bin --set FALLBACK_OPENGL_DRIVER ${mesa}